Fix unescaped commas in generated ArgumentParser completions (#1066 )

* Fix unescaped commas in generated ArgumentParser completions * Improve completion hints
Update docs, examples and CI to Sequoia
2026-10-01 19:51:10 +02:00 · 2025-05-06 14:24:43 +04:00 · 2025-05-04 20:49:23 -04:00 · 2025-04-29 21:15:03 +04:00 · 2025-04-29 18:10:14 +04:00 · 2025-04-22 10:17:19 -04:00
76 changed files with 2417 additions and 474 deletions
@@ -3,3 +3,5 @@
 TMPFILE=$(mktemp)
 envsubst < Sources/tart/CI/CI.swift > $TMPFILE
 mv $TMPFILE Sources/tart/CI/CI.swift
+
+/usr/libexec/PlistBuddy -c "Add :CFBundleShortVersionString string ${CIRRUS_TAG}" Resources/Info.plist
@@ -1,7 +1,7 @@
 use_compute_credits: true

 task:
-  name: Test on Sonoma
+  name: Test on Sequoia
  alias: test
  persistent_worker:
    labels:
@@ -17,8 +17,7 @@ task:
    - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
    # Run integration tests
    - cd integration-tests
-    - HOMEBREW_NO_AUTO_UPDATE=1 brew install virtualenv
-    - virtualenv venv
+    - python3 -m venv --symlinks venv
    - source venv/bin/activate
    - pip install -r requirements.txt
    - pytest --verbose --junit-xml=pytest-junit.xml
@@ -38,7 +37,7 @@ task:
  name: Lint
  alias: lint
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
  lint_script:
    - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
  always:
@@ -55,7 +54,7 @@ task:
  name: Build ($BUILD_ARCH)
  alias: build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
  build_script: swift build --arch $BUILD_ARCH --product tart
  sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/$BUILD_ARCH-apple-macosx/debug/tart
  binary_artifacts:
@@ -68,7 +67,7 @@ task:
    - lint
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -103,7 +102,7 @@ task:
    - test
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -121,7 +120,7 @@ task:
    - security import certificate.p12 -k build.keychain -P password101 -T /usr/bin/codesign -T /usr/bin/pkgbuild
    - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
    - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
-  install_script: 
+  install_script:
    - brew install go goreleaser/tap/goreleaser-pro getsentry/tools/sentry-cli
    - brew install mitchellh/gon/gon
  info_script:
@@ -153,7 +152,7 @@ task:
    registry_config: ENCRYPTED[!cf1a0f25325aa75bad3ce6ebc890bc53eb0044c02efa70d8cefb83ba9766275a994b4831706c52630a0692b2fa9cfb9e!]
  env:
    DEPLOY_TOKEN: ENCRYPTED[!45ed45666558902ed1c2400add734ec063103bec31841847e8c8764802fca229bfa6d85c690e16ad159e047574b48793!]
-  deploy_script: 
+  deploy_script:
    - git config --global user.name "Cirrus CI"
    - git config --global user.name "hello@cirruslabs.org"
    - git remote set-url origin https://$DEPLOY_TOKEN@github.com/cirruslabs/tart/
@@ -1,11 +1,12 @@
+version: 2
+
 project_name: tart

 before:
  hooks:
    - .ci/set-version.sh
-    - swift build --arch x86_64 -c release --product tart
-    - swift build --arch arm64 -c release --product tart
-    - gon gon.hcl
+    - swift build --arch arm64 --configuration release --product tart
+    - swift build --arch x86_64 --configuration release --product tart

 builds:
  - id: tart
@@ -20,8 +21,14 @@ builds:
    prebuilt:
      path: '.build/{{- if eq .Arch "arm64" }}arm64{{- else }}x86_64{{ end }}-apple-macosx/release/tart'

+universal_binaries:
+  - name_template: tart.app/Contents/MacOS/tart
+    replace: true
+    hooks:
+      post: gon gon.hcl
+
 archives:
-  - name_template: "{{ .ProjectName }}-{{ .Arch }}"
+  - name_template: "{{ .ProjectName }}"
    files:
      - src: Resources/embedded.provisionprofile
        dst: tart.app/Contents
@@ -42,7 +49,13 @@ brews:
    repository:
      owner: cirruslabs
      name: homebrew-cli
-    caveats: See the GitHub repository for more information
+    caveats: |
+      Tart has been installed. You might want to reduce the default DHCP lease time
+      from 86,400 to 600 seconds to avoid DHCP shortage when running lots of VMs daily:
+
+        sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.InternetSharing.default.plist bootpd -dict DHCPLeaseTimeSecs -int 600
+
+      See https://tart.run/faq/#changing-the-default-dhcp-lease-time for more details.
    homepage: https://github.com/cirruslabs/tart
    license: "Fair Source"
    description: Run macOS and Linux VMs on Apple Hardware
@@ -0,0 +1,64 @@
+# Profiling Tart
+
+## Using `time(1)`
+
+Perhaps, the easiest, but not the most comprehensive way to tell what's going on with Tart is to use the [`time(1)`](https://ss64.com/mac/time.html) command.
+
+In the example below, you will run `tart pull` via `time(1)` to gather generalized CPU, I/O and memory usage metrics:
+
+```shell
+/usr/bin/time -l tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+**Note:** you need to specify a full path to `time(1)` binary, otherwise the shell's built-in `time` command will be invoked, which doesn't have the `-l` command-line argument.
+
+**Note:** The `-l` command-line argument makes `time(1)` return much more useful information, for example, maximum memory usage.
+
+When running the command above, you'll see the `tart pull` output first as it pulls the image, and then the `time(1)` output, which will be printed once the Tart process finishes:
+
+```
+      172.17 real        10.29 user         8.36 sys
+           353796096  maximum resident set size
+                   0  average shared memory size
+                   0  average unshared data size
+                   0  average unshared stack size
+               23838  page reclaims
+                  35  page faults
+                   0  swaps
+                   0  block input operations
+                   0  block output operations
+                   8  messages sent
+                   8  messages received
+                   0  signals received
+                 146  voluntary context switches
+              222950  involuntary context switches
+         39683070975  instructions retired
+         27562035252  cycles elapsed
+           170920448  peak memory footprint
+```
+
+From the output above, you can tell that `tart pull` spent nearly 90% of time off-CPU (`real` > `user` + `sys`), which means that Tart was mostly waiting for the I/O (be it a network or disk), instead of decompressing disk layers or doing other useful computations.
+
+## Using `xctrace(1)`
+
+[`xctrace(1)`](https://keith.github.io/xcode-man-pages/xctrace.1.html) is a `.trace` format recorder for the [Instruments](https://en.wikipedia.org/wiki/Instruments_(software)) app, which yields much more powerful insights compared to `time(1)`. For example, it can tell which Tart functions spent the most time on the CPU, thus allowing the Tart developers to further optimize these functions.
+
+To use it, make sure that [Xcode](https://developer.apple.com/xcode/resources/) is installed. If you're installing Xcode for the first time on the machine, you'll need to launch it once and click the blue "Install" button. There's no need to choose any platforms except for the macOS.
+
+Once done, you can create a CPU profile of `tart pull`:
+
+```shell
+xctrace record --template "CPU Profiler" --target-stdout - --launch -- /opt/homebrew/bin/tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+Now that `xctrace(1)` is running, you'll see the `tart pull`-related output first, and once finished, the following line will appear:
+
+```
+Output file saved as: Launch_[...].trace
+```
+
+To view this trace in the Instruments app, simply find this directory in Finder and double-click it. Instruments app will appear:
+
+![](Resources/Instruments.png)
+
+To send this trace, right-click its directory in Finder and choose "Compress [...]". This will result in a similarly named file with a `.zip` at the end, which can now be conveniently sent via email or uploaded.
@@ -1,13 +1,13 @@
 {
-  "originHash" : "2c514a4a1d7e106713db744bee89edb40d75da63e6611990ec2f4b0da53c0455",
+  "originHash" : "22b3726bc4e4c6e9c04ac97cb08a82967feb39960a93d2909768a16e11576748",
  "pins" : [
    {
      "identity" : "antlr4",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/antlr/antlr4",
      "state" : {
-        "branch" : "dev",
-        "revision" : "2703a8516c0fb7fe92db6b9c40e0113f577646d2"
+        "revision" : "cc82115a4e7f53d71d9d905caa2c2dfa4da58899",
+        "version" : "4.13.2"
      }
    },
    {
@@ -24,8 +24,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/groue/Semaphore",
      "state" : {
-        "revision" : "f1c4a0acabeb591068dea6cffdd39660b86dec28",
-        "version" : "0.0.8"
+        "revision" : "2543679282aa6f6c8ecf2138acd613ed20790bc2",
+        "version" : "0.1.0"
      }
    },
    {
@@ -33,8 +33,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/getsentry/sentry-cocoa",
      "state" : {
-        "revision" : "ef4fec9dfb8dd5027b09a4a5c9362feafd118e1a",
-        "version" : "8.24.0"
+        "revision" : "5575af93efb776414f243e93d6af9f6258dc539a",
+        "version" : "8.36.0"
      }
    },
    {
@@ -51,17 +51,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-argument-parser",
      "state" : {
-        "revision" : "46989693916f56d1186bd59ac15124caef896560",
-        "version" : "1.3.1"
-      }
-    },
-    {
-      "identity" : "swift-async-algorithms",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-async-algorithms",
-      "state" : {
-        "branch" : "main",
-        "revision" : "f05e450f0b909c0e80670a47516c4b9700b9e5da"
+        "revision" : "41982a3656a71c768319979febd796c6fd111d5c",
+        "version" : "1.5.0"
      }
    },
    {
@@ -73,22 +64,13 @@
        "version" : "1.2.0"
      }
    },
-    {
-      "identity" : "swift-collections",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-collections.git",
-      "state" : {
-        "revision" : "f504716c27d2e5d4144fa4794b12129301d17729",
-        "version" : "1.0.3"
-      }
-    },
    {
      "identity" : "swift-log",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-log.git",
      "state" : {
-        "revision" : "e97a6fcb1ab07462881ac165fdbb37f067e205d5",
-        "version" : "1.5.4"
+        "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537",
+        "version" : "1.6.1"
      }
    },
    {
@@ -105,8 +87,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/fumoboy007/swift-retry",
      "state" : {
-        "revision" : "9f133487ffc2ab4539688c29efe57bb1ba31d7b0",
-        "version" : "0.2.3"
+        "revision" : "df9d7b185d2e433147ec0083a73c257e665eea0d",
+        "version" : "0.2.4"
      }
    },
    {
@@ -141,8 +123,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/nicklockwood/SwiftFormat",
      "state" : {
-        "revision" : "9df3b01f477163b33d5e63c5e2e5b9f946a49c56",
-        "version" : "0.53.6"
+        "revision" : "ab6844edb79a7b88dc6320e6cee0a0db7674dac3",
+        "version" : "0.54.5"
      }
    },
    {
@@ -13,12 +13,11 @@ let package = Package(
    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.1"),
    .package(url: "https://github.com/mhdhejazi/Dynamic", branch: "master"),
    .package(url: "https://github.com/apple/swift-algorithms", from: "1.2.0"),
-    .package(url: "https://github.com/apple/swift-async-algorithms", branch: "main"),
    .package(url: "https://github.com/malcommac/SwiftDate", from: "7.0.0"),
-    .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
+    .package(url: "https://github.com/antlr/antlr4", exact: "4.13.2"),
    .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.2.0")),
    .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.53.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.24.0"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.36.0"),
    .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.8.0"),
    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.1"),
@@ -29,7 +28,6 @@ let package = Package(
  targets: [
    .executableTarget(name: "tart", dependencies: [
      .product(name: "Algorithms", package: "swift-algorithms"),
-      .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
      .product(name: "ArgumentParser", package: "swift-argument-parser"),
      .product(name: "Dynamic", package: "Dynamic"),
      .product(name: "SwiftDate", package: "SwiftDate"),
@@ -66,8 +66,8 @@ Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura)

 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
-tart run sonoma-base
+tart clone ghcr.io/cirruslabs/macos-sequoia-base:latest sequoia-base
+tart run sequoia-base
 ```

 Please check the [official documentation](https://tart.run) for more information and/or feel free to use [discussions](https://github.com/cirruslabs/tart/discussions)
@@ -2,22 +2,26 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-    <key>CFBundleName</key>
-    <string>tart</string>
-    <key>CFBundleIdentifier</key>
-    <string>org.cirruslabs.tart</string>
-    <key>CFBundleExecutable</key>
-    <string>tart</string>
-    <key>LSBackgroundOnly</key>
-    <string>1</string>
-    <key>CFBundleIconFiles</key>
-    <array>
-        <string>AppIcon.png</string>
-    </array>
-    <key>NSAppTransportSecurity</key>
-    <dict>
-        <key>NSAllowsArbitraryLoads</key>
-        <true/>
-    </dict>
+	<key>CFBundleName</key>
+	<string>Tart</string>
+	<key>CFBundleDisplayName</key>
+	<string>Tart</string>
+	<key>CFBundleIdentifier</key>
+	<string>org.cirruslabs.tart</string>
+	<key>CFBundleExecutable</key>
+	<string>tart</string>
+  <key>LSApplicationCategoryType</key>
+  <string>public.app-category.developer-tools</string>
+	<key>CFBundleIconFiles</key>
+	<array>
+		<string>AppIcon.png</string>
+	</array>
+	<key>NSAppTransportSecurity</key>
+	<dict>
+		<key>NSAllowsArbitraryLoads</key>
+		<true/>
+	</dict>
+  <key>NSLocalNetworkUsageDescription</key>
+  <string>Access to OCI registries on the local network</string>
 </dict>
 </plist>
@@ -9,12 +9,10 @@ struct Clone: AsyncParsableCommand {
    Creates a local virtual machine by cloning either a remote or another local virtual machine.

    Due to copy-on-write magic in Apple File System, a cloned VM won't actually claim all the space right away.
-    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
-    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
-    will be modified.
+    Only changes to a cloned disk will be written and claim new space. This also speeds up clones enormously.

-    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable. This might be helpful
-    for use cases when the original image is very big and a workload is known to only modify a fraction of the cloned disk.
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the cloned image
+    to fit. This behaviour is called "automatic pruning" and can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
    """
  )

@@ -30,6 +28,9 @@ struct Clone: AsyncParsableCommand {
  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
  var concurrency: UInt = 4

+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
  func validate() throws {
    if newName.contains("/") {
      throw ValidationError("<new-name> should be a local name")
@@ -47,7 +48,7 @@ struct Clone: AsyncParsableCommand {
    if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
      // Pull the VM in case it's OCI-based and doesn't exist locally yet
      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
    }

    let sourceVM = try VMStorageHelper.open(sourceName)
@@ -70,10 +71,14 @@ struct Clone: AsyncParsableCommand {

      try lock.unlock()

-      // APFS is doing copy-on-write so the above cloning operation (just copying files on disk)
+      // APFS is doing copy-on-write, so the above cloning operation (just copying files on disk)
      // is not actually claiming new space until the VM is started and it writes something to disk.
-      // So once we clone the VM let's try to claim a little bit of space for the VM to run.
-      try Prune.reclaimIfNeeded(UInt64(sourceVM.allocatedSizeBytes()), sourceVM)
+      //
+      // So, once we clone the VM let's try to claim the rest of space for the VM to run without errors.
+      let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes()
+      if unallocatedBytes > 0 {
+        try Prune.reclaimIfNeeded(UInt64(unallocatedBytes), sourceVM)
+      }
    }, onCancel: {
      try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
    })
@@ -10,7 +10,7 @@ struct Create: AsyncParsableCommand {
  @Argument(help: "VM name")
  var name: String

-  @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file or URL (or \"latest\", to fetch the latest supported IPSW automatically)", valueName: "path"))
+  @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file or URL (or \"latest\", to fetch the latest supported IPSW automatically)", valueName: "path"), completion: .file())
  var fromIPSW: String?

  @Flag(help: "create a Linux VM")
@@ -7,7 +7,7 @@ struct Export: AsyncParsableCommand {
  @Argument(help: "Source VM name.", completion: .custom(completeMachines))
  var name: String

-  @Argument(help: "Path to the destination file.")
+  @Argument(help: "Path to the destination file.", completion: .file())
  var path: String?

  func run() async throws {
@@ -4,10 +4,10 @@ import Foundation
 struct Import: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Import VM from a compressed .tvm file")

-  @Argument(help: "Path to a file created with \"tart export\".")
+  @Argument(help: "Path to a file created with \"tart export\".", completion: .file())
  var path: String

-  @Argument(help: "Destination VM name.")
+  @Argument(help: "Destination VM name.", completion: .custom(completeLocalMachines))
  var name: String

  func validate() throws {
@@ -18,7 +18,7 @@ struct List: AsyncParsableCommand {
  @Option(help: ArgumentHelp("Only display VMs from the specified source (e.g. --source local, --source oci)."))
  var source: String?

-  @Option(help: "Output format: text or json")
+  @Option(help: "Output format: text or json", completion: .list(["text", "json"]))
  var format: Format = .text

  @Flag(name: [.short, .long], help: ArgumentHelp("Only display VM names."))
@@ -7,7 +7,7 @@ import SwiftDate
 struct Prune: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches or local VMs")

-  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."))
+  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."), completion: .list(["caches", "vms"]))
  var entries: String = "caches"

  @Option(help: ArgumentHelp("Remove entries that were last accessed more than n days ago",
@@ -9,8 +9,8 @@ struct Pull: AsyncParsableCommand {
    Pulls a virtual machine from a remote OCI-compatible registry. Supports authorization via Keychain (see "tart login --help"),
    Docker credential helpers defined in ~/.docker/config.json or via TART_REGISTRY_USERNAME/TART_REGISTRY_PASSWORD environment variables.

-    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image to fit via "tart prune".
-    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image
+    to fit. This behaviour is called "automatic pruning" and can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
    """
  )

@@ -23,6 +23,9 @@ struct Pull: AsyncParsableCommand {
  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
  var concurrency: UInt = 4

+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
  func validate() throws {
    if concurrency < 1 {
      throw ValidationError("network concurrency cannot be less than 1")
@@ -43,6 +46,6 @@ struct Pull: AsyncParsableCommand {

    defaultLogger.appendNewLine("pulling \(remoteName)...")

-    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
  }
 }
@@ -26,6 +26,11 @@ struct Push: AsyncParsableCommand {
                             """))
  var chunkSize: Int = 0

+
+  @Option(name: [.customLong("label")], help: ArgumentHelp("additional metadata to attach to the OCI image configuration in key=value format",
+                                                           discussion: "Can be specified multiple times to attach multiple labels."))
+  var labels: [String] = []
+
  @Option(help: .hidden)
  var diskFormat: String = "v2"

@@ -81,7 +86,8 @@ struct Push: AsyncParsableCommand {
          references: references,
          chunkSizeMb: chunkSize,
          diskFormat: diskFormat,
-          concurrency: concurrency
+          concurrency: concurrency,
+          labels: parseLabels()
        )
        // Populate the local cache (if requested)
        if populateCache {
@@ -115,6 +121,28 @@ struct Push: AsyncParsableCommand {
    return RemoteName(host: registry.host!, namespace: registry.namespace,
                      reference: Reference(digest: digest))
  }
+
+  // Helper method to convert labels array to dictionary
+  func parseLabels() -> [String: String] {
+    var result = [String: String]()
+
+    for label in labels {
+      let parts = label.trimmingCharacters(in: .whitespaces).split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false)
+
+      let key = parts.count > 0 ? String(parts[0]) : ""
+      let value = parts.count > 1 ? String(parts[1]) : ""
+
+      // It sometimes makes sense to provide an empty value,
+      // but not an empty key
+      if key.isEmpty {
+        continue
+      }
+
+      result[key] = value
+    }
+
+    return result
+  }  
 }

 extension Collection where Element == RemoteName {
@@ -45,6 +45,23 @@ extension VZDiskImageSynchronizationMode {
  }
 }

+extension VZDiskImageCachingMode {
+  public init?(_ description: String) throws {
+    switch description {
+    case "automatic":
+      self = .automatic
+    case "cached":
+      self = .cached
+    case "uncached":
+      self = .uncached
+    case "":
+      return nil
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image caching mode: \"\(description)\"")
+    }
+  }
+}
+
 struct Run: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Run a VM")

@@ -64,7 +81,7 @@ struct Run: AsyncParsableCommand {
  @Option(help: ArgumentHelp(
    "Attach an externally created serial console",
    discussion: "Alternative to `--serial` flag for programmatic integrations."
-  ))
+  ), completion: .file())
  var serialPath: String?

  @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .private))
@@ -75,7 +92,7 @@ struct Run: AsyncParsableCommand {

  @Flag(help: ArgumentHelp(
    "Disable clipboard sharing between host and guest.",
-    discussion: "Only works with Linux-based guest operating systems."))
+    discussion: "Clipboard sharing requires spice-vdagent package on Linux and https://github.com/cirruslabs/tart-guest-agent on macOS."))
  var noClipboard: Bool = false

  #if arch(arm64)
@@ -99,8 +116,12 @@ struct Run: AsyncParsableCommand {
  #endif
  var vncExperimental: Bool = false

+  // Note that the valueName here should really be "path[:options]" instead of just "path",
+  // see ArgumentParser issue[1] for more details.
+  //
+  // [1]: https://github.com/apple/swift-argument-parser/issues/761
  @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only and synchronization options (e.g. --disk="disk.bin", --disk="ubuntu.iso:ro", --disk="/dev/disk0", --disk "ghcr.io/cirruslabs/xcode:16.0:ro" or --disk="nbd://localhost:10809/myDisk:sync=none")
+  Additional disk attachments with an optional read-only and synchronization options in the form of path[:options] (e.g. --disk="disk.bin", --disk="ubuntu.iso:ro", --disk="/dev/disk0", --disk "ghcr.io/cirruslabs/xcode:16.0:ro" or --disk="nbd://localhost:10809/myDisk:sync=none")
  """, discussion: """
  The disk attachment can be a:

@@ -121,8 +142,8 @@ struct Run: AsyncParsableCommand {

  To work around this pass TART_HOME explicitly:

-  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
-  """, valueName: "path[:options]"))
+  sudo TART_HOME="$HOME/.tart" tart run sequoia --disk=/dev/disk0
+  """, valueName: "path"), completion: .file())
  var disk: [String] = []

  #if arch(arm64)
@@ -141,7 +162,11 @@ struct Run: AsyncParsableCommand {
  #endif
  var rosettaTag: String?

-  @Option(help: ArgumentHelp("Additional directory shares with an optional read-only and mount tag options (e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")", discussion: """
+  // Note that the valueName here should really be "[name:]path[:options]" instead of just "path",
+  // see ArgumentParser issue[1] for more details.
+  //
+  // [1]: https://github.com/apple/swift-argument-parser/issues/761
+  @Option(help: ArgumentHelp("Additional directory shares with an optional read-only and mount tag options in the form of [name:]path[:options] (e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")", discussion: """
  Requires host to be macOS 13.0 (Ventura) or newer. macOS guests must be running macOS 13.0 (Ventura) or newer too.

  Options are comma-separated and are as follows:
@@ -153,9 +178,12 @@ struct Run: AsyncParsableCommand {
  Mount tag can be overridden by appending tag property to the directory share (e.g. --dir=\"~/src/build:tag=build\" or --dir=\"~/src/build:ro,tag=build\"). Then it can be mounted via "mount_virtiofs build ~/build" inside guest macOS and "mount -t virtiofs build ~/build" inside guest Linux.

  In case of passing multiple directories per mount tag it is required to prefix them with names e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\". These names will be used as directory names under the mounting point inside guests. For the example above it will be "/Volumes/My Shared Files/build" and "/Volumes/My Shared Files/sources" respectively.
-  """, valueName: "[name:]path[:options]"))
+  """, valueName: "path"), completion: .directory)
  var dir: [String] = []

+  @Flag(help: ArgumentHelp("Enable nested virtualization if possible"))
+  var nested: Bool = false
+
  @Option(help: ArgumentHelp("""
  Use bridged networking instead of the default shared (NAT) networking \n(e.g. --net-bridged=en0 or --net-bridged=\"Wi-Fi\")
  """, discussion: """
@@ -163,17 +191,53 @@ struct Run: AsyncParsableCommand {
  """, valueName: "interface name"))
  var netBridged: [String] = []

-  @Flag(help: ArgumentHelp("Use software networking instead of the default shared (NAT) networking",
-                           discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
+  @Flag(help: ArgumentHelp("Use software networking provided by Softnet instead of the default shared (NAT) networking",
+                           discussion: """
+                           Softnet provides better network isolation and alleviates DHCP shortage on production systems. Tart invokes Softnet when this option is specified as a sub-process and communicates with it over socketpair(2).
+
+                           It is essentially a userspace packet filter which restricts the VM networking and prevents a class of security issues, such as ARP spoofing. By default, the VM will only be able to:
+
+                           * send traffic from its own MAC-address
+                           * send traffic from the IP-address assigned to it by the DHCP
+                           * send traffic to globally routable IPv4 addresses
+                           * send traffic to gateway IP of the vmnet bridge (this would normally be \"bridge100\" interface)
+                           * receive any incoming traffic
+
+                           In addition, Softnet tunes macOS built-in DHCP server to decrease its lease time from the default 86,400 seconds (one day) to 600 seconds (10 minutes). This is especially important when you use Tart to clone and run a lot of ephemeral VMs over a period of one day.
+
+                           More on Softnet here: https://github.com/cirruslabs/softnet
+                           """))
  var netSoftnet: Bool = false

-  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", valueName: "comma-separated CIDRs"))
+  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", discussion: """
+  This option allows you bypass the private IPv4 address space restrictions imposed by --net-softnet.
+
+  For example, you can allow the VM to communicate with the local network with e.g. --net-softnet-allow=10.0.0.0/16 or to completely disable the destination based restrictions with --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
+  """, valueName: "comma-separated CIDRs"))
  var netSoftnetAllow: String?

+  @Option(help: ArgumentHelp("Comma-separated list of TCP ports to expose (e.g. --net-softnet-expose 2222:22,8080:80)", discussion: """
+  Options are comma-separated and are as follows:
+
+  * EXTERNAL_PORT:INTERNAL_PORT — forward TCP traffic from the EXTERNAL_PORT on a host's egress interface (automatically detected and could be Wi-Fi, Ethernet and a VPN interface) to the INTERNAL_PORT on guest's IP (as reported by "tart ip")
+
+  Note that for the port forwarding to work correctly:
+
+  * the software in guest listening on INTERNAL_PORT should either listen on 0.0.0.0 or on an IP address assigned to that guest
+  * connection to the EXTERNAL_PORT should be performed from the local network that the host is attached to or from the internet, it's not possible to connect to that forwarded port from the host itself
+
+  Another thing to keep in mind is that regular Softnet restrictions will still apply even to port forwarding. So if you're planning to access your VM from local network, and your local network is 192.168.0.0/24, for example, then add --net-softnet-allow=192.168.0.0/24. If you only need port forwarding, to completely disable Softnet restrictions you can use --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
+  """, valueName: "comma-separated port specifications"))
+  var netSoftnetExpose: String?
+
  @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
  var netHost: Bool = false

-  @Option(help: ArgumentHelp("Set the root disk options (e.g. --root-disk-opts=\"ro\" or --root-disk-opts=\"sync=none\")",
+  @Option(help: ArgumentHelp("Set the root disk options (e.g. --root-disk-opts=\"ro\" or --root-disk-opts=\"caching=cached,sync=none\")",
                             discussion: """
                             Options are comma-separated and are as follows:

@@ -184,6 +248,12 @@ struct Run: AsyncParsableCommand {
                             * sync=fsync — enable data synchronization with the permanent storage, but don't ensure that it was actually written (e.g. --root-disk-opts="sync=fsync")

                             * sync=full — enable data synchronization with the permanent storage and ensure that it was actually written (e.g. --root-disk-opts="sync=full")
+
+                             * caching=automatic — allows the virtualization framework to automatically determine whether to enable data caching
+
+                             * caching=cached — enabled data caching
+
+                             * caching=uncached — disables data caching
                             """, valueName: "options"))
  var rootDiskOpts: String = ""

@@ -198,12 +268,22 @@ struct Run: AsyncParsableCommand {
  #endif
  var captureSystemKeys: Bool = false

+  #if arch(arm64)
+    @Flag(help: ArgumentHelp("Don't add trackpad as a pointing device on macOS VMs"))
+  #endif
+  var noTrackpad: Bool = false
+
  mutating func validate() throws {
    if vnc && vncExperimental {
      throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
    }

-    // check that not more than one network option is specified
+    // Automatically enable --net-softnet when any of its related options are specified
+    if netSoftnetAllow != nil || netSoftnetExpose != nil {
+      netSoftnet = true
+    }
+
+    // Check that no more than one network option is specified
    var netFlags = 0
    if netBridged.count > 0 { netFlags += 1 }
    if netSoftnet { netFlags += 1 }
@@ -221,6 +301,14 @@ struct Run: AsyncParsableCommand {
      throw ValidationError("--captures-system-keys can only be used with the default VM view")
    }

+    if nested {
+      if #unavailable(macOS 15) {
+        throw ValidationError("Nested virtualization is supported on hosts starting with macOS 15 (Sequoia), and later.")
+      } else if !VZGenericPlatformConfiguration.isNestedVirtualizationSupported {
+        throw ValidationError("Nested virtualization is available for Mac with the M3 chip, and later.")
+      }
+    }
+
    let localStorage = VMStorageLocal()
    let vmDir = try localStorage.open(name)
    if try vmDir.state() == .Suspended {
@@ -229,12 +317,23 @@ struct Run: AsyncParsableCommand {

    if suspendable {
      let config = try VMConfig.init(fromURL: vmDir.configURL)
-      if (config.platform is Linux) {
+      if !(config.platform is PlatformSuspendable) {
        throw ValidationError("You can only suspend macOS VMs")
      }
      if dir.count > 0 {
        throw ValidationError("Suspending VMs with shared directories is not supported")
      }
+
+      if noTrackpad {
+        throw ValidationError("--no-trackpad cannot be used with --suspendable")
+      }
+    }
+
+    if noTrackpad {
+      let config = try VMConfig.init(fromURL: vmDir.configURL)
+      if config.os != .darwin {
+        throw ValidationError("--no-trackpad can only be used with macOS VMs")
+      }
    }

    for disk in disk {
@@ -294,9 +393,12 @@ struct Run: AsyncParsableCommand {
      directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
      serialPorts: serialPorts,
      suspendable: suspendable,
+      nested: nested,
      audio: !noAudio,
      clipboard: !noClipboard,
-      sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw)
+      sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw),
+      caching: VZDiskImageCachingMode(diskOptions.cachingModeRaw),
+      noTrackpad: noTrackpad
    )

    let vncImpl: VNC? = try {
@@ -346,7 +448,35 @@ struct Run: AsyncParsableCommand {
          }
        #endif

-        try await vm!.start(recovery: recovery, resume: resume)
+        do {
+          try await vm!.start(recovery: recovery, resume: resume)
+        } catch let error as VZError {
+          if error.code == .virtualMachineLimitExceeded {
+            var hint = ""
+
+            do {
+              let runningVMs: [String] = try localStorage.list().compactMap { (name, vmDir) in
+                if try !vmDir.running() {
+                  return nil
+                }
+
+                return name
+              }
+
+              if !runningVMs.isEmpty {
+                let runningVMsJoined = runningVMs.joined(separator: ", ")
+
+                hint = " (other running VMs: \(runningVMsJoined))"
+              }
+            } catch {
+              // we can't provide any hint
+            }
+
+            throw RuntimeError.VirtualMachineLimitExceeded(hint)
+          }
+
+          throw error
+        }

        if let vncImpl = vncImpl {
          let vncURL = try await vncImpl.waitForURL(netBridged: !netBridged.isEmpty)
@@ -432,8 +562,10 @@ struct Run: AsyncParsableCommand {

    let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
    if noGraphics || useVNCWithoutGraphics {
-      // enter the main even loop, without bringing up any UI,
-      // and just wait for the VM to exit.
+      // Enter the main event loop without bringing up any UI,
+      // waiting for the VM to exit.
+      NSApplication.shared.setActivationPolicy(.prohibited)
+
      NSApplication.shared.run()
    } else {
      runUI(suspendable, captureSystemKeys)
@@ -461,6 +593,10 @@ struct Run: AsyncParsableCommand {
      softnetExtraArguments += ["--allow", netSoftnetAllow]
    }

+    if let netSoftnetExpose = netSoftnetExpose {
+      softnetExtraArguments += ["--expose", netSoftnetExpose]
+    }
+
    if netSoftnet {
      let config = try VMConfig.init(fromURL: vmDir.configURL)

@@ -589,7 +725,7 @@ struct MainApp: App {
  static var suspendable: Bool = false
  static var capturesSystemKeys: Bool = false

-  @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
+  @NSApplicationDelegateAdaptor private var appDelegate: AppDelegate

  var body: some Scene {
    WindowGroup(vm!.name) {
@@ -645,18 +781,7 @@ struct MainApp: App {
  }
 }

-// The only way to fully remove Edit menu item.
-class MinimalMenuAppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
-  let indexOfEditMenu = 2
-
-  func applicationDidFinishLaunching(_ : Notification) {
-    NSApplication.shared.mainMenu?.removeItem(at: indexOfEditMenu)
-
-    let nsApp = NSApplication.shared
-    nsApp.setActivationPolicy(.regular)
-    nsApp.activate(ignoringOtherApps: true)
-  }
-
+class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
  func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply {
    if (kill(getpid(), MainApp.suspendable ? SIGUSR1 : SIGINT) == 0) {
      return .terminateLater
@@ -709,12 +834,12 @@ struct VMView: NSViewRepresentable {

    machineView.capturesSystemKeys = capturesSystemKeys

-    // Enable automatic display reconfiguration
-    // for guests that support it
+    // If not specified, enable automatic display
+    // reconfiguration for guests that support it
    //
    // This is disabled for Linux because of poor HiDPI
    // support, which manifests in fonts being too small
-    if #available(macOS 14.0, *), vm.config.os != .linux {
+    if #available(macOS 14.0, *), vm.config.displayRefit ?? (vm.config.os != .linux) {
      machineView.automaticallyReconfiguresDisplay = true
    }

@@ -730,12 +855,12 @@ struct AdditionalDisk {
  let configuration: VZStorageDeviceConfiguration

  init(parseFrom: String) throws {
-    let (diskPath, readOnly, syncModeRaw) = Self.parseOptions(parseFrom)
+    let (diskPath, readOnly, syncModeRaw, cachingModeRaw) = Self.parseOptions(parseFrom)

-    self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw)
+    self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw, cachingModeRaw: cachingModeRaw)
  }

-  static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String) throws -> VZStorageDeviceConfiguration {
+  static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String, cachingModeRaw: String) throws -> VZStorageDeviceConfiguration {
    let diskURL = URL(string: diskPath)

    if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
@@ -812,14 +937,14 @@ struct AdditionalDisk {
    let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
      url: diskFileURL,
      readOnly: diskReadOnly,
-      cachingMode: .automatic,
+      cachingMode: try VZDiskImageCachingMode(cachingModeRaw) ?? .automatic,
      synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
    )

    return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
  }

-  static func parseOptions(_ parseFrom: String) -> (String, Bool, String) {
+  static func parseOptions(_ parseFrom: String) -> (String, Bool, String, String) {
    var arguments = parseFrom.split(separator: ":")

    let options = DiskOptions(String(arguments.last!))
@@ -827,13 +952,14 @@ struct AdditionalDisk {
      arguments.removeLast()
    }

-    return (arguments.joined(separator: ":"), options.readOnly, options.syncModeRaw)
+    return (arguments.joined(separator: ":"), options.readOnly, options.syncModeRaw, options.cachingModeRaw)
  }
 }

 struct DiskOptions {
  var readOnly: Bool = false
  var syncModeRaw: String = ""
+  var cachingModeRaw: String = ""
  var foundAtLeastOneOption: Bool = false

  init(_ parseFrom: String) {
@@ -847,6 +973,9 @@ struct DiskOptions {
      case option.hasPrefix("sync="):
        self.syncModeRaw = String(option.dropFirst("sync=".count))
        self.foundAtLeastOneOption = true
+      case option.hasPrefix("caching="):
+        self.cachingModeRaw = String(option.dropFirst("caching=".count))
+        self.foundAtLeastOneOption = true
      default:
        continue
      }
@@ -970,7 +1099,7 @@ struct DirectoryShare {
    process.standardInput = inPipe
    process.launch()

-    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.write(contentsOf: response!.data)
    try inPipe.fileHandleForWriting.close()
    process.waitUntilExit()

@@ -17,6 +17,9 @@ struct Set: AsyncParsableCommand {
  @Option(help: "VM display resolution in a format of <width>x<height>. For example, 1200x800")
  var display: VMDisplayConfig?

+  @Flag(inversion: .prefixedNo, help: ArgumentHelp("Whether to automatically reconfigure the VM's display to fit the window"))
+  var displayRefit: Bool? = nil
+
  @Flag(help: ArgumentHelp("Generate a new random MAC address for the VM."))
  var randomMAC: Bool = false

@@ -30,15 +33,7 @@ struct Set: AsyncParsableCommand {

  @Option(help: ArgumentHelp("Resize the VMs disk to the specified size in GB (note that the disk size can only be increased to avoid losing data)",
                             discussion: """
-                             Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images
-                             have the \"cloud-initramfs-growroot\" package installed that runs on boot) and on the rest of the
-                             distributions by running the \"growpart\" or \"resize2fs\" commands.
-
-                             For macOS, however, things are a bit more complicated: you need to remove the recovery partition
-                             first and then run various \"diskutil\" commands, see Tart's packer plugin source code for more
-                             details[1].
-
-                             [1]: https://github.com/cirruslabs/packer-plugin-tart/blob/main/builder/tart/step_disk_resize.go
+                             See https://tart.run/faq/#disk-resizing for more details.
                             """))
  var diskSize: UInt16?

@@ -63,13 +58,14 @@ struct Set: AsyncParsableCommand {
      }
    }

+    vmConfig.displayRefit = displayRefit
+
    if randomMAC {
      vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
    }

    #if arch(arm64)
-      if randomSerial {
-        let oldPlatform = vmConfig.platform as! Darwin
+      if randomSerial, let oldPlatform = vmConfig.platform as? Darwin {
        vmConfig.platform = Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: oldPlatform.hardwareModel)
      }
    #endif
@@ -36,12 +36,17 @@ class DockerConfigCredentialsProvider: CredentialsProvider {

    process.launch()

-    inPipe.fileHandleForWriting.write("\(host)\n".data(using: .utf8)!)
+    do {
+      try inPipe.fileHandleForWriting.write(contentsOf: "\(host)\n".data(using: .utf8)!)
+    } catch {
+      throw CredentialsProviderError.Failed(message: "Failed to write host to Docker helper!")
+    }
    inPipe.fileHandleForWriting.closeFile()

+    let outputData = try outPipe.fileHandleForReading.readToEnd()
+
    process.waitUntilExit()

-    let outputData = try outPipe.fileHandleForReading.readToEnd()
    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
      if let outputData = outputData {
        print(String(decoding: outputData, as: UTF8.self))
@@ -1,69 +1,6 @@
 import Foundation
-import AsyncAlgorithms

-fileprivate let urlSession = createURLSession()
-
-class DownloadDelegate: NSObject, URLSessionTaskDelegate {
-  let progress: Progress
-  init(_ progress: Progress) throws {
-    self.progress = progress
-  }
-
-  func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) {
-    self.progress.addChild(task.progress, withPendingUnitCount: self.progress.totalUnitCount)
-  }
-}
-
-class Fetcher {
-  static func fetch(_ request: URLRequest, viaFile: Bool = false, progress: Progress? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let delegate = progress != nil ? try DownloadDelegate(progress!) : nil
-
-    if viaFile {
-      return try await fetchViaFile(request, delegate: delegate)
-    }
-
-    return try await fetchViaMemory(request, delegate: delegate)
-  }
-
-  private static func fetchViaMemory(_ request: URLRequest, delegate: URLSessionTaskDelegate? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let dataCh = AsyncThrowingChannel<Data, Error>()
-
-    let (data, response) = try await urlSession.data(for: request, delegate: delegate)
-
-    Task {
-      await dataCh.send(data)
-
-      dataCh.finish()
-    }
-
-    return (dataCh, response as! HTTPURLResponse)
-  }
-
-  private static func fetchViaFile(_ request: URLRequest, delegate: URLSessionTaskDelegate? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let dataCh = AsyncThrowingChannel<Data, Error>()
-
-    let (fileURL, response) = try await urlSession.download(for: request, delegate: delegate)
-
-    // Acquire a handle to the downloaded file and then remove it.
-    //
-    // This keeps a working reference to that file, yet we don't
-    // have to deal with the cleanup any more.
-    let mappedFile = try Data(contentsOf: fileURL, options: [.alwaysMapped])
-    try FileManager.default.removeItem(at: fileURL)
-
-    Task {
-      for chunk in (0 ..< mappedFile.count).chunks(ofCount: 64 * 1024 * 1024) {
-        await dataCh.send(mappedFile.subdata(in: chunk))
-      }
-
-      dataCh.finish()
-    }
-
-    return (dataCh, response as! HTTPURLResponse)
-  }
-}
-
-fileprivate func createURLSession() -> URLSession {
+fileprivate var urlSession: URLSession = {
  let config = URLSessionConfiguration.default

  // Harbor expects a CSRF token to be present if the HTTP client
@@ -77,4 +14,84 @@ fileprivate func createURLSession() -> URLSession {
  config.httpShouldSetCookies = false

  return URLSession(configuration: config)
+}()
+
+class Fetcher {
+  static func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
+    let task = urlSession.dataTask(with: request)
+
+    let delegate = Delegate()
+    task.delegate = delegate
+
+    let stream = AsyncThrowingStream<Data, Error> { continuation in
+      delegate.streamContinuation = continuation
+    }
+
+    let response = try await withCheckedThrowingContinuation { continuation in
+      delegate.responseContinuation = continuation
+      task.resume()
+    }
+
+    return (stream, response as! HTTPURLResponse)
+  }
+}
+
+fileprivate class Delegate: NSObject, URLSessionDataDelegate {
+  var responseContinuation: CheckedContinuation<URLResponse, Error>?
+  var streamContinuation: AsyncThrowingStream<Data, Error>.Continuation?
+
+  private var buffer: Data = Data()
+  private let bufferFlushSize = 16 * 1024 * 1024
+
+  func urlSession(
+    _ session: URLSession,
+    dataTask: URLSessionDataTask,
+    didReceive response: URLResponse,
+    completionHandler: @escaping (URLSession.ResponseDisposition) -> Void
+  ) {
+    // Soft-limit for the maximum buffer capacity
+    let capacity = min(response.expectedContentLength, Int64(bufferFlushSize))
+
+    // Pre-initialize buffer as we now know the capacity
+    buffer = Data(capacity: Int(capacity))
+
+    responseContinuation?.resume(returning: response)
+    responseContinuation = nil
+    completionHandler(.allow)
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    dataTask: URLSessionDataTask,
+    didReceive data: Data
+  ) {
+    buffer.append(data)
+
+    if buffer.count >= bufferFlushSize {
+      streamContinuation?.yield(buffer)
+      buffer.removeAll(keepingCapacity: true)
+    }
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    task: URLSessionTask,
+    didCompleteWithError error: Error?
+  ) {
+    if let error = error {
+      responseContinuation?.resume(throwing: error)
+      responseContinuation = nil
+
+      streamContinuation?.finish(throwing: error)
+      streamContinuation = nil
+    } else {
+      if !buffer.isEmpty {
+        streamContinuation?.yield(buffer)
+        buffer.removeAll(keepingCapacity: true)
+      }
+
+      streamContinuation?.finish()
+      streamContinuation = nil
+    }
+  }
 }
@@ -52,6 +52,11 @@ struct ARPCache {
    process.standardInput = FileHandle.nullDevice

    try process.run()
+
+    guard let arpCommandOutput = try pipe.fileHandleForReading.readToEnd() else {
+      throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
+    }
+
    process.waitUntilExit()

    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
@@ -60,10 +65,6 @@ struct ARPCache {
        terminationStatus: process.terminationStatus)
    }

-    guard let arpCommandOutput = try pipe.fileHandleForReading.readToEnd() else {
-      throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
-    }
-
    self.arpCommandOutput = arpCommandOutput
  }

@@ -2,5 +2,5 @@ import Foundation

 protocol Disk {
  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?) async throws
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
 }
@@ -45,7 +45,7 @@ class DiskV1: Disk {
    return pushedLayers
  }

-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
      throw OCIError.FailedToCreateVmFile
    }
@@ -57,7 +57,7 @@ class DiskV1: Disk {
    // Decompress the layers onto the disk in a single stream
    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
      if let data = data {
-        disk.write(data)
+        try disk.write(contentsOf: data)
      }
    }

@@ -1,11 +1,27 @@
 import Foundation
 import Compression
 import System
+import Retry

 class DiskV2: Disk {
  private static let bufferSizeBytes = 4 * 1024 * 1024
  private static let layerLimitBytes = 512 * 1024 * 1024

+  // A zero chunk for faster than byte-by-byte comparisons
+  //
+  // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
+  // on the last block since it costs only 4 MiB of excess data per 512 MiB layer.
+  //
+  // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
+  // +--------------------------------------+---------------------------------------------------+
+  // | Operation                            | time(1) result                                    |
+  // +--------------------------------------+---------------------------------------------------+
+  // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
+  // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
+  // +--------------------------------------+---------------------------------------------------+
+  private static let holeGranularityBytes = 4 * 1024 * 1024
+  private static let zeroChunk = Data(count: holeGranularityBytes)
+
  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
    var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []

@@ -26,8 +42,22 @@ class DiskV2: Disk {
        // Launch a disk layer pushing task
        group.addTask {
          let compressedData = try (data as NSData).compressed(using: .lz4) as Data
+          let compressedDataDigest = Digest.hash(compressedData)

-          let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+          try await retry(maxAttempts: 5) {
+            if try await !registry.blobExists(compressedDataDigest) {
+              _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error: \(error.localizedDescription)")
+              print("Attempting to re-try...")
+
+              return .retry
+            }
+
+            return .throw
+          }

          // Update progress using a relative value
          progress.completedUnitCount += Int64(data.count)
@@ -35,7 +65,7 @@ class DiskV2: Disk {
          return (index, OCIManifestLayer(
            mediaType: diskV2MediaType,
            size: compressedData.count,
-            digest: layerDigest,
+            digest: compressedDataDigest,
            uncompressedSize: UInt64(data.count),
            uncompressedContentDigest: Digest.hash(data)
          ))
@@ -54,12 +84,12 @@ class DiskV2: Disk {
    }
  }

-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
    // Support resumable pulls
    let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)

    if !pullResumed {
-      if let localLayerCache = localLayerCache {
+      if deduplicate, let localLayerCache = localLayerCache {
        // Clone the local layer cache's disk and use it as a base, potentially
        // reducing the space usage since some blocks won't be written at all
        try FileManager.default.copyItem(at: localLayerCache.diskURL, to: diskURL)
@@ -136,26 +166,31 @@ class DiskV2: Disk {

          // Also open the disk file for reading and verifying
          // its contents in case the local layer cache is used
-          let rdisk: FileHandle? = if localLayerCache != nil {
+          let rdisk: FileHandle? = if deduplicate && localLayerCache != nil {
            try FileHandle(forReadingFrom: diskURL)
          } else {
            nil
          }

-          // Check if we already have this layer contents in the local layer cache
-          if let localLayerCache = localLayerCache, let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset) {
-            // indicates that the locally cloned disk image has the same content at the given offset
-            let localHit = localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest
-              && localLayerInfo.range.lowerBound == diskWritingOffset
-            // doesn't seem that localHit can ever be false if the localLayerCache is not nil
-            // but let's just add extra safety here and check it
-            if !localHit {
+          // Check if we already have this layer contents in the local layer cache,
+          // or perhaps even on the cloned disk (when the deduplication is enabled)
+          if let localLayerCache = localLayerCache,
+             let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset),
+             localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest {
+            if deduplicate && localLayerInfo.range.lowerBound == diskWritingOffset {
+              // Do nothing, because the data is already on the disk that we've inherited from
+            } else {
              // Fulfil the layer contents from the local blob cache
              let data = localLayerCache.subdata(localLayerInfo.range)
              _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
            }
+
            try disk.close()

+            if let rdisk = rdisk {
+              try rdisk.close()
+            }
+
            // Update the progress
            progress.completedUnitCount += Int64(diskLayer.size)

@@ -173,16 +208,35 @@ class DiskV2: Disk {
            diskWritingOffset = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
          }

-          try await registry.pullBlob(diskLayer.digest) { data in
-            try filter.write(data)
+          var rangeStart: Int64 = 0

-            // Update the progress
-            progress.completedUnitCount += Int64(data.count)
+          try await retry(maxAttempts: 5) {
+            try await registry.pullBlob(diskLayer.digest, rangeStart: rangeStart) { data in
+              try filter.write(data)
+
+              // Update the progress
+              progress.completedUnitCount += Int64(data.count)
+
+              // Update the current range start
+              rangeStart += Int64(data.count)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error pulling disk layer \(index + 1): \"\(error.localizedDescription)\", attempting to re-try...")
+
+              return .retry
+            }
+
+            return .throw
          }

          try filter.finalize()

          try disk.close()
+
+          if let rdisk = rdisk {
+            try rdisk.close()
+          }
        }

        globalDiskWritingOffset += uncompressedLayerSize
@@ -191,22 +245,6 @@ class DiskV2: Disk {
  }

  private static func zeroSkippingWrite(_ disk: FileHandle, _ rdisk: FileHandle?, _ fsBlockSize: UInt64, _ offset: UInt64, _ data: Data) throws -> UInt64 {
-    let holeGranularityBytes = 64 * 1024
-
-    // A zero chunk for faster than byte-by-byte comparisons
-    //
-    // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
-    // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
-    //
-    // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
-    // +--------------------------------------+---------------------------------------------------+
-    // | Operation                            | time(1) result                                    |
-    // +--------------------------------------+---------------------------------------------------+
-    // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
-    // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
-    // +--------------------------------------+---------------------------------------------------+
-    let zeroChunk = Data(count: holeGranularityBytes)
-
    var offset = offset

    for chunk in data.chunks(ofCount: holeGranularityBytes) {
@@ -230,7 +268,7 @@ class DiskV2: Disk {

          if chunk != actualContentsOnDisk {
            try disk.seek(toOffset: offset)
-            disk.write(chunk)
+            try disk.write(contentsOf: chunk)
          }
        }

@@ -244,7 +282,7 @@ class DiskV2: Disk {
      // is zeroed via truncate(2)
      if chunk != zeroChunk {
        try disk.seek(toOffset: offset)
-        disk.write(chunk)
+        try disk.write(contentsOf: chunk)
      }

      offset += UInt64(chunk.count)
@@ -66,6 +66,11 @@ struct OCIManifest: Codable, Equatable {
 struct OCIConfig: Codable {
  var architecture: Architecture = .arm64
  var os: OS = .darwin
+  var config: ConfigContainer?
+
+  struct ConfigContainer: Codable {
+    var Labels: [String: String]?
+  }

  func toJSON() throws -> Data {
    try Config.jsonEncoder().encode(self)
@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2

 import Antlr4

@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4

 open class ReferenceLexer: Lexer {
@@ -49,7 +49,7 @@ open class ReferenceLexer: Lexer {

 	public
 	required init(_ input: CharStream) {
-	    RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION)
+	    RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION)
 		super.init(input)
 		_interp = LexerATNSimulator(self, ReferenceLexer._ATN, ReferenceLexer._decisionToDFA, ReferenceLexer._sharedContextCache)
 	}
@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4

 /**
@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4

 open class ReferenceParser: Parser {
@@ -41,7 +41,7 @@ open class ReferenceParser: Parser {
 	static let VOCABULARY = Vocabulary(_LITERAL_NAMES, _SYMBOLIC_NAMES)

 	override open
-	func getGrammarFileName() -> String { return "java-escape" }
+	func getGrammarFileName() -> String { return "Reference.g4" }

 	override open
 	func getRuleNames() -> [String] { return ReferenceParser.ruleNames }
@@ -60,7 +60,7 @@ open class ReferenceParser: Parser {

 	override public
 	init(_ input:TokenStream) throws {
-	    RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION)
+	    RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION)
 		try super.init(input)
 		_interp = ParserATNSimulator(self,ReferenceParser._ATN,ReferenceParser._decisionToDFA, ReferenceParser._sharedContextCache)
 	}
@@ -460,7 +460,7 @@ open class ReferenceParser: Parser {
 		 		setState(63)
 		 		try _errHandler.sync(self)
 		 		_la = try _input.LA(1)
-		 		if ((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0) {
+		 		if (((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
 		 			setState(62)
 		 			try separator()

@@ -611,7 +611,7 @@ open class ReferenceParser: Parser {
 		 	setState(84)
 		 	try _errHandler.sync(self)
 		 	_la = try _input.LA(1)
-		 	while ((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0) {
+		 	while (((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
 		 		setState(79)
 		 		try separator()
 		 		setState(80)
@@ -664,7 +664,7 @@ open class ReferenceParser: Parser {
 		 	try enterOuterAlt(_localctx, 1)
 		 	setState(87)
 		 	_la = try _input.LA(1)
-		 	if (!((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
+		 	if (!(((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0))) {
 		 	try _errHandler.recoverInline(self)
 		 	}
 		 	else {
@@ -1,6 +1,5 @@
 import Foundation
 import Algorithms
-import AsyncAlgorithms

 enum RegistryError: Error {
  case UnexpectedHTTPStatusCode(when: String, code: Int, details: String = "")
@@ -10,6 +9,7 @@ enum RegistryError: Error {
 }

 enum HTTPMethod: String {
+  case HEAD = "HEAD"
  case GET = "GET"
  case POST = "POST"
  case PUT = "PUT"
@@ -20,21 +20,35 @@ enum HTTPCode: Int {
  case Ok = 200
  case Created = 201
  case Accepted = 202
+  case PartialContent = 206
  case Unauthorized = 401
+  case NotFound = 404
 }

 extension Data {
  func asText() -> String {
    String(decoding: self, as: UTF8.self)
  }
+
+  func asTextPreview(limit: Int = 1000) -> String {
+    guard count > limit else {
+      return asText()
+    }
+
+    return "\(asText().prefix(limit))..."
+  }
 }

-extension AsyncThrowingChannel<Data, Error> {
-  func asData() async throws -> Data {
+extension AsyncThrowingStream<Data, Error> {
+  func asData(limitBytes: Int64? = nil) async throws -> Data {
    var result = Data()

    for try await chunk in self {
      result += chunk
+
+      if let limitBytes, result.count > limitBytes {
+        return result
+      }
    }

    return result
@@ -158,7 +172,7 @@ class Registry {
                                                 body: manifestJSON)
    if response.statusCode != HTTPCode.Created.rawValue {
      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing manifest", code: response.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
    }

    return Digest.hash(manifestJSON)
@@ -169,7 +183,7 @@ class Registry {
                                                 headers: ["Accept": ociManifestMediaType])
    if response.statusCode != HTTPCode.Ok.rawValue {
      throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling manifest", code: response.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
    }

    let manifest = try OCIManifest(fromJSON: data)
@@ -189,19 +203,19 @@ class Registry {
    return URLComponents(url: uploadLocation.absolutize(baseURL), resolvingAgainstBaseURL: true)!
  }

-  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0) async throws -> String {
+  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0, digest: String? = nil) async throws -> String {
    // Initiate a blob upload
    let (data, postResponse) = try await dataRequest(.POST, endpointURL("\(namespace)/blobs/uploads/"),
                                                     headers: ["Content-Length": "0"])
    if postResponse.statusCode != HTTPCode.Accepted.rawValue {
      throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (POST)", code: postResponse.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
    }

    // Figure out where to upload the blob
    var uploadLocation = try uploadLocationFromResponse(postResponse)

-    let digest = Digest.hash(fromData)
+    let digest = digest ?? Digest.hash(fromData)

    if chunkSizeMb == 0 {
      // monolithic upload
@@ -216,7 +230,7 @@ class Registry {
      )
      if response.statusCode != HTTPCode.Created.rawValue {
        throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT) to \(uploadLocation)",
-                                                     code: response.statusCode, details: data.asText())
+                                                     code: response.statusCode, details: data.asTextPreview())
      }
      return digest
    }
@@ -239,7 +253,7 @@ class Registry {
      // always accept both statuses since AWS ECR is not following specification
      if response.statusCode != HTTPCode.Created.rawValue && response.statusCode != HTTPCode.Accepted.rawValue {
        throw RegistryError.UnexpectedHTTPStatusCode(when: "streaming blob to \(uploadLocation)",
-                                                     code: response.statusCode, details: data.asText())
+                                                     code: response.statusCode, details: data.asTextPreview())
      }
      uploadedBytes += chunk.count
      // Update location for the next chunk
@@ -249,10 +263,35 @@ class Registry {
    return digest
  }

-  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
-    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
-    if response.statusCode != HTTPCode.Ok.rawValue {
-      let body = try await channel.asData().asText()
+  public func blobExists(_ digest: String) async throws -> Bool {
+    let (data, response) = try await dataRequest(.HEAD, endpointURL("\(namespace)/blobs/\(digest)"))
+
+    switch response.statusCode {
+    case HTTPCode.Ok.rawValue:
+      return true
+    case HTTPCode.NotFound.rawValue:
+      return false
+    default:
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asTextPreview())
+    }
+  }
+
+  public func pullBlob(_ digest: String, rangeStart: Int64 = 0, handler: (Data) async throws -> Void) async throws {
+    var expectedStatusCode = HTTPCode.Ok
+    var headers: [String: String] = [:]
+
+    // Send Range header and expect HTTP 206 in return
+    //
+    // However, do not send Range header at all when rangeStart is 0,
+    // because it makes no sense and we might get HTTP 200 in return
+    if rangeStart != 0 {
+      expectedStatusCode = HTTPCode.PartialContent
+      headers["Range"] = "bytes=\(rangeStart)-"
+    }
+
+    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), headers: headers, viaFile: true)
+    if response.statusCode != expectedStatusCode.rawValue {
+      let body = try await channel.asData(limitBytes: 4096).asTextPreview()
      throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling blob", code: response.statusCode,
                                                   details: body)
    }
@@ -292,7 +331,7 @@ class Registry {
    body: Data? = nil,
    doAuth: Bool = true,
    viaFile: Bool = false
-  ) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  ) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
    var urlComponents = urlComponents

    if urlComponents.queryItems == nil && !parameters.isEmpty {
@@ -312,12 +351,11 @@ class Registry {
      request.httpBody = body
    }

-    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)

    if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
-      _ = try await channel.asData()
      try await auth(response: response)
-      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
    }

    return (channel, response)
@@ -377,7 +415,7 @@ class Registry {
    let (data, response) = try await dataRequest(.GET, authenticateURL, headers: headers, doAuth: false)
    if response.statusCode != HTTPCode.Ok.rawValue {
      throw RegistryError.AuthFailed(why: "received unexpected HTTP status code \(response.statusCode) "
-        + "while retrieving an authentication token", details: data.asText())
+        + "while retrieving an authentication token", details: data.asTextPreview())
    }

    await authenticationKeeper.set(try TokenResponse.parse(fromData: data))
@@ -398,11 +436,13 @@ class Registry {
    return nil
  }

-  private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  private func authAwareRequest(request: URLRequest, viaFile: Bool = false, doAuth: Bool) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
    var request = request

-    if let (name, value) = await authenticationKeeper.header() {
-      request.addValue(value, forHTTPHeaderField: name)
+    if doAuth {
+      if let (name, value) = await authenticationKeeper.header() {
+        request.addValue(value, forHTTPHeaderField: name)
+      }
    }

    request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",
@@ -11,7 +11,7 @@ class PIDLock {
    if fd == -1 {
      let details = Errno(rawValue: CInt(errno))

-      throw RuntimeError.PIDLockFailed("failed to open lock file \(url): \(details)")
+      throw RuntimeError.PIDLockMissing("failed to open lock file \(url): \(details)")
    }
  }

@@ -8,7 +8,7 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {

 #if arch(arm64)

-  struct Darwin: Platform {
+  struct Darwin: PlatformSuspendable {
    var ecid: VZMacMachineIdentifier
    var hardwareModel: VZMacHardwareModel

@@ -38,7 +38,7 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
      }
      guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
-        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+        throw UnsupportedHostOSError()
      }
      self.hardwareModel = hardwareModel
    }
@@ -58,7 +58,11 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
      VZMacOSBootLoader()
    }

-    func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
+    func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+      if needsNestedVirtualization {
+        throw RuntimeError.VMConfigurationError("macOS virtual machines do not support nested virtualization")
+      }
+
      let result = VZMacPlatformConfiguration()

      result.machineIdentifier = ecid
@@ -103,18 +107,37 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
    func keyboards() -> [VZKeyboardConfiguration] {
      if #available(macOS 14, *) {
        // Mac keyboard is only supported by guests starting with macOS Ventura
-        return [VZMacKeyboardConfiguration()]
+        return [VZUSBKeyboardConfiguration(), VZMacKeyboardConfiguration()]
      } else {
        return [VZUSBKeyboardConfiguration()]
      }
    }

+    func keyboardsSuspendable() -> [VZKeyboardConfiguration] {
+      if #available(macOS 14, *) {
+        return [VZMacKeyboardConfiguration()]
+      } else {
+        // fallback to the regular configuration
+        return keyboards()
+      }
+    }
+
    func pointingDevices() -> [VZPointingDeviceConfiguration] {
-      if #available(macOS 13, *) {
+      // Trackpad is only supported by guests starting with macOS Ventura
+      [VZUSBScreenCoordinatePointingDeviceConfiguration(), VZMacTrackpadConfiguration()]
+    }
+
+    func pointingDevicesSimplified() -> [VZPointingDeviceConfiguration] {
+      // Only include the USB pointing device, not the trackpad
+      return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+    }
+
+    func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration] {
+      if #available(macOS 14, *) {
        return [VZMacTrackpadConfiguration()]
      } else {
        // fallback to the regular configuration
-        return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+        return pointingDevices()
      }
    }
  }
@@ -14,8 +14,12 @@ struct Linux: Platform {
    return result
  }

-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
-    VZGenericPlatformConfiguration()
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+    let config = VZGenericPlatformConfiguration()
+    if #available(macOS 15, *) {
+      config.isNestedVirtualizationEnabled = needsNestedVirtualization
+    }
+    return config
  }

  func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
@@ -38,4 +42,9 @@ struct Linux: Platform {
  func pointingDevices() -> [VZPointingDeviceConfiguration] {
    [VZUSBScreenCoordinatePointingDeviceConfiguration()]
  }
+
+  func pointingDevicesSimplified() -> [VZPointingDeviceConfiguration] {
+    // Linux doesn't support trackpad, so just return the regular pointing devices
+    return pointingDevices()
+  }
 }
@@ -3,8 +3,14 @@ import Virtualization
 protocol Platform: Codable {
  func os() -> OS
  func bootLoader(nvramURL: URL) throws -> VZBootLoader
-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration
  func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
  func keyboards() -> [VZKeyboardConfiguration]
  func pointingDevices() -> [VZPointingDeviceConfiguration]
+  func pointingDevicesSimplified() -> [VZPointingDeviceConfiguration]
+}
+
+protocol PlatformSuspendable: Platform {
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration]
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration]
 }
@@ -1,4 +1,5 @@
 import Foundation
+import System

 extension URL {
  func accessDate() throws -> Date {
@@ -13,7 +14,9 @@ extension URL {
    let times = [accessDate.asTimeval(), modificationDate.asTimeval()]
    let ret = utimes(path, times)
    if ret != 0 {
-      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(ret.explanation())")
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(details)")
    }
  }
 }
@@ -1,6 +1,5 @@
 import Foundation
 import Virtualization
-import AsyncAlgorithms
 import Semaphore

 struct UnsupportedRestoreImageError: Error {
@@ -47,9 +46,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
       serialPorts: [VZSerialPortConfiguration] = [],
       suspendable: Bool = false,
+       nested: Bool = false,
       audio: Bool = true,
       clipboard: Bool = true,
-       sync: VZDiskImageSynchronizationMode = .full
+       sync: VZDiskImageSynchronizationMode = .full,
+       caching: VZDiskImageCachingMode? = nil,
+       noTrackpad: Bool = false
  ) throws {
    name = vmDir.name
    config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -66,9 +68,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
                                                directorySharingDevices: directorySharingDevices,
                                                serialPorts: serialPorts,
                                                suspendable: suspendable,
+                                                nested: nested,
                                                audio: audio,
                                                clipboard: clipboard,
-                                                sync: sync
+                                                sync: sync,
+                                                caching: caching,
+                                                noTrackpad: noTrackpad
    )
    virtualMachine = VZVirtualMachine(configuration: configuration)

@@ -96,16 +101,13 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    // Download the IPSW
    defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")

-    let downloadProgress = Progress(totalUnitCount: 100)
-    ProgressObserver(downloadProgress).log(defaultLogger)
-
    let request = URLRequest(url: remoteURL)
-    let (channel, response) = try await Fetcher.fetch(request, viaFile: true, progress: downloadProgress)
+    let (channel, response) = try await Fetcher.fetch(request, viaFile: true)

    let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".ipsw")
-    defaultLogger.appendNewLine("Computing digest for \(temporaryLocation.path)...")
-    let digestProgress = Progress(totalUnitCount: response.expectedContentLength)
-    ProgressObserver(digestProgress).log(defaultLogger)
+
+    let progress = Progress(totalUnitCount: response.expectedContentLength)
+    ProgressObserver(progress).log(defaultLogger)

    FileManager.default.createFile(atPath: temporaryLocation.path, contents: nil)
    let lock = try FileLock(lockURL: temporaryLocation)
@@ -115,10 +117,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    let digest = Digest()

    for try await chunk in channel {
-      let chunkAsData = Data(chunk)
-      fileHandle.write(chunkAsData)
-      digest.update(chunkAsData)
-      digestProgress.completedUnitCount += Int64(chunk.count)
+      try fileHandle.write(contentsOf: chunk)
+      digest.update(chunk)
+      progress.completedUnitCount += Int64(chunk.count)
    }

    try fileHandle.close()
@@ -295,9 +296,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
    serialPorts: [VZSerialPortConfiguration],
    suspendable: Bool = false,
+    nested: Bool = false,
    audio: Bool = true,
    clipboard: Bool = true,
-    sync: VZDiskImageSynchronizationMode = .full
+    sync: VZDiskImageSynchronizationMode = .full,
+    caching: VZDiskImageCachingMode? = nil,
+    noTrackpad: Bool = false
  ) throws -> VZVirtualMachineConfiguration {
    let configuration = VZVirtualMachineConfiguration()

@@ -309,7 +313,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    configuration.memorySize = vmConfig.memorySize

    // Platform
-    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL)
+    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL, needsNestedVirtualization: nested)

    // Display
    configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]
@@ -317,20 +321,33 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    // Audio
    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()

-    let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-    let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
-
    if audio && !suspendable {
+      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
+      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+
      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
      outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+
+      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
+    } else {
+      // just a null speaker
+      soundDeviceConfiguration.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()]
    }

-    soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
    configuration.audioDevices = [soundDeviceConfiguration]

    // Keyboard and mouse
-    configuration.keyboards = vmConfig.platform.keyboards()
-    configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
+      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
+      configuration.pointingDevices = platformSuspendable.pointingDevicesSuspendable()
+    } else {
+      configuration.keyboards = vmConfig.platform.keyboards()
+      if noTrackpad {
+        configuration.pointingDevices = vmConfig.platform.pointingDevicesSimplified()
+      } else {
+        configuration.pointingDevices = vmConfig.platform.pointingDevices()
+      }
+    }

    // Networking
    configuration.networkDevices = network.attachments().map {
@@ -341,28 +358,29 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    }

    // Clipboard sharing via Spice agent
-    if clipboard && vmConfig.os == .linux {
+    if clipboard {
      let spiceAgentConsoleDevice = VZVirtioConsoleDeviceConfiguration()
      let spiceAgentPort = VZVirtioConsolePortConfiguration()
      spiceAgentPort.name = VZSpiceAgentPortAttachment.spiceAgentPortName
-      spiceAgentPort.attachment = VZSpiceAgentPortAttachment()
+      let spiceAgentPortAttachment = VZSpiceAgentPortAttachment()
+      spiceAgentPortAttachment.sharesClipboard = true
+      spiceAgentPort.attachment = spiceAgentPortAttachment
      spiceAgentConsoleDevice.ports[0] = spiceAgentPort
      configuration.consoleDevices.append(spiceAgentConsoleDevice)
    }

    // Storage
-    let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
-      // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: sync) :
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .automatic, synchronizationMode: sync)
+    var attachment = try VZDiskImageStorageDeviceAttachment(
+      url: diskURL,
+      readOnly: false,
+      // When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1]
+      //
+      // [1]: https://github.com/cirruslabs/tart/pull/675
+      cachingMode: caching ?? (vmConfig.os == .linux ? .cached : .automatic),
+      synchronizationMode: sync
+    )

-    var device: VZStorageDeviceConfiguration
-    if #available(macOS 14, *), vmConfig.os == .linux {
-      device = VZNVMExpressControllerDeviceConfiguration(attachment: attachment)
-    } else {
-      device = VZVirtioBlockDeviceConfiguration(attachment: attachment)
-    }
-    var devices: [VZStorageDeviceConfiguration] = [device]
+    var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)]
    devices.append(contentsOf: additionalStorageDevices)
    configuration.storageDevices = devices

@@ -24,6 +24,7 @@ enum CodingKeys: String, CodingKey {
  case memorySize
  case macAddress
  case display
+  case displayRefit

  // macOS-specific keys
  case ecid
@@ -52,6 +53,7 @@ struct VMConfig: Codable {
  private(set) var memorySize: UInt64
  var macAddress: VZMACAddress
  var display: VMDisplayConfig = VMDisplayConfig()
+  var displayRefit: Bool?

  init(
    platform: Platform,
@@ -121,6 +123,7 @@ struct VMConfig: Codable {
    self.macAddress = macAddress

    display = try container.decodeIfPresent(VMDisplayConfig.self, forKey: .display) ?? VMDisplayConfig()
+    displayRefit = try container.decodeIfPresent(Bool.self, forKey: .displayRefit)
  }

  func encode(to encoder: Encoder) throws {
@@ -136,6 +139,9 @@ struct VMConfig: Codable {
    try container.encode(memorySize, forKey: .memorySize)
    try container.encode(macAddress.string, forKey: .macAddress)
    try container.encode(display, forKey: .display)
+    if let displayRefit = displayRefit {
+      try container.encode(displayRefit, forKey: .displayRefit)
+    }
  }

  mutating func setCPU(cpuCount: Int) throws {
@@ -11,7 +11,7 @@ enum OCIError: Error {
 }

 extension VMDirectory {
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
    // Pull VM's config file layer and re-serialize it into a config file
    let configLayers = manifest.layers.filter {
      $0.mediaType == configMediaType
@@ -24,7 +24,7 @@ extension VMDirectory {
    }
    let configFile = try FileHandle(forWritingTo: configURL)
    try await registry.pullBlob(configLayers.first!.digest) { data in
-      configFile.write(data)
+      try configFile.write(contentsOf: data)
    }
    try configFile.close()

@@ -54,12 +54,13 @@ extension VMDirectory {
    do {
      try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
                                  concurrency: concurrency, progress: progress,
-                                  localLayerCache: localLayerCache)
+                                  localLayerCache: localLayerCache,
+                                  deduplicate: deduplicate)
    } catch let error where error is FilterError {
      throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
    }

-    if let llc = localLayerCache {
+    if deduplicate, let llc = localLayerCache {
      // set custom attribute to remember deduplicated bytes
      diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
    }
@@ -78,7 +79,7 @@ extension VMDirectory {
    }
    let nvram = try FileHandle(forWritingTo: nvramURL)
    try await registry.pullBlob(nvramLayers.first!.digest) { data in
-      nvram.write(data)
+      try nvram.write(contentsOf: data)
    }
    try nvram.close()

@@ -86,7 +87,7 @@ extension VMDirectory {
    try manifest.toJSON().write(to: manifestURL)
  }

-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String, concurrency: UInt) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String, concurrency: UInt, labels: [String: String] = [:]) async throws -> RemoteName {
    var layers = Array<OCIManifestLayer>()

    // Read VM's config and push it as blob
@@ -120,7 +121,8 @@ extension VMDirectory {
    layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest))

    // Craft a stub OCI config for Docker Hub compatibility
-    let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os).toJSON()
+    let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels)
+    let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON()
    let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb)
    let manifest = OCIManifest(
      config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
@@ -24,6 +24,8 @@ class VMStorageHelper {
  private static func missingVMWrap<R: Any>(_ name: String, closure: () throws -> R) throws -> R {
    do {
      return try closure()
+    } catch RuntimeError.PIDLockMissing {
+      throw RuntimeError.VMDoesNotExist(name: name)
    } catch {
      if error.isFileNotFound() {
        throw RuntimeError.VMDoesNotExist(name: name)
@@ -59,6 +61,7 @@ enum RuntimeError : Error {
  case InvalidDiskSize(_ message: String)
  case FailedToUpdateAccessDate(_ message: String)
  case PIDLockFailed(_ message: String)
+  case PIDLockMissing(_ message: String)
  case FailedToParseRemoteName(_ message: String)
  case VMTerminationFailed(_ message: String)
  case ImproperlyFormattedHost(_ host: String, _ hint: String)
@@ -71,6 +74,7 @@ enum RuntimeError : Error {
  case OCIUnsupportedDiskFormat(_ format: String)
  case SuspendFailed(_ message: String)
  case PullFailed(_ message: String)
+  case VirtualMachineLimitExceeded(_ hint: String)
 }

 protocol HasExitCode {
@@ -104,6 +108,8 @@ extension RuntimeError : CustomStringConvertible {
      return message
    case .PIDLockFailed(let message):
      return message
+    case .PIDLockMissing(let message):
+      return message
    case .FailedToParseRemoteName(let cause):
      return "failed to parse remote name: \(cause)"
    case .VMTerminationFailed(let message):
@@ -128,6 +134,8 @@ extension RuntimeError : CustomStringConvertible {
      return "Failed to suspend the VM: \(message)"
    case .PullFailed(let message):
      return message
+    case .VirtualMachineLimitExceeded(let hint):
+      return "The number of VMs exceeds the system limit\(hint)"
    }
  }
 }
@@ -140,7 +140,7 @@ class VMStorageOCI: PrunableStorage {
    try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
  }

-  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
    SentrySDK.configureScope { scope in
      scope.setContext(value: ["imageName": name.description], key: "OCI")
    }
@@ -196,26 +196,29 @@ class VMStorageOCI: PrunableStorage {
      }

      try await withTaskCancellationHandler(operation: {
-        try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+        try await retry(maxAttempts: 5) {
          // Choose the best base image which has the most deduplication ratio
          let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)

          if let llc = localLayerCache {
            let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)

-            defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+            if deduplicate {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+            } else {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to avoid fetching \(deduplicatedHuman), will try use it...")
+            }
          }

-          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache)
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate)
        } recoverFromFailure: { error in
-          if error is RuntimeError {
-            return .throw
+          if error is URLError {
+            print("Error pulling image: \"\(error.localizedDescription)\", attempting to re-try...")
+
+            return .retry
          }

-          print("Error: \(error.localizedDescription)")
-          print("Attempting to re-try...")
-
-          return .retry
+          return .throw
        }
        try move(digestName, from: tmpVMDir)
        transaction.finish()
@@ -235,6 +238,9 @@ class VMStorageOCI: PrunableStorage {
      // are excluded from garbage collection
      VMDirectory(baseURL: vmURL(name)).markExplicitlyPulled()
    }
+
+    // to explicitly set the image as being accessed so it won't get pruned immediately
+    _ = try VMStorageOCI().open(name)
  }

  func linked(from: RemoteName, to: RemoteName) -> Bool {
@@ -31,7 +31,7 @@ class RegistryRunner {

  init() async throws {
    // Start container
-    let container = try Self.dockerCmd("run", "-d", "--rm", "-p", "5000", "registry:2")
+    let container = try Self.dockerCmd("run", "-d", "--rm", "-p", "127.0.0.1:0:5000", "registry:2")
      .trimmingCharacters(in: CharacterSet.newlines)
    containerID = container

@@ -2,10 +2,18 @@

 Tart comes with a Golang-based benchmarking utility that allows one to easily compare host and guest performance.

-Currently, only Flexible I/O tester workloads are supported. To run them, [install Golang](https://go.dev/) and run the following command from this (`benchmark/`) directory:
+Currently, only Flexible I/O tester workloads are supported. To run them, first make sure that [passwordless sudo](https://serverfault.com/questions/160581/how-to-setup-passwordless-sudo-on-linux) is configured.
+
+Then, [install Golang](https://go.dev/). The easiest way is through [Homebrew](https://brew.sh/):

 ```shell
-go run cmd/main.go fio
+brew install go
+```
+
+Finally, run the following command from this (`benchmark/`) directory:
+
+```shell
+go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sequoia-base:latest --prepare 'sudo purge && sync'
 ```

 You can also enable the debugging output to diagnose issues:
@@ -16,6 +24,8 @@ go run cmd/main.go fio --debug

 ## Results

+### Mar 27, 2024
+
 Host:

 * Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
@@ -37,3 +47,164 @@ Random writing of 100MB 	Tart    	2.0 GB/s 	493.31 kIOPS
 Random writing of 1000MB	local   	1.7 GB/s 	414.89 kIOPS  
 Random writing of 1000MB	Tart    	1.1 GB/s 	287.4 kIOPS  
 ```
+
+### Dec 2, 2024
+
+Host:
+
+* Hardware: MacBook Pro (Apple M1 Pro, 8 performance and 2 efficiency cores, 32 GB RAM, `MacBookPro18,3`)
+* OS: macOS Sequoia 15.1.1
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)          	Latency (write)         	Latency (sync)      
+Single 4KiB random write process        	local                                             	0 B/s     	19 MB/s    	0 IOPS     	4.81 kIOPS  	0s ± 0s                 	203.418µs ± 155.865µs   	0s ± 0s             
+Single 4KiB random write process        	Tart                                              	0 B/s     	18 MB/s    	0 IOPS     	4.54 kIOPS  	0s ± 0s                 	213.655µs ± 188.822µs   	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	19 MB/s    	0 IOPS     	4.68 kIOPS  	0s ± 0s                 	208.413µs ± 183.45µs    	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	24 MB/s    	0 IOPS     	6.11 kIOPS  	0s ± 0s                 	158.07µs ± 2.294654ms   	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	22 MB/s    	0 IOPS     	5.49 kIOPS  	0s ± 0s                 	173.414µs ± 310.213µs   	0s ± 0s             
+16 parallel 64KiB random write processes	local                                             	0 B/s     	18 GB/s    	0 IOPS     	273.76 kIOPS	0s ± 0s                 	323.423µs ± 604.999µs   	0s ± 0s             
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	16 GB/s    	0 IOPS     	273.48 kIOPS	0s ± 0s                 	335.086µs ± 7.591748ms  	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	18 GB/s    	0 IOPS     	281.49 kIOPS	0s ± 0s                 	326.655µs ± 7.485473ms  	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	17 GB/s    	0 IOPS     	266.79 kIOPS	0s ± 0s                 	340µs ± 7.868384ms      	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	16 GB/s    	0 IOPS     	251.02 kIOPS	0s ± 0s                 	355.077µs ± 8.354218ms  	0s ± 0s             
+Single 1MiB random write process        	local                                             	0 B/s     	1.3 GB/s   	0 IOPS     	1.31 kIOPS  	0s ± 0s                 	751.716µs ± 370.731µs   	0s ± 0s             
+Single 1MiB random write process        	Tart                                              	0 B/s     	1.1 GB/s   	0 IOPS     	1.1 kIOPS   	0s ± 0s                 	885.833µs ± 3.572539ms  	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.1 GB/s   	0 IOPS     	1.08 kIOPS  	0s ± 0s                 	898.427µs ± 3.464261ms  	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	1000 MB/s  	0 IOPS     	976.47 IOPS 	0s ± 0s                 	972.491µs ± 6.87654ms   	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	1.1 GB/s   	0 IOPS     	1.03 kIOPS  	0s ± 0s                 	925.545µs ± 4.261693ms  	0s ± 0s             
+Random reads/writes (4k)                	local                                             	62 MB/s   	62 MB/s    	15.37 kIOPS	15.37 kIOPS 	2.059453ms ± 1.431822ms 	2.098761ms ± 1.445082ms 	0s ± 0s             
+Random reads/writes (4k)                	Tart                                              	38 MB/s   	38 MB/s    	9.6 kIOPS  	9.61 kIOPS  	3.30369ms ± 1.500464ms  	3.350589ms ± 1.512986ms 	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	39 MB/s   	39 MB/s    	9.82 kIOPS 	9.83 kIOPS  	3.228106ms ± 1.367512ms 	3.27626ms ± 1.385964ms  	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	35 MB/s   	35 MB/s    	8.74 kIOPS 	8.76 kIOPS  	3.640772ms ± 15.472355ms	3.661779ms ± 15.264288ms	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	24 MB/s   	24 MB/s    	5.98 kIOPS 	5.99 kIOPS  	5.31188ms ± 4.55205ms   	5.375047ms ± 5.113847ms 	0s ± 0s             
+Random reads/writes (64k)               	local                                             	435 MB/s  	436 MB/s   	6.79 kIOPS 	6.8 kIOPS   	4.955892ms ± 2.066685ms 	4.440414ms ± 1.860036ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart                                              	352 MB/s  	353 MB/s   	5.5 kIOPS  	5.51 kIOPS  	5.946067ms ± 2.041124ms 	5.658948ms ± 1.928372ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	331 MB/s  	332 MB/s   	5.16 kIOPS 	5.17 kIOPS  	6.330765ms ± 1.726782ms 	6.033862ms ± 1.671028ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	428 MB/s  	428 MB/s   	6.68 kIOPS 	6.69 kIOPS  	4.661666ms ± 18.342779ms	4.904961ms ± 18.396772ms	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	297 MB/s  	298 MB/s   	4.64 kIOPS 	4.65 kIOPS  	6.591009ms ± 2.827053ms 	7.166883ms ± 3.001036ms 	0s ± 0s             
+sync test                               	local                                             	0 B/s     	48 MB/s    	0 IOPS     	21.15 kIOPS 	0s ± 0s                 	23.471µs ± 81.868µs     	23.374µs ± 6.255µs  
+sync test                               	Tart                                              	0 B/s     	24 MB/s    	0 IOPS     	10.72 kIOPS 	0s ± 0s                 	24.983µs ± 61.761µs     	67.575µs ± 76.196µs 
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	21 MB/s    	0 IOPS     	9.5 kIOPS   	0s ± 0s                 	26.973µs ± 63.935µs     	77.388µs ± 47.103µs 
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	30 MB/s    	0 IOPS     	13.19 kIOPS 	0s ± 0s                 	11.923µs ± 25.225µs     	62.894µs ± 208.933µs
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	38 MB/s    	0 IOPS     	17.02 kIOPS 	0s ± 0s                 	10.124µs ± 21.868µs     	47.803µs ± 33.706µs
+```
+
+### Dec 4, 2024
+
+Host:
+
+* AWS instance: `mac2.metal` + `gp3` EBS volume
+* Hardware: Mac mini (Apple M1, 4 performance and 4 efficiency cores, 16 GB RAM, `Macmini9,1`)
+* OS: macOS Sequoia 15.0
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)            	Latency (write)           	Latency (sync)         
+Single 4KiB random write process        	local                                             	0 B/s     	4.4 MB/s   	0 IOPS     	1.1 kIOPS   	0s ± 0s                   	702.357µs ± 359.925µs     	0s ± 0s                
+Single 4KiB random write process        	Tart                                              	0 B/s     	2.6 MB/s   	0 IOPS     	656.37 IOPS 	0s ± 0s                   	1.140086ms ± 1.450472ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	2.7 MB/s   	0 IOPS     	677.07 IOPS 	0s ± 0s                   	1.179872ms ± 1.219626ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	3.3 MB/s   	0 IOPS     	832.66 IOPS 	0s ± 0s                   	948.648µs ± 94.141338ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	15 MB/s    	0 IOPS     	3.65 kIOPS  	0s ± 0s                   	260.717µs ± 19.977757ms   	0s ± 0s                
+16 parallel 64KiB random write processes	local                                             	0 B/s     	9.5 GB/s   	0 IOPS     	147.89 kIOPS	0s ± 0s                   	753.289µs ± 8.028974ms    	0s ± 0s                
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	10 GB/s    	0 IOPS     	176.96 kIOPS	0s ± 0s                   	429.83µs ± 33.792264ms    	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	12 GB/s    	0 IOPS     	180.89 kIOPS	0s ± 0s                   	383.524µs ± 17.524971ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	336 MB/s   	0 IOPS     	5.24 kIOPS  	0s ± 0s                   	9.970844ms ± 365.808663ms 	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	9.4 GB/s   	0 IOPS     	147.04 kIOPS	0s ± 0s                   	524.139µs ± 34.100009ms   	0s ± 0s                
+Single 1MiB random write process        	local                                             	0 B/s     	178 MB/s   	0 IOPS     	173.36 IOPS 	0s ± 0s                   	3.835103ms ± 2.917977ms   	0s ± 0s                
+Single 1MiB random write process        	Tart                                              	0 B/s     	140 MB/s   	0 IOPS     	136.48 IOPS 	0s ± 0s                   	4.721178ms ± 7.744965ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	144 MB/s   	0 IOPS     	140.63 IOPS 	0s ± 0s                   	4.443507ms ± 11.572454ms  	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	47 MB/s    	0 IOPS     	45.55 IOPS  	0s ± 0s                   	13.267881ms ± 358.283094ms	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	196 MB/s   	0 IOPS     	191.4 IOPS  	0s ± 0s                   	4.102516ms ± 73.117503ms  	0s ± 0s                
+Random reads/writes (4k)                	local                                             	8.7 MB/s  	8.7 MB/s   	2.16 kIOPS 	2.16 kIOPS  	193.370794ms ± 42.593607ms	222.272016ms ± 56.586971ms	0s ± 0s                
+Random reads/writes (4k)                	Tart                                              	4.1 MB/s  	4.1 MB/s   	1.02 kIOPS 	1.03 kIOPS  	31.038867ms ± 13.508668ms 	31.184305ms ± 14.032766ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	4.2 MB/s  	4.2 MB/s   	1.04 kIOPS 	1.05 kIOPS  	30.368422ms ± 13.505627ms 	30.595412ms ± 13.840944ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	2.2 MB/s  	2.2 MB/s   	545.33 IOPS	548.86 IOPS 	59.31316ms ± 716.351086ms 	57.647852ms ± 711.503882ms	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	6.0 MB/s  	6.0 MB/s   	1.5 kIOPS  	1.5 kIOPS   	21.244222ms ± 47.808399ms 	21.39459ms ± 44.716307ms  	0s ± 0s                
+Random reads/writes (64k)               	local                                             	121 MB/s  	121 MB/s   	1.89 kIOPS 	1.89 kIOPS  	61.894699ms ± 21.353345ms 	73.176462ms ± 13.02948ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart                                              	72 MB/s   	72 MB/s    	1.12 kIOPS 	1.12 kIOPS  	27.842263ms ± 15.320781ms 	29.161858ms ± 15.765314ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	71 MB/s   	72 MB/s    	1.11 kIOPS 	1.11 kIOPS  	28.009493ms ± 16.333136ms 	29.285868ms ± 16.540589ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	28 MB/s   	28 MB/s    	441.85 IOPS	444.81 IOPS 	71.726725ms ± 633.215756ms	72.597238ms ± 630.969305ms	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	81 MB/s   	81 MB/s    	1.26 kIOPS 	1.26 kIOPS  	24.872043ms ± 36.980111ms 	25.568559ms ± 37.027145ms 	0s ± 0s                
+sync test                               	local                                             	0 B/s     	1.9 MB/s   	0 IOPS     	868.08 IOPS 	0s ± 0s                   	92.08µs ± 233.598µs       	1.059033ms ± 98.751µs  
+sync test                               	Tart                                              	0 B/s     	1.5 MB/s   	0 IOPS     	649.42 IOPS 	0s ± 0s                   	146.737µs ± 434.261µs     	1.391898ms ± 699.148µs 
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.3 MB/s   	0 IOPS     	568.82 IOPS 	0s ± 0s                   	158.736µs ± 504.002µs     	1.59798ms ± 14.161331ms
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	13 MB/s    	0 IOPS     	5.77 kIOPS  	0s ± 0s                   	26.596µs ± 832.169µs      	145.785µs ± 2.864048ms 
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	19 MB/s    	0 IOPS     	8.37 kIOPS  	0s ± 0s                   	20.135µs ± 108.817µs      	98.274µs ± 239.631µs   
+```
+
+Host:
+
+* AWS instance: `mac2.metal` + `gp3` EBS volume
+* Hardware: Mac mini (Apple M1, 4 performance and 4 efficiency cores, 16 GB RAM, `Macmini9,1`)
+* OS: macOS Sequoia 15.0
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sequoia 15.1
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)            	Latency (write)           	Latency (sync)         
+Single 4KiB random write process        	local                                             	0 B/s     	4.8 MB/s   	0 IOPS     	1.19 kIOPS  	0s ± 0s                   	690.818µs ± 326.595µs     	0s ± 0s                
+Single 4KiB random write process        	Tart                                              	0 B/s     	2.8 MB/s   	0 IOPS     	700.94 IOPS 	0s ± 0s                   	1.090362ms ± 918.444µs    	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	3.0 MB/s   	0 IOPS     	746.23 IOPS 	0s ± 0s                   	1.028192ms ± 974.533µs    	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	4.2 MB/s   	0 IOPS     	1.04 kIOPS  	0s ± 0s                   	916.36µs ± 105.318323ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	14 MB/s    	0 IOPS     	3.57 kIOPS  	0s ± 0s                   	269.796µs ± 22.419599ms   	0s ± 0s                
+16 parallel 64KiB random write processes	local                                             	0 B/s     	9.5 GB/s   	0 IOPS     	148.74 kIOPS	0s ± 0s                   	753.46µs ± 8.06509ms      	0s ± 0s                
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	5.2 GB/s   	0 IOPS     	81.46 kIOPS 	0s ± 0s                   	778.624µs ± 11.705178ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	5.3 GB/s   	0 IOPS     	83.47 kIOPS 	0s ± 0s                   	865.448µs ± 38.369176ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	116 MB/s   	0 IOPS     	1.8 kIOPS   	0s ± 0s                   	37.601112ms ± 727.319309ms	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	5.3 GB/s   	0 IOPS     	83.19 kIOPS 	0s ± 0s                   	900.751µs ± 51.223205ms   	0s ± 0s                
+Single 1MiB random write process        	local                                             	0 B/s     	177 MB/s   	0 IOPS     	173.27 IOPS 	0s ± 0s                   	3.833194ms ± 2.873871ms   	0s ± 0s                
+Single 1MiB random write process        	Tart                                              	0 B/s     	151 MB/s   	0 IOPS     	147.44 IOPS 	0s ± 0s                   	4.925853ms ± 7.793808ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	151 MB/s   	0 IOPS     	147.87 IOPS 	0s ± 0s                   	4.884797ms ± 7.563512ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	72 MB/s    	0 IOPS     	69.9 IOPS   	0s ± 0s                   	8.909771ms ± 214.311644ms 	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	159 MB/s   	0 IOPS     	155.69 IOPS 	0s ± 0s                   	4.863448ms ± 88.965211ms  	0s ± 0s                
+Random reads/writes (4k)                	local                                             	8.7 MB/s  	8.7 MB/s   	2.16 kIOPS 	2.16 kIOPS  	193.353233ms ± 42.728494ms	222.325905ms ± 56.901372ms	0s ± 0s                
+Random reads/writes (4k)                	Tart                                              	3.5 MB/s  	3.5 MB/s   	862.89 IOPS	865.54 IOPS 	36.893229ms ± 12.644216ms 	37.143334ms ± 12.772017ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	3.6 MB/s  	3.6 MB/s   	907.4 IOPS 	911.55 IOPS 	35.048969ms ± 10.559354ms 	35.3046ms ± 10.67824ms    	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	2.7 MB/s  	2.8 MB/s   	684.11 IOPS	688.05 IOPS 	48.815322ms ± 687.806727ms	44.395556ms ± 635.532064ms	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	7.0 MB/s  	7.0 MB/s   	1.74 kIOPS 	1.74 kIOPS  	18.00448ms ± 93.784447ms  	18.617037ms ± 107.423001ms	0s ± 0s                
+Random reads/writes (64k)               	local                                             	121 MB/s  	121 MB/s   	1.89 kIOPS 	1.89 kIOPS  	61.983727ms ± 21.324782ms 	73.228597ms ± 12.730945ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart                                              	75 MB/s   	75 MB/s    	1.17 kIOPS 	1.17 kIOPS  	26.830538ms ± 7.643051ms  	27.709602ms ± 7.830965ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	76 MB/s   	77 MB/s    	1.19 kIOPS 	1.19 kIOPS  	26.255337ms ± 7.302592ms  	27.256805ms ± 7.388266ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	32 MB/s   	33 MB/s    	505.26 IOPS	508.66 IOPS 	65.170269ms ± 747.794957ms	61.062186ms ± 695.614904ms	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	79 MB/s   	79 MB/s    	1.23 kIOPS 	1.23 kIOPS  	25.861503ms ± 171.669777ms	25.992302ms ± 164.647788ms	0s ± 0s                
+sync test                               	local                                             	0 B/s     	1.9 MB/s   	0 IOPS     	865.16 IOPS 	0s ± 0s                   	100.95µs ± 268.722µs      	1.054051ms ± 365.377µs 
+sync test                               	Tart                                              	0 B/s     	1.6 MB/s   	0 IOPS     	704.13 IOPS 	0s ± 0s                   	133.886µs ± 390.263µs     	1.285085ms ± 575.27µs  
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.6 MB/s   	0 IOPS     	728.26 IOPS 	0s ± 0s                   	129.246µs ± 472.724µs     	1.242713ms ± 1.281286ms
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	35 MB/s    	0 IOPS     	15.67 kIOPS 	0s ± 0s                   	11.319µs ± 24.771µs       	51.731µs ± 42.208µs    
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	17 MB/s    	0 IOPS     	7.39 kIOPS  	0s ± 0s                   	21.23µs ± 81.749µs        	113.239µs ± 191.266µs  
+```
+
+### March 23, 2025
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sequoia 15.3.2
+* Xcode: 16.2
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 15.3.2 
+* Xcode: 16.2
+
+```
+Name                    	Executor                                          	Time 
+XcodeBenchmark (d869315)	local                                             	2m19s
+XcodeBenchmark (d869315)	Tart                                              	3m59s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	3m48s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	3m35s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	3m14s
+```
@@ -1,21 +1,24 @@
 module github.com/cirruslabs/tart/benchmark

 go 1.22.1
+toolchain go1.24.1

 require (
 	github.com/avast/retry-go/v4 v4.5.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/google/uuid v1.6.0
 	github.com/gosuri/uitable v0.0.4
+	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.21.0
+	golang.org/x/crypto v0.35.0
 )

 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -23,7 +26,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
@@ -7,6 +7,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
@@ -26,26 +28,31 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
+golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
@@ -7,23 +7,63 @@ type Benchmark struct {

 var benchmarks = []Benchmark{
 	{
-		Name: "Random writing of 1MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 1MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Ars Technica's "Single 4KiB random write process" test[1]
+		// with JSON output and created file cleanup
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "Single 4KiB random write process",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=4k --size=4g --numjobs=1 --iodepth=1 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1",
 	},
 	{
-		Name: "Random writing of 10MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 10MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Ars Technica's "16 parallel 64KiB random write processes" test[1]
+		// with JSON outpu, created file cleanup and group reporting (for
+		// easier analysis)
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "16 parallel 64KiB random write processes",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=64k --size=256m --numjobs=16 --iodepth=16 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1 --group_reporting",
 	},
 	{
-		Name: "Random writing of 100MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 100MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Ars Technica's "16 parallel 64KiB random write processes" test[1]
+		// with JSON output, created file cleanup and reduced file I/O size
+		// from 16 to 10 GB to avoid "No space left on device".
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "Single 1MiB random write process",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=1m --size=10g --numjobs=1 --iodepth=1 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1",
 	},
 	{
-		Name: "Random writing of 1000MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 1000MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Oracle's "Test random read/writes" (in IOPS Performance Tests[1]) category
+		// with JSON output, created file cleanup, without ETA newline, without custom
+		// file path, with file I/O size reduced from 500GB to 2GB to prevent
+		// "No space left on device" and with posixaio instead of libaio.
+		//
+		// [1]: https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm#FIO_Commands
+		Name: "Random reads/writes (4k)",
+		Command: "fio --name=benchmark --size=2GB --direct=1 --rw=randrw --bs=4k --ioengine=posixaio --iodepth=256 --runtime=120 --numjobs=4 --time_based --group_reporting" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// Oracle's "Test random read/writes" (in Throughput Performance Tests[1]) category
+		// with JSON output, created file cleanup, without ETA newline, without custom
+		// file path, with file I/O size reduced from 500GB to 2GB to prevent
+		// "No space left on device" and with posixaio instead of libaio.
+		//
+		// [1]: https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm#Throughput_Performance_Tests
+		Name: "Random reads/writes (64k)",
+		Command: "fio --name=benchmark --size=2GB --direct=1 --rw=randrw --bs=64k --ioengine=posixaio --iodepth=64 --runtime=120 --numjobs=4 --time_based --group_reporting" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// RedHat's "How can I test to see if my environment is fast enough for etcd"[1]
+		// with custom name
+		//
+		// [1]: https://access.redhat.com/solutions/5726511
+		Name: "sync test",
+		Command: "mkdir -p test-data && fio --name=benchmark --rw=write --ioengine=sync --fdatasync=1 --directory=test-data --size=22m --bs=2300" +
+			" --output-format json --unlink 1",
 	},
 }
@@ -1,20 +1,21 @@
 package fio

 import (
-	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"github.com/cirruslabs/tart/benchmark/internal/executor"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
 	"github.com/dustin/go-humanize"
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
 )

 var debug bool
+var image string
+var prepare string

 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
@@ -24,6 +25,8 @@ func NewCommand() *cobra.Command {
 	}

 	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-base:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")

 	return cmd
 }
@@ -41,28 +44,50 @@ func run(cmd *cobra.Command, args []string) error {
 		_ = logger.Sync()
 	}()

-	executors, err := initializeExecutors(cmd.Context(), logger)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		errs := []error{err}
-
-		for _, executor := range executors {
-			if err := executor.Close(); err != nil {
-				errs = append(errs, fmt.Errorf("failed to close executor %s: %w", executor.Name(), err))
-			}
-		}
-
-		err = errors.Join(errs...)
-	}()
-
 	table := uitable.New()
-	table.AddRow("Name", "Executor", "Bandwidth", "I/O operations")
+	table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)",
+		"Latency (read)", "Latency (write)", "Latency (sync)")

 	for _, benchmark := range benchmarks {
-		for _, executor := range executors {
-			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, executor.Name())
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("installing Flexible I/O tester (fio) on executor %s",
+				executorInitializer.Name)
+
+			if _, err := executor.Run(cmd.Context(), "brew install fio"); err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)

 			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
 			if err != nil {
@@ -82,12 +107,28 @@ func run(cmd *cobra.Command, args []string) error {

 			job := fioResult.Jobs[0]

+			readBandwidth := humanize.Bytes(uint64(job.Read.BW)*humanize.KByte) + "/s"
+			readIOPS := humanize.SIWithDigits(job.Read.IOPS, 2, "IOPS")
+
+			logger.Sugar().Infof("read bandwidth: %s, read IOPS: %s, read latency: %s",
+				readBandwidth, readIOPS, job.Read.LatencyNS.String())
+
 			writeBandwidth := humanize.Bytes(uint64(job.Write.BW)*humanize.KByte) + "/s"
 			writeIOPS := humanize.SIWithDigits(job.Write.IOPS, 2, "IOPS")

-			logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s\n", writeBandwidth, writeIOPS)
+			logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s, write latency: %s",
+				writeBandwidth, writeIOPS, job.Write.LatencyNS.String())

-			table.AddRow(benchmark.Name, executor.Name(), writeBandwidth, writeIOPS)
+			logger.Sugar().Infof("sync latency: %s", job.Sync.LatencyNS.String())
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, readBandwidth, writeBandwidth,
+				readIOPS, writeIOPS, job.Read.LatencyNS.String(), job.Write.LatencyNS.String(),
+				job.Sync.LatencyNS.String())
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
 		}
 	}

@@ -95,37 +136,3 @@ func run(cmd *cobra.Command, args []string) error {

 	return nil
 }
-
-func initializeExecutors(ctx context.Context, logger *zap.Logger) ([]executor.Executor, error) {
-	var result []executor.Executor
-
-	logger.Info("initializing local executor")
-
-	local, err := local.New(logger)
-	if err != nil {
-		return nil, err
-	}
-	result = append(result, local)
-
-	logger.Info("local executor initialized")
-
-	logger.Info("initializing Tart executor")
-
-	tart, err := tart.New(ctx, logger)
-	if err != nil {
-		return nil, err
-	}
-	result = append(result, tart)
-
-	logger.Info("Tart executor initialized")
-
-	for _, executor := range result {
-		logger.Sugar().Infof("installing Flexible I/O tester (fio) on %s executor", executor.Name())
-
-		if _, err := executor.Run(ctx, "brew install fio"); err != nil {
-			return nil, err
-		}
-	}
-
-	return result, nil
-}
@@ -1,15 +1,35 @@
 package fio

+import (
+	"fmt"
+	"time"
+)
+
 type Result struct {
 	Jobs []Job `json:"jobs"`
 }

 type Job struct {
 	Name  string `json:"jobname"`
-	Write Write  `json:"write"`
+	Read  Stats  `json:"read"`
+	Write Stats  `json:"write"`
+	Sync  Stats  `json:"sync"`
 }

-type Write struct {
-	BW   float64 `json:"bw"`
-	IOPS float64 `json:"iops"`
+type Stats struct {
+	BW        float64 `json:"bw"`
+	IOPS      float64 `json:"iops"`
+	LatencyNS Latency `json:"lat_ns"`
+}
+
+type Latency struct {
+	Mean   float64 `json:"mean"`
+	Stddev float64 `json:"stddev"`
+}
+
+func (latency Latency) String() string {
+	meanDuration := time.Duration(latency.Mean) * time.Nanosecond
+	stddevDuration := time.Duration(latency.Stddev) * time.Nanosecond
+
+	return fmt.Sprintf("%v ± %v", meanDuration, stddevDuration)
 }
@@ -2,6 +2,7 @@ package command

 import (
 	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
 	"github.com/spf13/cobra"
 )

@@ -14,6 +15,7 @@ func NewCommand() *cobra.Command {

 	cmd.AddCommand(
 		fio.NewCommand(),
+		xcode.NewCommand(),
 	)

 	return cmd
@@ -0,0 +1,13 @@
+package xcode
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		Name:    "XcodeBenchmark (d869315)",
+		Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && xcrun simctl list && sh benchmark.sh",
+	},
+}
@@ -0,0 +1,49 @@
+package xcode
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+type Output struct {
+	Started time.Time
+	Ended   time.Time
+}
+
+func ParseOutput(s string) (*Output, error) {
+	// Ensure that the build has succeeded
+	matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err)
+	}
+	if !matched {
+		return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " +
+			"not found on a separate line, make sure you have Xcode installed")
+	}
+
+	re := regexp.MustCompile("Started\\s+(?P<started>.*)\\n.*Ended\\s+(?P<ended>.*)\\n")
+
+	matches := re.FindStringSubmatch(s)
+
+	if len(matches) != re.NumSubexp()+1 {
+		return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times")
+	}
+
+	startedRaw := matches[re.SubexpIndex("started")]
+	started, err := time.Parse(time.TimeOnly, startedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw)
+	}
+
+	endedRaw := matches[re.SubexpIndex("ended")]
+	ended, err := time.Parse(time.TimeOnly, endedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw)
+	}
+
+	return &Output{
+		Started: started,
+		Ended:   ended,
+	}, nil
+}
@@ -0,0 +1,37 @@
+package xcode_test
+
+import (
+	"fmt"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestParseOutput(t *testing.T) {
+	result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec]
+
+System Version: 14.6
+Xcode 15.4
+Hardware Overview
+      Model Name: Apple Virtual Machine 1
+      Model Identifier: VirtualMac2,1
+      Total Number of Cores: 4
+      Memory: 8 GB
+
+✅ XcodeBenchmark has completed
+1️⃣  Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include:
+	- Build Time (See ** BUILD SUCCEEDED ** [XYZ sec])
+	- System Version
+	- Xcode Version
+	- Hardware Overview
+	- Started 13:46:20
+	- Ended   13:50:02
+	- Date Thu Jan 16 13:50:02 UTC 2025
+
+2️⃣  Share your results at https://github.com/devMEremenko/XcodeBenchmark
+`)
+	require.NoError(t, err)
+	fmt.Println(result)
+	require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started))
+}
@@ -0,0 +1,108 @@
+package xcode
+
+import (
+	"fmt"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
+)
+
+var debug bool
+var image string
+var prepare string
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "xcode",
+		Short: "run XCode benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sequoia-xcode:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "Time")
+
+	for _, benchmark := range benchmarks {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			output, err := ParseOutput(string(stdout))
+			if err != nil {
+				return err
+			}
+
+			duration := output.Ended.Sub(output.Started)
+
+			logger.Sugar().Infof("Xcode benchmark duration: %s", duration)
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, duration)
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}
@@ -5,7 +5,6 @@ import (
 )

 type Executor interface {
-	Name() string
 	Run(ctx context.Context, command string) ([]byte, error)
 	Close() error
 }
@@ -0,0 +1,57 @@
+package executor
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"go.uber.org/zap"
+)
+
+type Initializer struct {
+	Name string
+	Fn   func() (Executor, error)
+}
+
+func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer {
+	return []Initializer{
+		{
+			Name: "local",
+			Fn: func() (Executor, error) {
+				return local.New(logger)
+			},
+		},
+		{
+			Name: "Tart",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, nil, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"caching=cached",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none,caching=cached",
+				}, logger)
+			},
+		},
+	}
+}
@@ -7,15 +7,18 @@ import (
 	"fmt"
 	"github.com/avast/retry-go/v4"
 	"github.com/google/uuid"
+	"github.com/shirou/gopsutil/mem"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
 	"golang.org/x/crypto/ssh"
+	"io"
 	"net"
+	"runtime"
+	"strconv"
 	"strings"
 	"time"
 )

-const baseImage = "ghcr.io/cirruslabs/macos-sonoma-base:latest"
-
 type Tart struct {
 	vmRunCancel context.CancelFunc
 	vmName      string
@@ -23,17 +26,34 @@ type Tart struct {
 	logger      *zap.Logger
 }

-func New(ctx context.Context, logger *zap.Logger) (*Tart, error) {
+func New(ctx context.Context, image string, runArgsExtra []string, logger *zap.Logger) (*Tart, error) {
 	tart := &Tart{
 		vmName: fmt.Sprintf("tart-benchmark-%s", uuid.NewString()),
 		logger: logger,
 	}

-	if err := Cmd(ctx, tart.logger, "pull", baseImage); err != nil {
+	if err := Cmd(ctx, tart.logger, "pull", image); err != nil {
 		return nil, err
 	}

-	if err := Cmd(ctx, tart.logger, "clone", baseImage, tart.vmName); err != nil {
+	if err := Cmd(ctx, tart.logger, "clone", image, tart.vmName); err != nil {
+		return nil, err
+	}
+
+	vmStat, err := mem.VirtualMemory()
+	if err != nil {
+		return nil, err
+	}
+
+	cpus := strconv.Itoa(runtime.NumCPU())
+	memory := strconv.FormatUint(vmStat.Total/1024/1024, 10)
+	logger.Info("Setting resources", zap.String("cpus", cpus), zap.String("memory", memory))
+	setResourcesArguments := []string{
+		"set", tart.vmName,
+		"--cpu", cpus,
+		"--memory", memory,
+	}
+	if err := Cmd(ctx, tart.logger, setResourcesArguments...); err != nil {
 		return nil, err
 	}

@@ -41,7 +61,11 @@ func New(ctx context.Context, logger *zap.Logger) (*Tart, error) {
 	tart.vmRunCancel = vmRunCancel

 	go func() {
-		_ = Cmd(vmRunCtx, tart.logger, "run", "--no-graphics", tart.vmName)
+		runArgs := []string{"run", "--no-graphics", tart.vmName}
+
+		runArgs = append(runArgs, runArgsExtra...)
+
+		_ = Cmd(vmRunCtx, tart.logger, runArgs...)
 	}()

 	ip, err := CmdWithOutput(ctx, tart.logger, "ip", "--wait", "60", tart.vmName)
@@ -105,10 +129,12 @@ func (tart *Tart) Run(ctx context.Context, command string) ([]byte, error) {
 	}()
 	defer monitorCancel()

+	loggerWriter := &zapio.Writer{Log: tart.logger, Level: zap.DebugLevel}
 	stdoutBuf := &bytes.Buffer{}

 	sshSession.Stdin = bytes.NewBufferString(command)
-	sshSession.Stdout = stdoutBuf
+	sshSession.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	sshSession.Stderr = loggerWriter

 	if err := sshSession.Shell(); err != nil {
 		return nil, err
@@ -11,7 +11,7 @@ import (
 func TestTart(t *testing.T) {
 	ctx := context.Background()

-	tart, err := tart.New(ctx, zap.NewNop())
+	tart, err := tart.New(ctx, "ghcr.io/cirruslabs/macos-sonoma-base:latest", nil, zap.NewNop())
 	require.NoError(t, err)

 	output, err := tart.Run(ctx, "echo \"this is a test\"")
@@ -5,6 +5,16 @@ title: Frequently Asked Questions
 description: Advanced configuration and troubleshooting tips for advanced configurations.
 ---

+## Troubleshooting crashes
+
+If you experience a crash or encounter another error while using the tart executable, you can collect debug information to assist with troubleshooting. Run the following command in a separate terminal window to gather logs from the Tart process and the macOS Virtualization subsystem:
+
+```shell
+log stream --predicate='process=="tart" OR process CONTAINS "Virtualization"' > tart.log
+```
+
+While the events are being streamed, attempt to reproduce the issue. Once the issue is reproduced, stop the streaming by pressing Ctrl+C. Then, attach the tart.log file to your report.
+
 ## VM location on disk

 Tart stores all its files in `~/.tart/` directory. Local images that you can run are stored in `~/.tart/vms/`.
@@ -13,12 +23,12 @@ Remote images are pulled into `~/.tart/cache/OCIs/`.
 ## Nested virtualization support?

 Tart is limited by functionality of Apple's `Virtualization.Framework`. At the moment `Virtualization.Framework`
-doesn't support nested virtualization.
+supports nested virtualization only on M3 or M4 chips running macOS 15 (Sequoia). By default, it is disabled, but can be enabled by passing the `--nested` flag to `tart run`.

 ## Connecting to a service running on host

 To connect from within a virtual machine to a service running on the host machine
-please first make sure that the service is binded to `0.0.0.0`.
+please first make sure that the service is bound to `0.0.0.0`.

 Then from within a virtual machine you can access the service using the router's IP address that you can get either from `Preferences -> Network`
 or by running the following command in the Terminal:
@@ -29,7 +39,7 @@ netstat -nr | grep default | head -n 1 | awk '{print $2}'

 Note: that accessing host is only possible with the default NAT network. If you are running your virtual machines with
 [Softnet](https://github.com/cirruslabs/softnet) (via `tart run --net-softnet <VM NAME>)`, then the network isolation
-is stricter and it's not only possible to access the host.
+is stricter and it's not possible to access the host.

 ## Changing the default NAT subnet

@@ -57,7 +67,66 @@ This issue is worked around automatically [when using Softnet](http://github.com
 sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.InternetSharing.default.plist bootpd -dict DHCPLeaseTimeSecs -int 600
 ```

-Note that this tweak persists across reboots, so normally you'll only need to do it once per new host.
+This tweak persists across reboots, so normally you'll only need to do it once per new host.
+
+If that doesn't help after starting a new VM, it's possible that the `/var/db/dhcpd_leases` file is already overfilled with 86,400-second leases. You can remove it with the following command and try starting a new VM again:
+
+```shell
+sudo rm /var/db/dhcpd_leases
+```
+
+And no worries, this file will be re-created on the next `tart run`.
+
+## Unsupported DHCP client identifiers
+
+Due to the limitations of the macOS built-in DHCP server, `tart ip` is unable to correctly report the IP addresses for VMs using DHCP client identifiers that are not based on VMs link-layer addresses (MAC addresses).
+
+By default, when [no `--resolver=arp` is specified](#resolving-the-vms-ip-when-using-bridged-networking), `tart ip` reads the `/var/db/dhcpd_leases` file and tries to find the freshest entry that matches the VM's MAC address (based on the `hw_address` field).
+
+However, things starts to break when the VM uses a [DUID-EN](https://metebalci.com/blog/a-note-on-dhcpv6-duid-and-prefix-delegation#duid-types) identifier, for example. One of the notorious examples of this being Ubuntu, using this type of identifier by default on latest versions.  This results in the `/var/db/dhcpd_leases` entry for Ubuntu appearing as follows:
+
+```ini
+{
+    name=ubuntu
+    ip_address=192.168.64.3
+    hw_address=ff,f1:f5:dd:7f:0:2:0:0:ab:11:cb:fb:30:b0:97:b6:3a:67
+    identifier=ff,f1:f5:dd:7f:0:2:0:0:ab:11:cb:fb:30:b0:97:b6:3a:67
+    lease=0x678e2ce7
+}
+```
+
+Because the macOS built-in DHCP server overwrites the `hw_address` with the `identifier`, it leaves no information about the VM's MAC address to the `tart ip`.
+
+To avoid this issue, make sure that your VM only sends a DHCP client identifier (option 61) with link-layer address (MAC address) or that it doesn't send this option at all.
+
+For the aforementioned Ubuntu, the solution is outlined in the section [How to integrate with Windows DHCP Server](https://netplan.readthedocs.io/en/stable/examples/#how-to-integrate-with-windows-dhcp-server) of Canonical Netplan's documentation:
+
+```yaml
+network:
+  version: 2
+  ethernets:
+    enp3s0:
+      dhcp4: yes
+      dhcp-identifier: mac
+```
+
+## Resolving the VM's IP when using bridged networking
+
+When running `tart run` with `--net-bridged`, you need to invoke `tart ip` differently, because the macOS built-in DHCP server won't have any information about the VM's IP-address:
+
+```shell
+tart ip --resolver=arp <VM>
+```
+
+This causes the `tart ip` to consult the host's ARP table instead of the `/var/db/dhcpd_leases` file.
+
+Note that this method of resolving the IP heavily relies on the level of VM's activity on the network, namely, exchanging ARP requests between the guest and the host.
+
+This is normally not an issue for macOS VMs, but on Linux VMs you might need to install Samba, which includes a [NetBIOS name server](https://www.samba.org/samba/docs/current/man-html/nmbd.8.html) and exhibits the same behavior as macOS, resulting in the population of the ARP table of the host OS:
+
+```shell
+sudo apt-get install samba
+```

 ## Running login/clone/pull/push commands over SSH

@@ -89,4 +158,89 @@ or features supported. If there is some feature missing please don't hesitate to
 Instead of Anka Registry, Tart can work with any OCI-compatible container registry. This provides a much more consistent
 and scalable experience for distributing virtual machines.

-Tart doesn't yet have an analogue of Anka Controller for managing long living VMs but [soon will be](https://github.com/cirruslabs/tart/issues/372).
+Tart does have an analogue of Anka Controller for managing VMs across a cluster of Mac hosts called [Orchard](orchard/quick-start.md).
+
+## Automatic pruning
+
+`tart pull` and `tart clone` commands check the remaining space available on the volume associated with `TART_HOME` directory (defaults to `~/.tart`) before pulling or cloning anything.
+
+In case there's not enough space to fit the newly pulled or cloned VM image, Tart will remove the least recently accessed VMs from OCI cache and `.ipsw` files from IPSW cache until enough free space is available.
+
+To disable this functionality, set the `TART_NO_AUTO_PRUNE` environment variable either globally:
+
+```shell
+export TART_NO_AUTO_PRUNE=
+```
+
+...or per `tart pull` and `tart clone` invocation as follows:
+
+```shell
+TART_NO_AUTO_PRUNE= tart pull ...
+```
+
+## Disk resizing
+
+Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images have the `cloud-initramfs-growroot` package installed that runs on boot) and on the rest of the distributions by running the `growpart` or `resize2fs` commands.
+
+For macOS, however, things are a bit more complicated, and you generally have two options: automated and manual resizing.
+
+For the automated option, you can use [Packer](https://www.packer.io/) with the [Packer builder for Tart VMs](https://developer.hashicorp.com/packer/integrations/cirruslabs/tart/latest/components/builder/tart). The latter has two has configuration directives related to the disk resizing behavior:
+
+* [`disk_size_gb`](https://developer.hashicorp.com/packer/integrations/cirruslabs/tart/latest/components/builder/tart#configuration-reference) — controls the target disk size in gigabytes
+* [`recovery_partition`](https://developer.hashicorp.com/packer/integrations/cirruslabs/tart/latest/components/builder/tart#configuration-reference) — controls what to do with the recovery partition when resizing the disk
+    * you can either keep, delete or relocate it to the end of the disk
+
+For the manual approach, you have to remove the recovery partition first, repair the disk and the resize the APFS container.
+
+To do this, first we'll need to identify the primary disk and the APFS containers by running the command below from within a VM:
+
+```shell
+diskutil list physical
+```
+
+For example, the output might look like this:
+
+```plain
+/dev/disk0 (internal, physical):
+   #:                       TYPE NAME                    SIZE       IDENTIFIER
+   0:      GUID_partition_scheme                        *100.0 GB   disk0
+   1:             Apple_APFS_ISC Container disk1         524.3 MB   disk0s1
+   2:                 Apple_APFS Container disk3         44.1 GB    disk0s2
+   3:        Apple_APFS_Recovery Container disk2         5.4 GB     disk0s3
+                    (free space)                         50.0 GB    -
+```
+
+In the output, you'll normally see:
+
+* a single physical disk (`disk0`)
+* APFS container with the system partition which we're going to resize (`disk0s2`)
+* APFS container with the recovery partition which we're going to delete (`disk0s3`)
+* `(free space)` which we'll put to use
+
+To proceed, boot the VM in recovery mode using `tart run --recovery` and choose the "Options" item:
+
+![](assets/images/faq/tart-run-recovery-options.png){width="640" .center}
+
+When the recovery OS boots, open the Terminal app:
+
+![](assets/images/faq/tart-run-recovery-terminal.png){width="720" .center}
+
+In Terminal app, invoke the command below to remove the recovery partition:
+
+```shell
+diskutil eraseVolume free free disk0s3
+```
+
+Now, repair the disk:
+
+```shell
+yes | diskutil repairDisk disk0
+```
+
+Finally, resize the system APFS container to take all the remaining space:
+
+```shell
+diskutil apfs resizeContainer disk0s2 0
+```
+
+Now, you can shut down and `tart run` as you'd normally do.
@@ -18,9 +18,9 @@ steps:
 - command: uname -a
  plugins:
  - cirruslabs/tart#main:
-    image: ghcr.io/cirruslabs/macos-sonoma-base:latest
+    image: ghcr.io/cirruslabs/macos-sequoia-base:latest
 ```

-This will run `uname -r` in a macOS Tart VM cloned from `ghcr.io/cirruslabs/macos-sonoma-base:latest`.
+This will run `uname -r` in a macOS Tart VM cloned from `ghcr.io/cirruslabs/macos-sequoia-base:latest`.

 See plugin's [Configuration section](https://github.com/cirruslabs/tart-buildkite-plugin#configuration) for the full list of available options.
@@ -18,7 +18,7 @@ task:
  name: hello
  macos_instance:
    # can be a remote or a local virtual machine
-    image: ghcr.io/cirruslabs/macos-sonoma-base:latest
+    image: ghcr.io/cirruslabs/macos-sequoia-base:latest
  hello_script:
    - echo "Hello from within a Tart VM!"
    - echo "Here is my CPU info:"
@@ -50,7 +50,7 @@ exposes it via [`artifacts` instruction](https://cirrus-ci.org/guide/writing-tas
 task:
  name: Build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sequoia-xcode:latest
  build_script: swift build --product tart
  binary_artifacts:
    path: .build/debug/tart
@@ -42,7 +42,7 @@ Now you can use Tart Images in your `.gitlab-ci.yml`:
 ```yaml
 # You can use any remote Tart Image.
 # Tart Executor will pull it from the registry and use it for creating ephemeral VMs.
-image: ghcr.io/cirruslabs/macos-sonoma-base:latest
+image: ghcr.io/cirruslabs/macos-sequoia-base:latest

 test:
  tags:
@@ -7,7 +7,7 @@ description: Use Packer to build custom VM images, configure VMs and work with r

 ## Creating from scratch

-Tart supports macOS and Linux virtual machines. All commands like `run` and `pull` work the same way regarding of the underlying OS a particular VM image has.
+Tart supports macOS and Linux virtual machines. All commands like `run` and `pull` work the same way regardless of the underlying OS a particular VM image has.
 The only difference is how such VM images are created. Please check sections below for [macOS](#creating-a-macos-vm-image-from-scratch) and [Linux](#creating-a-linux-vm-image-from-scratch) instructions.

 ### Creating a macOS VM image from scratch
@@ -16,8 +16,8 @@ Tart can create VMs from `*.ipsw` files. You can download a specific `*.ipsw` fi
 use `latest` instead of a path to `*.ipsw` to download the latest available version:

 ```bash
-tart create --from-ipsw=latest sonoma-vanilla
-tart run sonoma-vanilla
+tart create --from-ipsw=latest sequoia-vanilla
+tart run sequoia-vanilla
 ```

 After the initial booting of the VM, you'll need to manually go through the macOS installation process. As a convention we recommend creating an `admin` user with an `admin` password. After the regular installation please do some additional modifications in the VM:
@@ -72,8 +72,8 @@ packer {
 }

 source "tart-cli" "tart" {
-  vm_base_name = "ghcr.io/cirruslabs/macos-sonoma-base:latest"
-  vm_name      = "my-custom-sonoma"
+  vm_base_name = "ghcr.io/cirruslabs/macos-sequoia-base:latest"
+  vm_name      = "my-custom-sequoia"
  cpu_count    = 4
  memory_gb    = 8
  disk_size_gb = 70
@@ -0,0 +1,67 @@
+## Architecture
+
+Orchard cluster consists of three components:
+
+* Controller — responsible for managing the cluster and scheduling of resources 
+* Worker — responsible for executing the VMs
+* Client — responsible for creating, modifying and removing the resources on the Controller, can either be an [Orchard CLI](/orchard/using-orchard-cli) or [an API consumer](/orchard/integration-guide)
+
+At the moment, only one Controller instance is currently supported, while you can deploy one or more Workers and run any number of Clients.
+
+In terms of networking requirements, only Controller needs to be directly accessible from Workers and Clients, while Workers and Clients can be deployed and run anywhere (e.g. behind a NAT).
+
+## Security
+
+When an Orchard Client or a Worker connects to the Controller, they need to establish trust and verify that they're talking to the right Controller, so that no [man-in-the-middle attack](https://en.wikipedia.org/wiki/Man-in-the-middle_attack) is possible.
+
+Similarly to web-browsers (that rely on the [public key infrastructure](https://en.wikipedia.org/wiki/Public_key_infrastructure)) and SSH (which relies on semi-automated fingerprint verification), Orchard combines these two traits in a hybrid approach by defaulting to automatic PKI verification (can be disabled by [`--no-pki`](#--no-pki-override)) and falling-back to a manual verification for self-signed certificates.
+
+This hybrid approach is needed because the Controller can be configured in two ways:
+
+* *Controller with a publicly valid certificate*
+    * can be configured manually by passing `--controller-cert` and `--controller-key` command-line arguments to `orchard controller run`
+* *Controller with a self-signed certificate*
+    * configured automatically on first Controller start-up when no `--controller-cert` and `--controller-key` command-line arguments are passed
+
+Below we'll explain how Orchard client and Worker secure the connection when accessing these two Controller types.
+
+### Client
+
+Client is associated with the Controller using a `orchard context create` command, which works as follows:
+
+* Client attempts to connect to the Controller and validate its certificate using host's root CA set (can be disabled with [`--no-pki`](#--no-pki-override))
+* if the Client encounters a  *Controller with a publicly valid certificate*, that would be the last step and the association would succeed
+* if the Client is dealing with *Controller with a self-signed certificate*, the Client will do another connection attempt to probe the Controller's certificate
+* the probed Controller's certificate fingerprint is then presented to the user, and if the user agrees to trust it, the Client then considers that certificate to be trusted for a given context
+* Client finally connects to the Controller again with a trusted CA set containing only that certificate, executes the final API sanity checks, and if everything is OK then the association succeeds
+
+Afterward, each interaction with the Controller  (e.g. `orchard create vm` command) will stick to the chosen verification method and will re-verify the presented Controller's certificate against:
+
+* *Controller with a self-signed certificate*: a trusted certificate stored in the Orchard's configuration file
+* *Controller with a publicly valid certificate*: host's root CA set
+
+### Worker
+
+To make the Worker connect to the Controller, a Bootstrap Token needs to be obtained using the `orchard get bootstrap-token` command.
+
+While this approach provides a less ad-hoc experience than that you'd have with `orchard context create`, it allows one to mass-deploy workers non-interactively, using tools such as Ansible.
+
+This resulting Bootstrap Token will either include the Controller's certificate (when the current context is with a *Controller with a self-signed certificate*) or omit it (when the current context is with a *Controller with a publicly valid certificate*).
+
+The way Worker connects to the Controller using the `orchard worker run` command is as follows:
+
+* when the Bootstrap Token contains the Controller's certificate:
+    * the Orchard Worker will try to connect to the Controller with a trusted CA set containing only that certificate
+* when the Bootstrap Token has no Controller's certificate:
+    * the Orchard Worker will try the PKI approach (can be disabled with [`--no-pki`](#--no-pki-override) to effectively prevent the Worker from connecting) and fail if certificate verification using PKI is not possible
+
+### `--no-pki` override
+
+If you only intend to access the *Controller with a self-signed certificate* and want to additionally guard yourself against [CA compromises](https://en.wikipedia.org/wiki/Certificate_authority#CA_compromise) and other PKI-specific attacks, pass a `--no-pki` command-line argument to the following commands:
+
+* `orchard context create --no-pki`
+    * this will prevent the Client from using PKI and will let you interactively verify the Controller's certificate fingerprint before connecting, thus creating a non-PKI association
+* `orchard worker run --no-pki`
+    * this will prevent the Worker from trying to use PKI when connecting to the Controller using a Bootstrap Token that has no certificate included in it, thus failing fast and letting you know that you need to create a proper Bootstrap Token
+
+We've deliberately chosen not to use environment variables (e.g. `ORCHARD_NO_PKI`) because they fail silently (e.g. due to a typo), compared to command-line arguments, which will result in an error that is much easier to detect.
@@ -0,0 +1,242 @@
+## Introduction
+
+Compared to Worker, which can only be deployed on a macOS machine, Controller can be also deployed on Linux.
+
+In fact, we've made a [container image](https://github.com/cirruslabs/orchard/pkgs/container/orchard) to ease deploying the Controller in container-native environments such as Kubernetes.
+
+Another thing to keep in mind that Orchard API is secured by default: all requests must be authenticated with the credentials of a service account. When you first run Orchard Controller, a `bootstrap-admin` service account will be created automatically and credentials will be printed to the standard output.
+
+If you already have a token in mind that you want to use for the `bootstrap-admin` service account, or you've got locked out and want this service account with a well-known password back, you can set the `ORCHARD_BOOTSTRAP_ADMIN_TOKEN` when running the controller.
+
+For example to use a secure, random value:
+
+```bash
+ORCHARD_BOOTSTRAP_ADMIN_TOKEN=$(openssl rand -hex 32) orchard controller run
+```
+
+## Customization
+
+Note that all the [Deployment Methods](#deployment-methods) essentially boil down to starting an `orchard controller run` command and keeping it alive.
+
+This means that by introducing additional command-line arguments, you can customize the Orchard Controller's behavior. Below, we list some of the common scenarios.
+
+### Customizing listening port
+
+* `--listen` — address to listen on (default `:6120`)
+
+### Customizing TLS
+
+* `--controller-cert` — use the controller certificate from the specified path instead of the auto-generated one (requires --controller-key)
+* `--controller-key` — use the controller certificate key from the specified path instead of the auto-generated one (requires --controller-cert)
+* `--insecure-no-tls` — disable TLS, making all connections to the controller unencrypted
+    * useful when deploying Orchard Controller behind a load balancer/ingress controller
+
+### Built-in SSH server
+
+Orchard Controller can act as a simple SSH server that port-forwards connections to the VMs running in the Orchard Cluster.
+
+This way you can completely skip the Orchard API when connecting to a given VM and only use the SSH client:
+
+```shell
+ssh -J <service account name>@orchard-controller.example.com <VM name>
+```
+
+To enable this functionality, pass `--listen-ssh` command-line argument to the `orchard controller run` command, for example:
+
+```ssh
+orchard controller run --listen-ssh 6122
+```
+
+Here's other command-line arguments associated with this functionality:
+
+* `--ssh-host-key` — use the SSH private host key from the specified path instead of the auto-generated one
+* `--insecure-ssh-no-client-auth` — allow SSH clients to connect to the controller's SSH server without authentication, thus only authenticating on the target worker/VM's SSH server
+    * useful when you already have strong credentials on your VMs, and you want to share these VMs to others without additionally giving out Orchard Cluster credentials
+
+Check out our [Jumping through the hoops: SSH jump host functionality in Orchard](/blog/2024/06/20/jumping-through-the-hoops-ssh-jump-host-functionality-in-orchard/) blog post for more information.
+
+## Deployment Methods
+
+While you can always start `orchard controller run` manually with the required arguments, this method is not recommended due to lack of persistence.
+
+In the following sections you'll find several examples of how to run Orchard Controller in various environments in a more persistent way. Feel free to submit PRs with more examples.
+
+### Google Compute Engine
+
+An example below will deploy a single instance of Orchard Controller in Google Cloud Compute Engine in `us-central1` region.
+
+First, let's create a static IP address for our instance:
+
+```bash
+gcloud compute addresses create orchard-ip --region=us-central1
+export ORCHARD_IP=$(gcloud compute addresses describe orchard-ip --format='value(address)' --region=us-central1)
+```
+
+Then, ensure that there exist a firewall rule targeting `https-server` tag and allowing access to TCP port 443. If that's not the case, create one:
+
+```shell
+gcloud compute firewall-rules create default-allow-https --direction=INGRESS --priority=1000 --network=default --action=ALLOW --rules=tcp:443 --source-ranges=0.0.0.0/0 --target-tags=https-server
+```
+
+Once we have the IP address and the firewall rule set up, we can create a new instance with Orchard Controller running inside a container:
+
+```bash
+gcloud compute instances create-with-container orchard-controller \
+  --machine-type=e2-micro \
+  --zone=us-central1-a \
+  --image-family cos-stable \
+  --image-project cos-cloud \
+  --tags=https-server \
+  --address=$ORCHARD_IP \
+  --container-image=ghcr.io/cirruslabs/orchard:latest \
+  --container-env=PORT=443 \
+  --container-env=ORCHARD_BOOTSTRAP_ADMIN_TOKEN=$ORCHARD_BOOTSTRAP_ADMIN_TOKEN \
+  --container-mount-host-path=host-path=/home/orchard-data,mode=rw,mount-path=/data
+```
+
+Now you can create a new context for your local client:
+
+```bash
+orchard context create --name production \
+  --service-account-name bootstrap-admin \
+  --service-account-token $ORCHARD_BOOTSTRAP_ADMIN_TOKEN \
+  https://$ORCHARD_IP:443
+```
+
+And select it as the default context:
+
+```bash
+orchard context default production
+```
+
+### Kubernetes (GKE, EKS, etc.)
+
+The easiest way to run Orchard Controller on Kubernetes is to expose it through the `LoadBalancer` service.
+
+This way no fiddling with the TLS certificates and HTTP proxying is needed, and most cloud providers will allocate a ready-to-use IP-address that can directly used in `orchard context create` and `orchard worker run` commands, or additionally assigned to a DNS domain name for a more memorable hostname.
+
+Do deploy on Kubernetes, only three resources are needed:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: orchard-controller
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  # Uncomment this when deploying on Amazon's EKS and
+  # change to the desired storage class name if needed
+  # storageClassName: gp2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: orchard-controller
+spec:
+  serviceName: orchard-controller
+  replicas: 1
+  selector:
+    matchLabels:
+      app: orchard-controller
+  template:
+    metadata:
+      labels:
+        app: orchard-controller
+    spec:
+      containers:
+        - name: orchard-controller
+          image: ghcr.io/cirruslabs/orchard:latest
+          volumeMounts:
+            - mountPath: /data
+              name: orchard-controller
+      volumes:
+        - name: orchard-controller
+          persistentVolumeClaim:
+            claimName: orchard-controller
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: orchard-controller
+spec:
+  selector:
+    app: orchard-controller
+  ports:
+    - protocol: TCP
+      port: 6120
+      targetPort: 6120
+  type: LoadBalancer
+```
+
+Once deployed, the bootstrap credentials will be printed to the standard output. You can inspect them by running `kubectl logs deployment/orchard-controller`.
+
+The resources above ensure that Controller's database is stored in a persistent storage and survives restats.
+
+You can further allocate a static IP address and use it by adding annotations to the `Service` resource. Here's how to do that:
+
+* on Google's GKE: <https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer-parameters#spd-static-ip>
+* on Amazon's EKS: <https://kubernetes.io/docs/reference/labels-annotations-taints/#service-beta-kubernetes-io-aws-load-balancer-eip-allocations>
+
+### systemd service on Debian-based distributions
+
+This should work for most Debian-based distributions like Debian, Ubuntu, etc.
+
+Firstly, make sure that the APT transport for downloading packages via HTTPS and common X.509 certificates are installed:
+
+```shell
+sudo apt-get update && sudo apt-get -y install apt-transport-https ca-certificates
+```
+
+Then, add the Cirrus Labs repository:
+
+```shell
+echo "deb [trusted=yes] https://apt.fury.io/cirruslabs/ /" | sudo tee /etc/apt/sources.list.d/cirruslabs.list
+```
+
+Update the package index files and install the Orchard Controller:
+
+```shell
+sudo apt-get update && sudo apt-get -y install orchard-controller
+```
+
+Finally, enable and start the Orchard Controller systemd service:
+
+```shell
+sudo systemctl enable orchard-controller
+sudo systemctl start orchard-controller
+```
+
+The bootstrap credentials will be printed to the standard output. You can inspect them by running `sudo systemctl status orhcard-controller` or `journalctl -u orchard-controller`.
+
+### systemd service on RPM-based distributions
+
+This should work for most RPM-based distributions like Fedora, CentOS, etc.
+
+First, create a `/etc/yum.repos.d/cirruslabs.repo` file with the following contents:
+
+```ini
+[cirruslabs]
+name=Cirrus Labs Repo
+baseurl=https://yum.fury.io/cirruslabs/
+enabled=1
+gpgcheck=0
+```
+
+Then, install the Orchard Controller:
+
+```shell
+sudo yum -y install orchard-controller
+```
+
+Finally, enable and start the Orchard Controller systemd service:
+
+```shell
+systemctl enable orchard-controller
+systemctl start orchard-controller
+```
+
+The bootstrap credentials will be printed to the standard output. You can inspect them by running `sudo systemctl status orhcard-controller` or `journalctl -u orchard-controller`.
@@ -0,0 +1,127 @@
+## Obtain a Boostrap Token
+
+First, create a service account with a minimal set of roles (`compute:read` and `compute:write`) required for proper Worker functioning:
+
+```bash
+orchard create service-account worker-pool-m1 --roles "compute:read" --roles "compute:write"
+```
+
+Then, generate a Bootstrap Token for this service account:
+
+```shell
+orchard get bootstrap-token worker-pool-m1
+```
+
+We will reference the value of the Bootstrap Token generated here as `${BOOTSTRAP_TOKEN}` below.
+
+Further, we assume that Orchard controller is available on `orchard.example.com`
+
+## Deployment Methods
+
+While you can always run `orchard worker run` manually with the required arguments, this method of deploying the Worker is not recommended.
+
+Instead, we've listed a more persistent methods of a Worker deployment below.
+
+### launchd
+
+[launchd](https://launchd.info/) is an init system for macOS that manages daemons, agents and other background processes.
+
+In this deployment method, we'll create a new job definition file for the launchd to manage on its behalf.
+
+To begin, first install Orchard:
+
+```shell
+brew install cirruslabs/cli/orchard
+```
+
+Ensure that the following command:
+
+```shell
+which orchard
+```
+
+...yields `/opt/homebrew/bin/orchard`. If not, you'll need to replace all of the occurences of `/opt/homebrew/bin/orchard` in the job definition below.
+
+Then, create a launchd job definition in `/Library/LaunchDaemons/org.cirruslabs.orchard.worker.plist` with the following contents:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>Label</key>
+    <string>org.cirruslabs.orchard.worker</string>
+    <key>Program</key>
+    <string>/opt/homebrew/bin/orchard</string>
+    <key>ProgramArguments</key>
+    <array>
+      <string>/opt/homebrew/bin/orchard</string>
+      <string>worker</string>
+      <string>run</string>
+      <string>--user</string>
+      <string>admin</string>
+      <string>--bootstrap-token</string>
+      <string>${BOOTSTRAP_TOKEN}</string>
+      <string>orchard.example.com</string>
+    </array>
+    <key>EnvironmentVariables</key>
+    <dict>
+      <key>PATH</key>
+      <string>/bin:/usr/bin:/usr/local/bin:/opt/homebrew/bin</string>
+    </dict>
+    <key>WorkingDirectory</key>
+    <string>/var/empty</string>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>KeepAlive</key>
+    <true/>
+    <key>StandardOutPath</key>
+    <string>/Users/admin/orchard-launchd.log</string>
+    <key>StandardErrorPath</key>
+    <string>/Users/admin/orchard-launchd.log</string>
+  </dict>
+</plist>
+```
+
+This assumes that your macOS user on the host is named `admin`. If not, change all occurrences of `admin` in the job definition above to `$USER`.
+
+Finally, change the `orchard.example.com` to the FQDN or an IP-address of your Orchard Controller.
+
+Now, you can start the job:
+
+```shell
+launchctl load -w /Library/LaunchDaemons/org.cirruslabs.orchard.worker.plist
+```
+
+### Ansible
+
+If you have a set of machines that you want to use as Orchard Workers, you can use [Ansible](https://docs.ansible.com/) to configure them.
+
+We've created the [cirruslabs/ansible-orchard](https://github.com/cirruslabs/ansible-orchard) repository with a basic Ansible playbook for convenient setup.
+
+To use it, clone it locally:
+
+```shell
+git clone https://github.com/cirruslabs/ansible-orchard.git
+cd ansible-orchard/
+```
+
+Make sure that the Ansible Galaxy dependencies are installed:
+
+```shell
+ansible-galaxy install -r requirements.yml
+```
+
+Then, edit the `production-pool` file and populate the following fields:
+
+* `hosts` — replace `worker-1.hosts.internal` with your worker FQDN or IP-address and add more hosts if needed
+* `ansible_user` — set it macOS user on the host for the SSH to work
+* `orchard_worker_user` — set it macOS user on the host under which the Worker will run, e.g. `admin`
+* `orchard_worker_controller_url` — set it to FQDN or an IP-address of your Orchard Controller, for example, `orchard.example.com`
+* `orchard_worker_bootstrap_token` — set it to `${BOOTSTRAP_TOKEN}` we've generated above
+
+Deploy the playbook:
+
+```shell
+ansible-playbook --inventory-file production-pool --ask-pass playbook-workers.yml
+```
@@ -0,0 +1,187 @@
+Orchard has a REST API that follows [OpenAPI specification](https://swagger.io/specification/) and is described in [`api/openapi.yaml`](https://github.com/cirruslabs/orchard/blob/main/api/openapi.yaml).
+
+You can run `orchard dev` locally and navigate to `http://127.0.0.1:6120/v1/` for interactive documentation.
+
+![](/assets/images/orchard/orchard-api-documentation-browser.png)
+
+## Using the API
+
+Below you'll find examples of using Orchard API via vanilla Python's request library and Golang package that Orchard CLI build on top of.
+
+### Authentication
+
+When running in non-development mode, Orchard API expects a [basic access authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) to be provided for each API call.
+
+Below you'll find two snippets that retrieve controller's information and output its version:
+
+#### Authentication in Python
+
+```python
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+def main():
+  # Authentication
+  basic_auth = HTTPBasicAuth("service account name", "service account token")
+
+  response = requests.get("http://127.0.0.1:6120/v1/info", auth=basic_auth)
+
+  print(response.json()["version"])
+
+
+if __name__ == '__main__':
+  main()
+```
+
+#### Authentication in Golang
+
+```go
+package main
+
+import (
+  "context"
+  "fmt"
+  "github.com/cirruslabs/orchard/pkg/client"
+  "log"
+)
+
+func main() {
+  client, err := client.New()
+  if err != nil {
+    log.Fatalf("failed to initialize Orchard API client: %v", err)
+  }
+
+  controllerInfo, err := client.Controller().Info(context.Background())
+  if err != nil {
+    log.Fatalf("failed to retrieve controller's information: %v", err)
+  }
+
+  fmt.Println(controllerInfo.Version)
+}
+```
+
+Note that we don't provide any credentials for Golang's version of the snippet: this is because Orchard's Golang API client (`github.com/cirruslabs/orchard/pkg/client`) has the ability to read the current's user Orchard context automatically.
+
+### Creating a VM
+
+A more intricate example would be spinning off a VM with a startup script that outputs date, reading its logs and removing it from the controller:
+
+#### Creating a VM in Python
+
+```python
+import time
+import uuid
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+def main():
+  vm_name = str(uuid.uuid4())
+
+  basic_auth = HTTPBasicAuth("service account name", "service account token")
+
+  # Create VM
+  response = requests.post("http://127.0.0.1:6120/v1/vms", auth=basic_auth, json={
+    "name": vm_name,
+    "image": "ghcr.io/cirruslabs/macos-sequoia-base:latest",
+    "cpu": 4,
+    "memory": 4096,
+    "startup_script": {
+      "script_content": "date",
+    }
+  })
+  response.raise_for_status()
+
+  # Retrieve VM's logs
+  while True:
+    response = requests.get(f"http://127.0.0.1:6120/v1/vms/{vm_name}/events", auth=basic_auth)
+    response.raise_for_status()
+
+    result = response.json()
+
+    if isinstance(result, list) and len(result) != 0:
+      print(result[0]["payload"])
+      break
+
+    time.sleep(1)
+
+  # Delete VM
+  response = requests.delete(f"http://127.0.0.1:6120/v1/vms/{vm_name}", auth=basic_auth)
+  response.raise_for_status()
+
+
+if __name__ == '__main__':
+  main()
+```
+
+#### Creating a VM in Golang
+
+```go
+package main
+
+import (
+  "context"
+  "fmt"
+  "github.com/cirruslabs/orchard/pkg/client"
+  v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
+  "github.com/google/uuid"
+  "log"
+  "time"
+)
+
+func main() {
+  vmName := uuid.New().String()
+
+  client, err := client.New()
+  if err != nil {
+    log.Fatalf("failed to initialize Orchard API client: %v", err)
+  }
+
+  // Create VM
+  err = client.VMs().Create(context.Background(), &v1.VM{
+    Meta: v1.Meta{
+      Name: vmName,
+    },
+    Image:  "ghcr.io/cirruslabs/macos-sequoia-base:latest",
+    CPU:    4,
+    Memory: 4096,
+    StartupScript: &v1.VMScript{
+      ScriptContent: "date",
+    },
+  })
+  if err != nil {
+    log.Fatalf("failed to create VM: %v")
+  }
+
+  // Retrieve VM's logs
+  for {
+    vmLogs, err := client.VMs().Logs(context.Background(), vmName)
+    if err != nil {
+      log.Fatalf("failed to retrieve VM logs")
+    }
+
+    if len(vmLogs) != 0 {
+      fmt.Println(vmLogs[0])
+      break
+    }
+
+    time.Sleep(time.Second)
+  }
+
+  // Delete VM
+  if err := client.VMs().Delete(context.Background(), vmName); err != nil {
+    log.Fatalf("failed to delete VM: %v", err)
+  }
+}
+```
+
+## Resource management
+
+Some resources, such as `Worker` and `VM`, have a `resource` field which is a dictionary that maps between resource names and their amounts (amount requested or amount provided, depending on the resource) and is useful for scheduling.
+
+Well-known resources:
+
+* `org.cirruslabs.tart-vms` — number of Tart VM slots available on the machine or requested by the VM
+    * this number is `2` for workers and `1` for VMs by default
@@ -0,0 +1,30 @@
+## Backups
+
+In order to backup the Orchard Controller, simply copy its `ORCHARD_HOME` (which defaults to `~/.orchard/`) directory somewhere safe and restore it when needed.
+
+This directory contains a BadgerDB database that Controller uses to store state and an X.509 certificate with key.
+
+## Upgrades
+
+Since the Orchard's initial release, we've managed to maintain the backwards compatibility between versions up to this day, so generally, it doesn't matter whether you upgrade the Controller or Worker(s) first.
+
+In case a new functionality is introduced, you might be required to finish the upgrade of both the Controller and the Worker(s) to be able to use it fully.
+
+In case there will be backwards-incompatible changes introduced in the future, we will try to do our best and highlight this in the [release notes](https://github.com/cirruslabs/orchard/releases) accordingly.
+
+## Observability
+
+Both the Controller and Worker produce some useful OpenTelemetry metrics. Metrics are scoped with `org.cirruslabs.orchard` prefix and include information about resource utilization, statuses or Workers, scheduling/pull time and many more.
+
+By default, the telemetry is sent to `https://localhost:4317` using the gRPC protocol and to `http://localhost:4318` using the HTTP protocol.
+
+You can override this by setting the [standard OpenTelemetry environment variable](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) `OTEL_EXPORTER_OTLP_ENDPOINT`.
+
+Please refer to [OTEL Collector documentation](https://opentelemetry.io/docs/collector/) for instruction on how to setup a sidecar for the metrics collections or find out if your SaaS monitoring has an available OTEL endpoint (see [Honeycomb](https://docs.honeycomb.io/send-data/opentelemetry/) as an example).
+
+### Sending metrics to Google Cloud Platform
+
+There are two standard options of ingesting metrics procuded by Orchard Controller and Workers into the GCP:
+
+* [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) + [Google Cloud Exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/googlecloudexporter/README.md) — open-source solution that can be later re-purposed to send metrics to any OTLP-compatible endpoint by swapping a single [exporter](https://opentelemetry.io/docs/collector/configuration/#exporters)
+* [Ops Agent](https://cloud.google.com/monitoring/agent/ops-agent/otlp) — Google-backed solution with a syntax similar to OpenTelemetry Collector, but tied to GCP-only
@@ -0,0 +1,101 @@
+Tart is great for running workloads on a single machine, but what if you have more than one computer at your disposal
+and
+a couple of VMs is not enough anymore for your needs? This is where [Orchard](https://github.com/cirruslabs/orchard)
+comes in to play!
+
+It allows you to orchestrate multiple Tart-capable hosts from either an Orchard CLI (which we demonstrate below)
+or [through the API](/orchard/integration-guide).
+
+The easiest way to start is to run Orchard in local development mode:
+
+```shell
+brew install cirruslabs/cli/orchard
+orchard dev
+```
+
+This will run an Orchard Controller and an Orchard Worker in a single process on your local machine, allowing you to
+test both the CLI functionality and the API from a tool like cURL or programming language of choice, without the need to
+authenticate requests.
+
+Note that in production deployments, these two components are started separately and enable security by default. Please
+refer to [Deploying Controller](/orchard/deploying-controller) and [Deploying Workers](/orchard/deploying-workers) for
+more information.
+
+## Creating Virtual Machines
+
+Now, let's create a Virtual Machine:
+
+```shell
+orchard create vm --image ghcr.io/cirruslabs/macos-sequoia-base:latest sequoia-base
+```
+
+You can check a list of VM resources to see if the Virtual Machine we've created above is already running:
+
+```shell
+orchard list vms
+```
+
+## Accessing Virtual Machines
+
+Orchard has an ability to do port forwarding that `ssh` and `vnc` commands are built on top of. All port forwarding
+connections are done via the Orchard Controller instance which "proxies" a secure connection to the Orchard Workers.
+
+Therefore, your workers can be located under a stricter firewall that only allows connections to the Orchard Controller
+instance. Orchard Controller instance is secured by default and all API calls are authenticated and authorized.
+
+### SSH
+
+To SSH into a VM, use the `orchard ssh` command:
+
+```shell
+orchard ssh vm sequoia-base
+```
+
+You can specify the `--username` and `--password` flags to specify the username/password pair to use for the SSH
+protocol. By default, `admin`/`admin` is used.
+
+You can also execute remote commands instead of spawning a login shell, similarly to how OpenSSH's `ssh` command accepts
+a command argument:
+
+```shell
+orchard ssh vm sequoia-base "uname -a"
+```
+
+You can execute scripts remotely this way, by telling the remote command-line interpreter to read from the standard
+input and using the redirection operator as follows:
+
+```shell
+orchard ssh vm sequoia-base "bash -s" < script.sh
+```
+
+### VNC
+
+Similarly to `ssh` command, you can use `vnc` command to open Screen Sharing into a remote VM:
+
+```shell
+orchard vnc vm sequoia-base
+```
+
+You can specify the `--username` and `--password` flags to specify the username/password pair to use for the VNC
+protocol. By default, `admin`/`admin` is used.
+
+## Deleting Virtual Machines
+
+The following command will delete the VM we've created above and clean-up the resources associated with it:
+
+```shell
+orchard delete vm sequoia-base
+```
+
+## Environment variables
+
+In addition to controlling the Orchard via the CLI arguments, there are environment variables that may be beneficial
+both when automating Orchard and in daily use:
+
+| Variable name                   | Description                                                                                                                                                                                                                                                                                                                                                                  |
+|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `ORCHARD_HOME`                  | Override Orchard's home directory. Useful when running multiple Orchard instances on the same host and when testing.                                                                                                                                                                                                                                                         |
+| `ORCHARD_LICENSE_TIER`          | The default license limit only allows connecting 4 Orchard Workers to the Orchard Controller. If you've purchased a [Gold Tier License](/licensing/), set this variable to `gold` to increase the limit to 20 Orchard Workers. And if you've purchased a [Platinum Tier License](/licensing/), set this variable to `platinum` to increase the limit to 200 Orchard Workers. |
+| `ORCHARD_URL`                   | Override controller URL on per-command basis.                                                                                                                                                                                                                                                                                                                                |
+| `ORCHARD_SERVICE_ACCOUNT_NAME`  | Override service account name (used for controller API auth) on per-command basis.                                                                                                                                                                                                                                                                                           |
+| `ORCHARD_SERVICE_ACCOUNT_TOKEN` | Override service account token (used for controller API auth) on per-command basis.                                                                                                                                                                                                                                                                                          |
@@ -0,0 +1,77 @@
+## Installation
+
+The easiest way to install Orchard CLI is through the [Homebrew](https://brew.sh/):
+
+```shell
+brew install cirruslabs/cli/orchard
+```
+
+Binaries and packages for other architectures can be found in [GitHub Releases](https://github.com/cirruslabs/orchard/releases).
+
+## Setting up a context
+
+The first step after installing the Orchard CLI is to configure its context. Configuring context is like pairing with the specified Orchard Controller, so that the commands like `orchard create vm`, `orchard ssh vm` will work.
+
+To configure a context, `orchard context` has a subfamily of commands:
+
+* `orchard context create <CONTROLLER ADDRESS>` — creates a new context to communicate with Orchard Controller available on the specified address
+* `orchard context default <CONTROLLER ADDRESS>` — sets a context with a given Orchard Controller address as default (in case there's more than one context configured)
+* `orchard context list` — lists all the configured contexts, indicating the default one
+* `orchard context delete <CONTROLLER ADDRESS>` — deletes a context for the specified Orchard Controller address
+
+Most of the time, you'll only need the `orchard context create`. For example, if you've deployed your Orchard Controller to `orchard-controller.example.com`, a new context can be configured like so:
+
+```shell
+orchard context create orchard-controller.example.com
+```
+
+`orchard context create` assumes port 6120 by default, so if you use a different port for the Orchard Controller, simply specify the port explicitly:
+
+```shell
+orchard context create orchard-controller.example.com:8080
+```
+
+When creating a new context you will be prompted for the service account name and token, which can be obtained from:
+
+* `orchard controller run` logs
+    * if this is a first start
+* `orchard get service-account`
+    * from an already configured Orchard CLI
+
+## Using labels when creating VMs
+
+Labels are useful if you want to restrict scheduling of a VM to workers whose labels include a subset of the VM's specified labels.
+
+For example, you might have an Orchard Cluster consisting of the following workers:
+
+* Mac Minis (`orchard worker run --labels location=DC1-R12-S4,model=macmini`)
+* Mac Studios (`orchard worker run --labels location=DC1-R18-S8,model=macstudio`)
+
+To create and run a VM specifically on Mac Studio machines, pass the `--labels` command-line argument to `orchard create vm` when creating a VM:
+
+```shell
+orchard create vm --labels model=macstudio <NAME>
+```
+
+When processing this VM, the scheduler will only place it on available Mac Studio workers.
+
+## Using resources when creating VMs
+
+Resources are useful if you want to restrict scheduling of a VM to workers that still have enough of the specified resource to fit the VM's requirements.
+
+The difference between the labels is that the resources are finite and are automatically accounted by the scheduler.
+
+To illustrate this with an example, let's say you have an Orchard Cluster consisting of the following workers:
+
+* Mac Mini with 1 Gbps bandwidth (`orchard worker run --resources bandwidth-mbps=1000`)
+* Mac Studio with 10 Gbps bandwidth (`orchard worker run --resources bandwidth-mbps=10000`)
+
+VM created using the command below will only be scheduled on a Mac Studio with 10 Gbps bandwidth:
+
+```shell
+orchard create vm --resources bandwidth-mbps=7500 <NAME>
+```
+
+However, after this VM is scheduled, the 10 Gbps Mac Studio will only be able to accommodate one more VM (due to internal Apple EULA limit for macOS virtualization) with `bandwidth-mbps=2500` or less.
+
+After the VM finishes, the unused resources will be available again.
@@ -9,18 +9,18 @@ Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura)

 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
-tart run sonoma-base
+tart clone ghcr.io/cirruslabs/macos-sequoia-base:latest sequoia-base
+tart run sequoia-base
 ```

 ??? info "Manual installation from a release archive"
    It's also possible to manually install `tart` binary from the latest released archive:

    ```bash
-    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart-arm64.tar.gz
-    tar -xzvf tart-arm64.tar.gz
-    ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
-    ./tart.app/Contents/MacOS/tart run sonoma-base
+    curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart.tar.gz
+    tar -xzvf tart.tar.gz
+    ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-sequoia-base:latest sequoia-base
+    ./tart.app/Contents/MacOS/tart run sequoia-base
    ```

    Please note that `./tart.app/Contents/MacOS/tart` binary is required to be used in order to trick macOS
@@ -34,6 +34,10 @@ tart run sonoma-base

 The following macOS images are currently available:

+* macOS 15 (Sequoia)
+    * `ghcr.io/cirruslabs/macos-sequoia-vanilla:latest`
+    * `ghcr.io/cirruslabs/macos-sequoia-base:latest`
+    * `ghcr.io/cirruslabs/macos-sequoia-xcode:latest`
 * macOS 14 (Sonoma)
    * `ghcr.io/cirruslabs/macos-sonoma-vanilla:latest`
    * `ghcr.io/cirruslabs/macos-sonoma-base:latest`
@@ -82,7 +86,7 @@ These credentials work both for logging in via GUI, console (Linux) and SSH.
 If the guest VM is running and configured to accept incoming SSH connections you can conveniently connect to it like so:

 ```bash
-ssh admin@$(tart ip sonoma-base)
+ssh admin@$(tart ip sequoia-base)
 ```

 !!! tip "Running scripts inside Tart virtual machines"
@@ -91,8 +95,8 @@ ssh admin@$(tart ip sonoma-base)

    ```bash
    brew install cirruslabs/cli/sshpass
-    sshpass -p admin ssh -o "StrictHostKeyChecking no" admin@$(tart ip sonoma-base) "uname -a"
-    sshpass -p admin ssh -o "StrictHostKeyChecking no" admin@$(tart ip sonoma-base) < script.sh
+    sshpass -p admin ssh -o "StrictHostKeyChecking no" admin@$(tart ip sequoia-base) "uname -a"
+    sshpass -p admin ssh -o "StrictHostKeyChecking no" admin@$(tart ip sequoia-base) < script.sh
    ```

 ## Mounting directories
@@ -1,7 +1,4 @@
-source = [
-  ".build/x86_64-apple-macosx/release/tart",
-  ".build/arm64-apple-macosx/release/tart"
-]
+source = [ "dist/tart_darwin_all/tart.app/Contents/MacOS/tart" ]
 bundle_id = "com.github.cirruslabs.tart"

 apple_id {
@@ -91,13 +91,20 @@ nav:
  - "Home": index.md
  - "Quick Start": quick-start.md
  - "Integrations":
-    - "Self-hosted CI": integrations/cirrus-cli.md
-    - "GitHub Actions": https://cirrus-runners.app/
-    - "GitLab Runner": integrations/gitlab-runner.md
-    - "Buildkite": integrations/buildkite.md
-    - "Managing VMs": integrations/vm-management.md
+      - "Self-hosted CI": integrations/cirrus-cli.md
+      - "GitHub Actions": https://cirrus-runners.app/
+      - "GitLab Runner": integrations/gitlab-runner.md
+      - "Buildkite": integrations/buildkite.md
+      - "Managing VMs": integrations/vm-management.md
  - "Support & Licensing": licensing.md
-  - "Orchestration": https://github.com/cirruslabs/orchard
+  - "Orchestration":
+      - "Quick Start": orchard/quick-start.md
+      - "Architecture and Security": orchard/architecture-and-security.md
+      - "Deploying Controller": orchard/deploying-controller.md
+      - "Deploying Workers": orchard/deploying-workers.md
+      - "Using Orchard CLI": orchard/using-orchard-cli.md
+      - "Managing the Cluster": orchard/managing-cluster.md
+      - "Integrating with the API": orchard/integration-guide.md
  - "FAQ": faq.md
  - "Legal":
      - 'Terms of Service': legal/terms.md
@@ -1,7 +1,7 @@
 #!/bin/sh

 # helper script to build and run a signed tart binary
-# usage: ./scripts/run-signed.sh run sonoma-base
+# usage: ./scripts/run-signed.sh run sequoia-base

 set -e
Author	SHA1	Message	Date
Nikolay Edigaryev	40ab5c3af4	Fix unescaped commas in generated ArgumentParser completions (#1066 ) * Fix unescaped commas in generated ArgumentParser completions * Improve completion hints	2025-05-06 14:24:43 +04:00
fedor	280a31f707	Update docs, examples and CI to Sequoia	2025-05-04 20:49:23 -04:00
Nikolay EdigaryevandFedor Korotkov	8d49404337	Enable clipboard sharing on macOS too (#1046 ) * Enable clipboard sharing on macOS too And document which packages need to be installed on these operating systems. * We now use Tart Guest Agent Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com> --------- Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com>	2025-04-29 21:15:03 +04:00
Nikolay EdigaryevandCopilot	64a3999a58	Improve Orchard docs (#1064 ) * Iterate over Orchard Architecture description * Document Orchard Controller customization (e.g. --listen-ssh) * New section: "Using Orchard CLI" * Fix Markdown unordered list indentation * Fix "fenced code blocks should have a language specified" * the context → a context * Clarify different port * Simplify labels explanation * Studios → Studio * Better explain resources * crate → create * only to place → only place Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Surround "Using resources when creating VMs" header by blank lines --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>	2025-04-29 18:10:14 +04:00
Fedor Korotkov	5c1f5a61c1	Add `--no-trackpad` option to disable trackpad on macOS VMs (#1060 ) * Add --no-trackpad option to disable trackpad on macOS VMs * Cleanup after AI	2025-04-22 10:17:19 -04:00
Fedor Korotkov	1310220f05	Add `NSLocalNetworkUsageDescription` (#1058 )	2025-04-18 18:27:32 +04:00
dependabot[bot]	1fe2f1ff88	Bump golang.org/x/crypto from 0.21.0 to 0.35.0 in /benchmark (#1057 ) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.21.0 to 0.35.0. - [Commits](https://github.com/golang/crypto/compare/v0.21.0...v0.35.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.35.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2025-04-18 18:01:55 +04:00
victorserbu2709 Nikolay Edigaryev Victor Serbu Fedor Korotkov	df3de33f1a	Posibility to add Labels when pushing OCI Image (#1052 ) * Posibility to add Labels when pushing OCI Image Example running: tart push $image ${registry}/org/${image}-testing --labels com.org.revision=testing --labels com.org.repo.buildid=123456 * Fix Linting Run swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore * Update Sources/tart/Commands/Push.swift Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Update Sources/tart/Commands/Push.swift Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Update Sources/tart/Commands/Push.swift Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Update Sources/tart/OCI/Manifest.swift Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Update Sources/tart/Commands/Push.swift Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> * Update Sources/tart/Commands/Push.swift * Do not use a variable to store parseLabels() results * Trim spaces before splitting labels and support empty values --------- Co-authored-by: Victor Serbu <victors@4psa.com> Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com>	2025-04-14 16:10:12 +00:00
Fedor Korotkov	1560e4d312	Update Orchard Worker Instructions (#1055 ) To include workaround for https://github.com/cirruslabs/orchard/pull/302	2025-04-14 14:21:28 +00:00
Samuel	318202fa81	fix: correct typo in validation error message for nested virtualization support (#1050 )	2025-04-04 11:20:43 -04:00
Fedor Korotkov	cb92a3fa67	Use full host resources for Xcode benchmarks (#1045 )	2025-03-23 16:55:32 -04:00
Nikolay Edigaryev	9c30638079	docs(FAQ): document disk resizing procedure (#1042 )	2025-03-18 21:36:30 +04:00
Nikolay Edigaryev	a4edc6af50	Make tart set --random-serial no-nop for Linux VMs (#1027 )	2025-02-11 19:16:03 +04:00
Gavinkaa	2890dda847	fixing typo (#1026 )	2025-02-11 14:25:04 +00:00
Nikolay Edigaryev	2d55f3b9fa	docs(FAQ): document unsupported DHCP client identifiers (#1009 ) * docs(FAQ): document unsupported DHCP client identifiers * New section "Resolving the VMs IP with bridged networking" And a more clearer explanation of what "tart ip" does. * Remove extraneous space in ` --resolver=arp` * Better section name * Add a note about Linux talkativeness * Explain "talkativeness" a bit better	2025-01-20 19:26:08 +04:00
Fedor Korotkov	d3104c71b9	[docs] update manual installation script (#1008 )	2025-01-20 12:56:50 +04:00
Andrew Malchuk	3ddad55372	Fix #1004 : Wrong binary path in distro (#1005 )	2025-01-18 13:32:54 +04:00
Nikolay Edigaryev	72a81ca84a	.goreleaser.yml: caveats stanza with DHCP fix information (#1002 )	2025-01-17 17:06:02 +04:00
Nikolay Edigaryev	d8945503d6	Benchmark: run XcodeBenchmark with different disk settings (#1000 ) * Benchmark: run XcodeBenchmark with different disk settings * Add Xcode benchmark results	2025-01-16 23:58:26 +04:00
Nikolay Edigaryev	a0fd5435de	tart run: automatically enable --net-softnet when its related opts used (#994 )	2025-01-16 17:47:39 +04:00
Andrew Malchuk	4cf68fc061	Build universal binary instead of architecture dependent (#995 ) * Build universal binary instead of architecture dependent * Added universal_binaries stage to goreleaser * Fixed paths to compiled binary in .cirrus.yml * Revert changes in .cirrus.yml, use builtin venv module instead of virtualenv only	2025-01-16 08:22:05 -05:00
Nikolay Edigaryev	b626ed415b	Always use write(contentsOf:) instead of write(_:) (#997 )	2025-01-15 00:26:30 +04:00
Nikolay Edigaryev	dd7bace92d	tart run: clarify --net-softnet-expose limitations w.r.t. PF rdr rules (#996 )	2025-01-14 21:52:36 +04:00
Nikolay Edigaryev	94376ca355	tart run: introduce --net-softnet-expose (#990 ) * tart run: introduce --net-softnet-expose * --net-softnet-expose: add discussion * --net-softnet-expose: add a note about Softnet restrictions ...and how to disable them. * LAN → local network * Better clarify what --net-softnet does And how --net-softnet-allow can change that behavior.	2025-01-10 05:36:51 +04:00
Fedor Korotkov	60a481857f	FAQ to help with troubleshooting (#988 )	2025-01-03 14:05:50 -05:00
Nikolay Edigaryev	876271dceb	docs: a firewall rule needs to be created when deploying Orchard to GCE (#983 )	2024-12-24 16:39:26 +04:00
Frederic BOLTZandNikolay Edigaryev	6dd43abf03	Update FAQ.md (#979 ) * Update FAQ.md * Update docs/faq.md Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com> --------- Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com>	2024-12-21 18:44:36 -10:00
Nikolay Edigaryev	04c6df2efb	Registry: limit the text output on unexpected status code (#981 ) * Registry: limit the text output on unexpected status code * pullBlob(): limit channel read-out on error to 4 KiB * No need to always read channel until end This was introduced in https://github.com/cirruslabs/tart/pull/284 because we were blocking in "urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data)", which we don't do anymore. * Fetcher.fetch(): remove "progress" argument as we don't need it anymore	2024-12-20 11:55:08 +04:00
Nikolay Edigaryev	5a8b48a392	Assorted documentation improvements (#982 )	2024-12-19 19:10:24 -10:00
Nikolay Edigaryev	b96ea087f5	tart pull: re-try disk layer downloads by specifying "Range" header (#980 )	2024-12-19 21:21:33 +04:00
Nikolay Edigaryev	eaec015edf	Fetcher: re-use URLSession (#976 ) Otherwise we start to periodically get RST's from GitHub, possibly because of too many connection opens, which has an effect of cancelling previously received bytes. These RST's can be observed in tcpdump/Wireshark or Console, emitted from the libusrtcp.dylib library, com.apple.network subsystem, for the Tart process: >tcp_input [C59.1.1.1:3] flags=[R] seq=1805021659, ack=0, win=0 state=CLOSED rcv_nxt=1805021659, snd_una=1752355607 You can also observe the "Received Bytes" in "Activity Monitor" for the Tart process while pulling ghcr.io/cirruslabs/macos-runner:sequoia, and this value will periodically decrease.	2024-12-17 23:41:24 +04:00
Nikolay Edigaryev	e27da23f4c	Fetcher: avoid response deadlock (#975 )	2024-12-17 01:19:06 +04:00
Nikolay Edigaryev	e6a30b07e3	clone: actually reclaim unallocated bytes (#974 )	2024-12-17 00:25:07 +04:00
Nikolay Edigaryev	2d7615bdf8	tart clone: only reclaim unallocated bytes (#973 )	2024-12-13 02:41:26 +04:00
Nikolay Edigaryev	31ab4218f7	tart pull: 284% faster pulls with default concurrency setting (#970 ) * DiskV2: avoid allocating zero chunk on each zeroSkippingWrite() call * Increase hole granularity size from 64 KiB to 4 MiB * Fetcher: never write to disk, thanks to URLSessionDataDelegate	2024-12-11 21:48:59 +04:00
Nikolay Edigaryev	32ebc5bdbc	New benchmark results on AWS mac2.metal for Sonoma and Sequoia guests (#965 ) * New benchmark results on AWS mac2.metal for Sonoma and Sequoia guests * Document the volume type used for EBS	2024-12-04 16:03:33 -05:00
Fedor Korotkov	c825ba4cb1	Better message if hardware model is not supported by the host (#962 ) Related to https://github.com/cirruslabs/tart/discussions/961 It seems `VZMacHardwareModel #init?(dataRepresentation: Data)` is nullable sometimes. Let's return a better message in this case.	2024-12-03 06:52:40 -05:00
Nikolay Edigaryev	2db3918930	Benchmark improvements (#960 ) * Get a fresh instance of executor for each benchmark invocation And don't pre-initialize all of the executors at once, as this might reach the maximum number of VMs limit in case we want to test multiple Tart executors. * Run benchmarks on Tart with different --root-disk-opts options * Fix TestTart * benchmark fio: introduce --prepare command-line argument To be able to specify --prepare='sudo purge && sync', similarly to Hyperfine[1]. [1]: https://github.com/sharkdp/hyperfine * Benchmark Tart with --root-disk-opts=caching=cached separately too * Add Ars Technica recommended benchmarks * Tart executor: log SSH session standard output and standard error * Reduce file I/O size from 16 to 10 GB to avoid "No space left on device" * Remove random writing tests to make space for more read/read-write tests * Add some "randrw"-style fio benchmarks * Show latency in benchmark results * Add sync benchmark and show read/write/sync latency * README.md: add new benchmark results	2024-12-03 00:26:26 +04:00
Nikolay Edigaryev	4256330f39	FAQ: document /var/db/dhcpd_leases and its removal (#957 )	2024-11-21 22:02:21 +04:00
Nikolay Edigaryev	0794edf15a	RegistryRunner: explicitly listen on localhost (#956 )	2024-11-21 09:24:37 -05:00
Nikolay Edigaryev	8536c16bcc	tart set: support --{,no-}display-auto-reconfigure (#954 ) * tart set: support --{,no-}display-auto-reconfigure * Remove extraneous spaces * displayAutoReconfigure → displayRefit	2024-11-20 23:55:11 +04:00
Nikolay Edigaryev	589d489782	tart run: support specifying disk caching mode (#953 )	2024-11-19 23:48:09 +04:00
Nikolay Edigaryev	b1e88e1e51	tart run: do not remove "Edit" menu as its not present anymore (#946 )	2024-11-18 09:55:55 +01:00
Nikolay Edigaryev	b4de3bee83	tart pull: retry if we get URLError (#947 )	2024-11-15 23:14:47 +01:00
Fedor Korotkov	cd0f238a67	Allow to specify custom image in benchmarks (#941 )	2024-11-08 16:41:05 +00:00
Fedor Korotkov	02f94720c5	Set application category (#940 ) Was looking into performance and was wondering about Game Mode on Sonoma. This change is unrelated. Just found they have a category for tools like Tart.	2024-11-07 21:12:30 +00:00
Nikolay Edigaryev	c0443060cf	tart run: set "prohibited" activation policy when --no-graphics is set (#939 )	2024-11-07 15:20:09 -05:00
Fedor Korotkov	9c879b3f55	`tart run --nested` to enable nested virtualization when available (#938 ) Only works for Linux VMs under Sequoia hosts. Fixes #933 Fixes #701	2024-11-06 21:27:29 +04:00
Nikolay Edigaryev	f7b38769a9	tart pull: open the VM directory after pulling under a lock (#936 )	2024-11-05 00:01:14 +01:00
Nikolay Edigaryev	7c1ed4640f	Info.plist: do not use LSBackgroundOnly (#935 )	2024-11-04 19:08:37 +00:00
Nikolay Edigaryev	3fb8069edd	Linux VMs: do not use NVMe storage device (#932 )	2024-10-31 16:35:12 -04:00
Nikolay Edigaryev	c78c89e274	utimes(2): use errno to explain the error (#931 )	2024-10-31 16:33:03 -04:00
Fedor Korotkov	770220f905	Fixed plist file in version update (#927 )	2024-10-29 13:01:47 +04:00
Nikolay Edigaryev	768d1f9bad	PROFILING.md: document how to profile Tart using time(1) and xctrace(1) (#926 )	2024-10-28 18:49:10 +04:00
Fedor Korotkov	d49ed46439	Update access time on `pull` (#925 ) To make sure we won't prune then immediately after. Useful for when scenarios similar to Cirrus CLI when we make sure that several images are up-to-date before every request for task execution.	2024-10-25 23:33:30 +04:00
Nikolay Edigaryev	b52a857698	tart {clone,pull}: make deduplication opt-in (#924 )	2024-10-25 17:56:38 +04:00
Nikolay Edigaryev	3bf0bb22f3	CI: populate CFBundleShortVersionString in Info.plist (#923 )	2024-10-24 18:47:50 +00:00
Nikolay Edigaryev	accbd0cb33	Registry: prevent double authorization when getting a new token (#922 )	2024-10-23 23:51:55 +04:00
Nikolay Edigaryev	c0b20932c7	Prevent pipe deadlock when spawning a Process() (#916 )	2024-10-02 09:48:23 -04:00
Nikolay Edigaryev	3694af946c	Document automatic pruning in FAQ (#913 ) * Document automatic pruning in FAQ * tart {pull,clone}: consistent automatic pruning documentation in --help	2024-09-30 21:56:55 +04:00
Nikolay Edigaryev	dbf711a6c9	tart delete: return human-friendly error when local VM doesn't exist (#910 )	2024-09-26 14:45:14 +04:00
Nikolay Edigaryev	b9f24a40c1	Info.plist: set CFBundleName and CFBundleDisplayName to Tart (#909 )	2024-09-24 17:24:50 +04:00
Nikolay Edigaryev	10c6ace671	Re-generate ANTLR files using ANTLR 4.13.2 (#907 ) * Re-generate ANTLR files using ANTLR 4.13.2 * Package.swift: require exactly ANTLR of version 4.13.2	2024-09-20 17:50:14 +04:00
Nikolay Edigaryev	b98e23956b	Package.swift: bump Sentry SDK to 8.36.0 + upgrade other packages (#905 ) * Package.swift: bump Sentry SDK to 8.36.0 * $ swift package update	2024-09-19 19:06:37 +00:00
Fedor Korotkov	ce23f9c2a7	Completely disable audio devices in case of `--no-audio` (#904 ) This way VM won't have empty audio device at all. This should fix with an issue like that https://github.com/actions/runner-images/issues/9330	2024-09-17 09:40:37 +00:00
Nikolay Edigaryev	3da91e6518	tart run: provide a hint with names of other running VMs (#900 ) When VM limit gets exceeded.	2024-09-09 20:45:59 +04:00
Nikolay Edigaryev	7046886713	docs(orchard): document Kubernetes and systemd service deployment (#899 )	2024-09-09 16:40:17 +04:00
Nikolay EdigaryevandFedor Korotkov	3fde7d08dd	Orchard documentation (#897 ) * Orchard documentation * Fix typo Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com> * architecture-and-security.md: change list order --------- Co-authored-by: Fedor Korotkov <fedor.korotkov@gmail.com>	2024-08-28 00:09:57 +04:00
Fedor Korotkov	227301436c	Revert "Drop Monterey Support (#843 )" (#893 ) This reverts commit `017592075f`.	2024-08-14 14:57:55 -04:00
Nikolay Edigaryev	106eb5a2c8	tart push: re-try when encountering errors when pushing disk layers (#888 ) * tart push: re-try when encountering errors when pushing disk layers * Only re-try on URLError	2024-08-10 13:06:39 -04:00
Nikolay Edigaryev	10bf706653	tart push: avoid uploading blobs if they are already present (#887 ) By issuing HEAD requests to the registry before doing the actual upload.	2024-08-09 17:26:20 +04:00