Fetcher: avoid response deadlock (#975 )

clone: actually reclaim unallocated bytes (#974 )
tart clone: only reclaim unallocated bytes (#973 )
2026-10-01 19:51:10 +02:00 · 2024-12-17 01:19:06 +04:00 · 2024-12-17 00:25:07 +04:00 · 2024-12-13 02:41:26 +04:00 · 2024-12-11 21:48:59 +04:00 · 2024-12-04 16:03:33 -05:00
113 changed files with 3587 additions and 795 deletions
@@ -3,3 +3,5 @@
 TMPFILE=$(mktemp)
 envsubst < Sources/tart/CI/CI.swift > $TMPFILE
 mv $TMPFILE Sources/tart/CI/CI.swift
+
+/usr/libexec/PlistBuddy -c "Add :CFBundleShortVersionString string ${CIRRUS_TAG}" Resources/Info.plist
@@ -1,8 +1,5 @@
 use_compute_credits: true

-env: 
-  XCODE_TAG: 15.2
-
 task:
  name: Test on Sonoma
  alias: test
@@ -11,11 +8,11 @@ task:
      name: dev-mini
    resources:
      tart-vms: 1
+  build_script:
+    - swift build
  test_script:
    - swift test
  integration_test_script:
-    # Build Tart
-    - swift build
    - codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
    - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
    # Run integration tests
@@ -41,7 +38,7 @@ task:
  name: Lint
  alias: lint
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  lint_script:
    - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
  always:
@@ -58,7 +55,7 @@ task:
  name: Build ($BUILD_ARCH)
  alias: build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  build_script: swift build --arch $BUILD_ARCH --product tart
  sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/$BUILD_ARCH-apple-macosx/debug/tart
  binary_artifacts:
@@ -71,7 +68,7 @@ task:
    - lint
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -93,7 +90,7 @@ task:
    - security find-identity -v
    - xcodebuild -version
    - swift -version
-  goreleaser_script: goreleaser release --skip-publish --snapshot --clean
+  goreleaser_script: goreleaser release --skip=publish --snapshot --clean
  always:
    dist_artifacts:
      path: "dist/*"
@@ -106,7 +103,7 @@ task:
    - test
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-runner:sonoma
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -114,7 +111,7 @@ task:
    GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!]
    SENTRY_ORG: cirrus-labs
    SENTRY_PROJECT: persistent-workers
-    SENTRY_AUTH_TOKEN: ENCRYPTED[!c16a5cf7da5f856b4bc2f21fe8cb7aa2a6c981f851c094ed4d3025fd02ea59a58a86cee8b193a69a1fc20fa217e56ac3!]
+    SENTRY_AUTH_TOKEN: ENCRYPTED[!9eaf2875d51b113e2f68598441ff8e6b2e53242e48fcb93633bd75a373fbe2e7caa900d837cc92f0b142b65579731644!]
  setup_script:
    - cd $HOME
    - echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
@@ -16,3 +16,6 @@ dist/

 # mkdocs
 .cache
+
+# mkdocs-material
+site
@@ -52,5 +52,6 @@ brews:
    install: |
      libexec.install Dir["*"]
      bin.write_exec_script "#{libexec}/tart.app/Contents/MacOS/tart"
+      generate_completions_from_executable(libexec/"tart.app/Contents/MacOS/tart", "--generate-completion-script")
    custom_block: |
      depends_on :macos => :ventura
@@ -0,0 +1,64 @@
+# Profiling Tart
+
+## Using `time(1)`
+
+Perhaps, the easiest, but not the most comprehensive way to tell what's going on with Tart is to use the [`time(1)`](https://ss64.com/mac/time.html) command.
+
+In the example below, you will run `tart pull` via `time(1)` to gather generalized CPU, I/O and memory usage metrics:
+
+```shell
+/usr/bin/time -l tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+**Note:** you need to specify a full path to `time(1)` binary, otherwise the shell's built-in `time` command will be invoked, which doesn't have the `-l` command-line argument.
+
+**Note:** The `-l` command-line argument makes `time(1)` return much more useful information, for example, maximum memory usage.
+
+When running the command above, you'll see the `tart pull` output first as it pulls the image, and then the `time(1)` output, which will be printed once the Tart process finishes:
+
+```
+      172.17 real        10.29 user         8.36 sys
+           353796096  maximum resident set size
+                   0  average shared memory size
+                   0  average unshared data size
+                   0  average unshared stack size
+               23838  page reclaims
+                  35  page faults
+                   0  swaps
+                   0  block input operations
+                   0  block output operations
+                   8  messages sent
+                   8  messages received
+                   0  signals received
+                 146  voluntary context switches
+              222950  involuntary context switches
+         39683070975  instructions retired
+         27562035252  cycles elapsed
+           170920448  peak memory footprint
+```
+
+From the output above, you can tell that `tart pull` spent nearly 90% of time off-CPU (`real` > `user` + `sys`), which means that Tart was mostly waiting for the I/O (be it a network or disk), instead of decompressing disk layers or doing other useful computations.
+
+## Using `xctrace(1)`
+
+[`xctrace(1)`](https://keith.github.io/xcode-man-pages/xctrace.1.html) is a `.trace` format recorder for the [Instruments](https://en.wikipedia.org/wiki/Instruments_(software)) app, which yields much more powerful insights compared to `time(1)`. For example, it can tell which Tart functions spent the most time on the CPU, thus allowing the Tart developers to further optimize these functions.
+
+To use it, make sure that [Xcode](https://developer.apple.com/xcode/resources/) is installed. If you're installing Xcode for the first time on the machine, you'll need to launch it once and click the blue "Install" button. There's no need to choose any platforms except for the macOS.
+
+Once done, you can create a CPU profile of `tart pull`:
+
+```shell
+xctrace record --template "CPU Profiler" --target-stdout - --launch -- /opt/homebrew/bin/tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+Now that `xctrace(1)` is running, you'll see the `tart pull`-related output first, and once finished, the following line will appear:
+
+```
+Output file saved as: Launch_[...].trace
+```
+
+To view this trace in the Instruments app, simply find this directory in Finder and double-click it. Instruments app will appear:
+
+![](Resources/Instruments.png)
+
+To send this trace, right-click its directory in Finder and choose "Compress [...]". This will result in a similarly named file with a `.zip` at the end, which can now be conveniently sent via email or uploaded.
@@ -1,12 +1,13 @@
 {
+  "originHash" : "22b3726bc4e4c6e9c04ac97cb08a82967feb39960a93d2909768a16e11576748",
  "pins" : [
    {
      "identity" : "antlr4",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/antlr/antlr4",
      "state" : {
-        "branch" : "dev",
-        "revision" : "2703a8516c0fb7fe92db6b9c40e0113f577646d2"
+        "revision" : "cc82115a4e7f53d71d9d905caa2c2dfa4da58899",
+        "version" : "4.13.2"
      }
    },
    {
@@ -18,13 +19,22 @@
        "revision" : "772883073d044bc754d401cabb6574624eb3778f"
      }
    },
+    {
+      "identity" : "semaphore",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/groue/Semaphore",
+      "state" : {
+        "revision" : "2543679282aa6f6c8ecf2138acd613ed20790bc2",
+        "version" : "0.1.0"
+      }
+    },
    {
      "identity" : "sentry-cocoa",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/getsentry/sentry-cocoa",
      "state" : {
-        "revision" : "d277532e1c8af813981ba01f591b15bbdd735615",
-        "version" : "8.8.0"
+        "revision" : "5575af93efb776414f243e93d6af9f6258dc539a",
+        "version" : "8.36.0"
      }
    },
    {
@@ -32,8 +42,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-algorithms",
      "state" : {
-        "revision" : "b14b7f4c528c942f121c8b860b9410b2bf57825e",
-        "version" : "1.0.0"
+        "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42",
+        "version" : "1.2.0"
      }
    },
    {
@@ -41,17 +51,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-argument-parser",
      "state" : {
-        "revision" : "f3c9084a71ef4376f2fabbdf1d3d90a49f1fabdb",
-        "version" : "1.1.2"
-      }
-    },
-    {
-      "identity" : "swift-async-algorithms",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-async-algorithms",
-      "state" : {
-        "branch" : "main",
-        "revision" : "f05e450f0b909c0e80670a47516c4b9700b9e5da"
+        "revision" : "41982a3656a71c768319979febd796c6fd111d5c",
+        "version" : "1.5.0"
      }
    },
    {
@@ -59,17 +60,17 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/apple/swift-atomics.git",
      "state" : {
-        "revision" : "919eb1d83e02121cdb434c7bfc1f0c66ef17febe",
-        "version" : "1.0.2"
+        "revision" : "cd142fd2f64be2100422d658e7411e39489da985",
+        "version" : "1.2.0"
      }
    },
    {
-      "identity" : "swift-collections",
+      "identity" : "swift-log",
      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-collections.git",
+      "location" : "https://github.com/apple/swift-log.git",
      "state" : {
-        "revision" : "f504716c27d2e5d4144fa4794b12129301d17729",
-        "version" : "1.0.3"
+        "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537",
+        "version" : "1.6.1"
      }
    },
    {
@@ -81,13 +82,31 @@
        "version" : "1.0.2"
      }
    },
+    {
+      "identity" : "swift-retry",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/fumoboy007/swift-retry",
+      "state" : {
+        "revision" : "df9d7b185d2e433147ec0083a73c257e665eea0d",
+        "version" : "0.2.4"
+      }
+    },
    {
      "identity" : "swift-sysctl",
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/sersoft-gmbh/swift-sysctl.git",
      "state" : {
-        "revision" : "71fd64ee84819bb19fbecfb36d5a4503726b6fb7",
-        "version" : "1.6.0"
+        "revision" : "a91be36de6803ebe48f678699dfd0694c2200d2f",
+        "version" : "1.8.0"
+      }
+    },
+    {
+      "identity" : "swift-xattr",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/jozefizso/swift-xattr",
+      "state" : {
+        "revision" : "f8605af7b3290dbb235fb182ec6e9035d0c8c3ac",
+        "version" : "3.0.0"
      }
    },
    {
@@ -95,8 +114,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/malcommac/SwiftDate",
      "state" : {
-        "revision" : "6190d0cefff3013e77ed567e6b074f324e5c5bf5",
-        "version" : "6.3.1"
+        "revision" : "5d943224c3bb173e6ecf27295611615eba90c80e",
+        "version" : "7.0.0"
      }
    },
    {
@@ -104,8 +123,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/nicklockwood/SwiftFormat",
      "state" : {
-        "revision" : "da637c398c5d08896521b737f2868ddc2e7996ae",
-        "version" : "0.50.6"
+        "revision" : "ab6844edb79a7b88dc6320e6cee0a0db7674dac3",
+        "version" : "0.54.5"
      }
    },
    {
@@ -127,5 +146,5 @@
      }
    }
  ],
-  "version" : 2
+  "version" : 3
 }
@@ -1,4 +1,4 @@
-// swift-tools-version:5.7
+// swift-tools-version:5.10

 import PackageDescription
 let package = Package(
@@ -10,23 +10,24 @@ let package = Package(
    .executable(name: "tart", targets: ["tart"])
  ],
  dependencies: [
-    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.1.2"),
+    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.1"),
    .package(url: "https://github.com/mhdhejazi/Dynamic", branch: "master"),
-    .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"),
-    .package(url: "https://github.com/apple/swift-async-algorithms", branch: "main"),
-    .package(url: "https://github.com/malcommac/SwiftDate", from: "6.3.1"),
-    .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
-    .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.0.0")),
-    .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.50.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.8.0"),
+    .package(url: "https://github.com/apple/swift-algorithms", from: "1.2.0"),
+    .package(url: "https://github.com/malcommac/SwiftDate", from: "7.0.0"),
+    .package(url: "https://github.com/antlr/antlr4", exact: "4.13.2"),
+    .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.2.0")),
+    .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.53.6"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.36.0"),
    .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
-    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.0.0"),
-    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.0")
+    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.8.0"),
+    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.1"),
+    .package(url: "https://github.com/groue/Semaphore", from: "0.0.8"),
+    .package(url: "https://github.com/fumoboy007/swift-retry", from: "0.2.3"),
+    .package(url: "https://github.com/jozefizso/swift-xattr", from: "3.0.0"),
  ],
  targets: [
    .executableTarget(name: "tart", dependencies: [
      .product(name: "Algorithms", package: "swift-algorithms"),
-      .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
      .product(name: "ArgumentParser", package: "swift-argument-parser"),
      .product(name: "Dynamic", package: "Dynamic"),
      .product(name: "SwiftDate", package: "SwiftDate"),
@@ -36,6 +37,9 @@ let package = Package(
      .product(name: "TextTable", package: "TextTable"),
      .product(name: "Sysctl", package: "swift-sysctl"),
      .product(name: "SwiftRadix", package: "SwiftRadix"),
+      .product(name: "Semaphore", package: "Semaphore"),
+      .product(name: "DMRetry", package: "swift-retry"),
+      .product(name: "XAttr", package: "swift-xattr"),
    ], exclude: [
      "OCI/Reference/Makefile",
      "OCI/Reference/Reference.g4",
@@ -12,28 +12,28 @@ Tart powers [Cirrus Runners](https://cirrus-runners.app/)
 service — a drop-in replacement for the standard GitHub-hosted runners, offering 2-3 times better performance for a fraction of the price.

 <p align="center">
-  <a href="https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral" target=_blank>
+  <a href="https://cirrus-runners.app/?utm_source=github&utm_medium=referral" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/CirrusRunnersForGHA.png" height="65"/>
  </a>
 </p>

-Many companies are using Tart in their internal setups. Here are a few of them:
+Many companies are using Tart in their internal setups. Here are just a few of them:

 <p align="center">
-  <a href="https://ahrefs.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/ahrefs.png" height="65"/>
+  <a href="https://atlassian.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Atlassian.png" height="65"/>
  </a>
-  <a href="https://krisp.ai/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
+  <a href="https://www.figma.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Figma.png" height="65"/>
  </a>
  <a href="https://mullvad.net/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Mullvad.png" height="65"/>
  </a>
-  <a href="https://shape.dk/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/shape.png" height="65"/>
+  <a href="https://krisp.ai/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
  </a>
-  <a href="https://suran.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Suran.png" height="65"/>
+  <a href="https://testingbot.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/TestingBot.png" height="65"/>
  </a>
  <a href="https://symflower.com/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Symflower.png" height="65"/>
@@ -41,15 +41,18 @@ Many companies are using Tart in their internal setups. Here are a few of them:
  <a href="https://transloadit.com/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png" height="65"/>
  </a>
-  <a href="https://uphold.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png" height="65"/>
+  <a href="https://cirrus-ci.org/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/CirrusCI.png" height="65"/>
  </a>
  <a href="https://www.pitsdatarecovery.net/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
  </a>
+  <a href="https://expo.dev/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Expo.png" height="65"/>
+  </a>
 </p>

-**Note:** If your company or project is using Tart please consider [adding yourself to the list above](/Resources/Users/HowToAddYourself.md).
+**Note:** If your company or project is using Tart please consider [sharing with the community](https://github.com/cirruslabs/tart/discussions/857).

 <p align="center">
  <a href="https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws?utm_source=github&utm_medium=referral" target=_blank>
@@ -2,17 +2,24 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-    <key>CFBundleName</key>
-    <string>tart</string>
-    <key>CFBundleIdentifier</key>
-    <string>org.cirruslabs.tart</string>
-    <key>CFBundleExecutable</key>
-    <string>tart</string>
-    <key>LSBackgroundOnly</key>
-    <string>1</string>
-    <key>CFBundleIconFiles</key>
-    <array>
-        <string>AppIcon.png</string>
-    </array>
+	<key>CFBundleName</key>
+	<string>Tart</string>
+	<key>CFBundleDisplayName</key>
+	<string>Tart</string>
+	<key>CFBundleIdentifier</key>
+	<string>org.cirruslabs.tart</string>
+	<key>CFBundleExecutable</key>
+	<string>tart</string>
+  <key>LSApplicationCategoryType</key>
+  <string>public.app-category.developer-tools</string>
+	<key>CFBundleIconFiles</key>
+	<array>
+		<string>AppIcon.png</string>
+	</array>
+	<key>NSAppTransportSecurity</key>
+	<dict>
+		<key>NSAllowsArbitraryLoads</key>
+		<true/>
+	</dict>
 </dict>
 </plist>
@@ -1,4 +0,0 @@
-If you'd like to highlight your use of Tart, please create a `456px` by `130px` logo and create a PR
-that adds it to `README.md` in alphabetical order. Don't forget to include a small description of your usage pattern.
-
-You can refer to `Background.png` as a base for your logo.
@@ -8,17 +8,15 @@ struct Clone: AsyncParsableCommand {
    discussion: """
    Creates a local virtual machine by cloning either a remote or another local virtual machine.

-    Due to copy-on-write magic in Apple File System a cloned VM won't actually claim all the space right away.
-    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
-    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
-    will be modified.
+    Due to copy-on-write magic in Apple File System, a cloned VM won't actually claim all the space right away.
+    Only changes to a cloned disk will be written and claim new space. This also speeds up clones enormously.

-    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable. This might be helpful
-    for use cases when the original image is very big and a workload is known to only modify a fraction of the cloned disk.
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the cloned image
+    to fit. This behaviour is called "automatic pruning" and can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
    """
  )

-  @Argument(help: "source VM name")
+  @Argument(help: "source VM name", completion: .custom(completeMachines))
  var sourceName: String

  @Argument(help: "new VM name")
@@ -30,6 +28,9 @@ struct Clone: AsyncParsableCommand {
  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
  var concurrency: UInt = 4

+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
  func validate() throws {
    if newName.contains("/") {
      throw ValidationError("<new-name> should be a local name")
@@ -47,7 +48,7 @@ struct Clone: AsyncParsableCommand {
    if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
      // Pull the VM in case it's OCI-based and doesn't exist locally yet
      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
    }

    let sourceVM = try VMStorageHelper.open(sourceName)
@@ -63,17 +64,21 @@ struct Clone: AsyncParsableCommand {
      try lock.lock()

      let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
-        && sourceVM.state() != "suspended"
+        && sourceVM.state() != .Suspended
      try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)

      try localStorage.move(newName, from: tmpVMDir)

      try lock.unlock()

-      // APFS is doing copy-on-write so the above cloning operation (just copying files on disk)
+      // APFS is doing copy-on-write, so the above cloning operation (just copying files on disk)
      // is not actually claiming new space until the VM is started and it writes something to disk.
-      // So once we clone the VM let's try to claim a little bit of space for the VM to run.
-      try Prune.reclaimIfNeeded(UInt64(sourceVM.allocatedSizeBytes()), sourceVM)
+      //
+      // So, once we clone the VM let's try to claim the rest of space for the VM to run without errors.
+      let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes()
+      if unallocatedBytes > 0 {
+        try Prune.reclaimIfNeeded(UInt64(unallocatedBytes), sourceVM)
+      }
    }, onCancel: {
      try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
    })
@@ -16,7 +16,7 @@ struct Create: AsyncParsableCommand {
  @Flag(help: "create a Linux VM")
  var linux: Bool = false

-  @Option(help: ArgumentHelp("Disk size in Gb")) 
+  @Option(help: ArgumentHelp("Disk size in GB"))
  var diskSize: UInt16 = 50

  func validate() throws {
@@ -5,7 +5,7 @@ import SwiftUI
 struct Delete: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Delete a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeMachines))
  var name: [String]

  func run() async throws {
@@ -4,7 +4,7 @@ import Foundation
 struct Export: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Export VM to a compressed .tvm file")

-  @Argument(help: "Source VM name.")
+  @Argument(help: "Source VM name.", completion: .custom(completeMachines))
  var name: String

  @Argument(help: "Path to the destination file.")
@@ -5,7 +5,7 @@ import SystemConfiguration
 struct FQN: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Get a fully-qualified VM name", shouldDisplay: false)

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeMachines))
  var name: String

  func run() async throws {
@@ -2,6 +2,7 @@ import ArgumentParser
 import Foundation

 fileprivate struct VMInfo: Encodable {
+  let OS: OS
  let CPU: Int
  let Memory: UInt64
  let Disk: Int
@@ -14,7 +15,7 @@ fileprivate struct VMInfo: Encodable {
 struct Get: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "get", abstract: "Get a VM's configuration")

-  @Argument(help: "VM name.")
+  @Argument(help: "VM name.", completion: .custom(completeLocalMachines))
  var name: String

  @Option(help: "Output format: text or json")
@@ -25,8 +26,7 @@ struct Get: AsyncParsableCommand {
    let vmConfig = try VMConfig(fromURL: vmDir.configURL)
    let memorySizeInMb = vmConfig.memorySize / 1024 / 1024

-    let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000),
-                      Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state())
+    let info = VMInfo(OS: vmConfig.os, CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000), Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state().rawValue)
    print(format.renderSingle(info))
  }
 }
@@ -13,7 +13,7 @@ enum IPResolutionStrategy: String, ExpressibleByArgument, CaseIterable {
 struct IP: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Get VM's IP address")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Option(help: "Number of seconds to wait for a potential VM booting")
@@ -61,7 +61,7 @@ struct IP: AsyncParsableCommand {
          return ip
        }
      case .dhcp:
-        if let leases = try Leases(), let ip = try leases.ResolveMACAddress(macAddress: vmMACAddress) {
+        if let leases = try Leases(), let ip = leases.ResolveMACAddress(macAddress: vmMACAddress) {
          return ip
        }
      }
@@ -7,6 +7,7 @@ fileprivate struct VMInfo: Encodable {
  let Name: String
  let Disk: Int
  let Size: Int
+  let SizeOnDisk: Int
  let Running: Bool
  let State: String
 }
@@ -38,13 +39,13 @@ struct List: AsyncParsableCommand {

    if source == nil || source == "local" {
      infos += sortedInfos(try VMStorageLocal().list().map { (name, vmDir) in
-        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state())
+        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), SizeOnDisk: vmDir.allocatedSizeGB() - vmDir.deduplicatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
      })
    }

    if source == nil || source == "oci" {
      infos += sortedInfos(try VMStorageOCI().list().map { (name, vmDir, _) in
-        try VMInfo(Source: "oci", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state())
+        try VMInfo(Source: "OCI", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), SizeOnDisk: vmDir.allocatedSizeGB() - vmDir.deduplicatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
      })
    }

@@ -49,9 +49,10 @@ struct Login: AsyncParsableCommand {
    ])

    if !noValidate {
+      let registry = try Registry(host: host, namespace: "", insecure: insecure,
+                                  credentialsProviders: [credentialsProvider])
+
      do {
-        let registry = try Registry(host: host, namespace: "", insecure: insecure,
-                                    credentialsProviders: [credentialsProvider])
        try await registry.ping()
      } catch {
        throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
@@ -9,8 +9,8 @@ struct Pull: AsyncParsableCommand {
    Pulls a virtual machine from a remote OCI-compatible registry. Supports authorization via Keychain (see "tart login --help"),
    Docker credential helpers defined in ~/.docker/config.json or via TART_REGISTRY_USERNAME/TART_REGISTRY_PASSWORD environment variables.

-    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image to fit via "tart prune".
-    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image
+    to fit. This behaviour is called "automatic pruning" and can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
    """
  )

@@ -23,6 +23,9 @@ struct Pull: AsyncParsableCommand {
  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
  var concurrency: UInt = 4

+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
  func validate() throws {
    if concurrency < 1 {
      throw ValidationError("network concurrency cannot be less than 1")
@@ -43,6 +46,6 @@ struct Pull: AsyncParsableCommand {

    defaultLogger.appendNewLine("pulling \(remoteName)...")

-    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
  }
 }
@@ -6,7 +6,7 @@ import Compression
 struct Push: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Push a VM to a registry")

-  @Argument(help: "local or remote VM name")
+  @Argument(help: "local or remote VM name", completion: .custom(completeMachines))
  var localName: String

  @Argument(help: "remote VM name(s)")
@@ -15,6 +15,9 @@ struct Push: AsyncParsableCommand {
  @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
  var insecure: Bool = false

+  @Option(help: "network concurrency to use when pushing a local VM to the OCI-compatible registry")
+  var concurrency: UInt = 4
+
  @Option(help: ArgumentHelp("chunk size in MB if registry supports chunked uploads",
                             discussion: """
                             By default monolithic method is used for uploading blobs to the registry but some registries support a more efficient chunked method.
@@ -33,6 +36,10 @@ struct Push: AsyncParsableCommand {
  func run() async throws {
    let ociStorage = VMStorageOCI()
    let localVMDir = try VMStorageHelper.open(localName)
+    let lock = try localVMDir.lock()
+    if try !lock.trylock() {
+      throw RuntimeError.VMIsRunning(localName)
+    }

    // Parse remote names supplied by the user
    let remoteNames = try remoteNames.map{
@@ -73,7 +80,8 @@ struct Push: AsyncParsableCommand {
          registry: registry,
          references: references,
          chunkSizeMb: chunkSize,
-          diskFormat: diskFormat
+          diskFormat: diskFormat,
+          concurrency: concurrency
        )
        // Populate the local cache (if requested)
        if populateCache {
@@ -2,9 +2,9 @@ import ArgumentParser
 import Foundation

 struct Rename: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Rename a VM")
+  static var configuration = CommandConfiguration(abstract: "Rename a local VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Argument(help: "new VM name")
@@ -20,7 +20,7 @@ struct Rename: AsyncParsableCommand {
    let localStorage = VMStorageLocal()

    if !localStorage.exists(name) {
-      throw ValidationError("failed to rename a non-existent VM: \(name)")
+      throw ValidationError("failed to rename a non-existent local VM: \(name)")
    }

    if localStorage.exists(newName) {
@@ -5,16 +5,67 @@ import Dispatch
 import SwiftUI
 import Virtualization
 import Sentry
+import System

 var vm: VM?

 struct IPNotFound: Error {
 }

+@available(macOS 14, *)
+extension VZDiskSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
+extension VZDiskImageSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "fsync":
+      self = .fsync
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
+extension VZDiskImageCachingMode {
+  public init?(_ description: String) throws {
+    switch description {
+    case "automatic":
+      self = .automatic
+    case "cached":
+      self = .cached
+    case "uncached":
+      self = .uncached
+    case "":
+      return nil
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image caching mode: \"\(description)\"")
+    }
+  }
+}
+
 struct Run: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Run a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Flag(help: ArgumentHelp(
@@ -33,12 +84,17 @@ struct Run: AsyncParsableCommand {
  ))
  var serialPath: String?

-  @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .hidden))
+  @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .private))
  var graphics: Bool = false

  @Flag(help: "Disable audio pass-through to host.")
  var noAudio: Bool = false

+  @Flag(help: ArgumentHelp(
+    "Disable clipboard sharing between host and guest.",
+    discussion: "Only works with Linux-based guest operating systems."))
+  var noClipboard: Bool = false
+
  #if arch(arm64)
    @Flag(help: "Boot into recovery mode")
  #endif
@@ -54,25 +110,36 @@ struct Run: AsyncParsableCommand {

  #if arch(arm64)
    @Flag(help: ArgumentHelp(
-      "Use Virtualization.Framework's VNC server instead of the build-in UI.",
+      "Use Virtualization.Framework's VNC server instead of the built-in UI.",
      discussion: "Useful since this type of VNC is available in recovery mode and in macOS installation.\n"
        + "Note that this feature is experimental and there may be bugs present when using VNC."))
  #endif
  var vncExperimental: Bool = false

  @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\" --disk=\"/dev/disk0\")
+  Additional disk attachments with an optional read-only and synchronization options (e.g. --disk="disk.bin", --disk="ubuntu.iso:ro", --disk="/dev/disk0", --disk "ghcr.io/cirruslabs/xcode:16.0:ro" or --disk="nbd://localhost:10809/myDisk:sync=none")
  """, discussion: """
-  Can be either a disk image file or a block device like a local SSD on AWS EC2 Mac instances.
+  The disk attachment can be a:

-  Learn how to create a disk image using Disk Utility here:
-  https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+  * path to a disk image file
+  * path to a block device (for example, a local SSD on AWS EC2 Mac instances)
+  * remote VM name whose disk will be mounted
+  * Network Block Device (NBD) URL

-  To work with block devices 'tart' binary must be executed as root which affects locating Tart VMs.
-  To workaround this issue pass TART_HOME explicitly:
+  Options are comma-separated and are as follows:
+
+  * ro — attach the specified disk in read-only mode instead of the default read-write (e.g. --disk="disk.img:ro")
+
+  * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --disk="disk.img:sync=none")
+
+  Learn how to create a disk image using Disk Utility here: https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+
+  To work with block devices, the easiest way is to modify their permissions (e.g. by using "sudo chown $USER /dev/diskX") or to run the Tart binary as root, which affects locating Tart VMs.
+
+  To work around this pass TART_HOME explicitly:

  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
-  """, valueName: "path[:ro]"))
+  """, valueName: "path[:options]"))
  var disk: [String] = []

  #if arch(arm64)
@@ -106,6 +173,9 @@ struct Run: AsyncParsableCommand {
  """, valueName: "[name:]path[:options]"))
  var dir: [String] = []

+  @Flag(help: ArgumentHelp("Enable nested virtualization if possible"))
+  var nested: Bool = false
+
  @Option(help: ArgumentHelp("""
  Use bridged networking instead of the default shared (NAT) networking \n(e.g. --net-bridged=en0 or --net-bridged=\"Wi-Fi\")
  """, discussion: """
@@ -123,6 +193,26 @@ struct Run: AsyncParsableCommand {
  @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
  var netHost: Bool = false

+  @Option(help: ArgumentHelp("Set the root disk options (e.g. --root-disk-opts=\"ro\" or --root-disk-opts=\"caching=cached,sync=none\")",
+                             discussion: """
+                             Options are comma-separated and are as follows:
+
+                             * ro — attach the root disk in read-only mode instead of the default read-write (e.g. --root-disk-opts="ro")
+
+                             * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --root-disk-opts="sync=none")
+
+                             * sync=fsync — enable data synchronization with the permanent storage, but don't ensure that it was actually written (e.g. --root-disk-opts="sync=fsync")
+
+                             * sync=full — enable data synchronization with the permanent storage and ensure that it was actually written (e.g. --root-disk-opts="sync=full")
+
+                             * caching=automatic — allows the virtualization framework to automatically determine whether to enable data caching
+
+                             * caching=cached — enabled data caching
+
+                             * caching=uncached — disables data caching
+                             """, valueName: "options"))
+  var rootDiskOpts: String = ""
+
  #if arch(arm64)
    @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
  #endif
@@ -149,21 +239,33 @@ struct Run: AsyncParsableCommand {
      throw ValidationError("--net-bridged, --net-softnet and --net-host are mutually exclusive")
    }

-    if graphics {
-      print("--graphics is deprecated and will be removed in the future.\n")
+    if graphics && noGraphics {
+      throw ValidationError("--graphics and --no-graphics are mutually exclusive")
    }

    if (noGraphics || vnc || vncExperimental) && captureSystemKeys {
      throw ValidationError("--captures-system-keys can only be used with the default VM view")
    }

+    if nested {
+      if #unavailable(macOS 15) {
+        throw ValidationError("Nested virtualization is supported on hosts starting with macOS 15 (Sequia), and later.")
+      } else if !VZGenericPlatformConfiguration.isNestedVirtualizationSupported {
+        throw ValidationError("Nested virtualization is available for Mac with the M3 chip, and later.")
+      }
+    }
+
    let localStorage = VMStorageLocal()
    let vmDir = try localStorage.open(name)
-    if try vmDir.state() == "suspended" {
+    if try vmDir.state() == .Suspended {
      suspendable = true
    }

    if suspendable {
+      let config = try VMConfig.init(fromURL: vmDir.configURL)
+      if !(config.platform is PlatformSuspendable) {
+        throw ValidationError("You can only suspend macOS VMs")
+      }
      if dir.count > 0 {
        throw ValidationError("Suspending VMs with shared directories is not supported")
      }
@@ -182,26 +284,22 @@ struct Run: AsyncParsableCommand {
    let vmDir = try localStorage.open(name)

    let storageLock = try FileLock(lockURL: Config().tartHomeDir)
-    if try vmDir.state() == "suspended" {
-      try storageLock.lock() // lock before checking
-      let needToGenerateNewMac = try localStorage.list().contains {
-        // check if there is a running VM with the same MAC but different name
-        try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
-      }
-
-      if needToGenerateNewMac {
-        print("There is already a running VM with the same MAC address!")
-        print("Resetting VM to assign a new MAC address...")
-        try vmDir.regenerateMACAddress()
-      }
+    try storageLock.lock()
+    // check if there is a running VM with the same MAC address
+    let hasRunningMACCollision = try localStorage.list().contains {
+      // check if there is a running VM with the same MAC but different name
+      try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
+    }
+    if hasRunningMACCollision {
+      print("There is already a running VM with the same MAC address!")
+      print("Resetting VM to assign a new MAC address...")
+      try vmDir.regenerateMACAddress()
    }

    if (netSoftnet || netHost) && isInteractiveSession() {
      try Softnet.configureSUIDBitIfNeeded()
    }

-    let additionalDiskAttachments = try additionalDiskAttachments()
-
    var serialPorts: [VZSerialPortConfiguration] = []
    if serial {
      let tty_fd = createPTY()
@@ -220,14 +318,21 @@ struct Run: AsyncParsableCommand {
      serialPorts.append(createSerialPortConfiguration(tty_read!, tty_write!))
    }

+    // Parse root disk options
+    let diskOptions = DiskOptions(rootDiskOpts)
+
    vm = try VM(
      vmDir: vmDir,
      network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalStorageDevices: additionalDiskAttachments,
+      additionalStorageDevices: try additionalDiskAttachments(),
      directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
      serialPorts: serialPorts,
      suspendable: suspendable,
-      audio: !noAudio
+      nested: nested,
+      audio: !noAudio,
+      clipboard: !noClipboard,
+      sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw),
+      caching: VZDiskImageCachingMode(diskOptions.cachingModeRaw)
    )

    let vncImpl: VNC? = try {
@@ -277,10 +382,38 @@ struct Run: AsyncParsableCommand {
          }
        #endif

-        try await vm!.start(recovery: recovery, resume: resume)
+        do {
+          try await vm!.start(recovery: recovery, resume: resume)
+        } catch let error as VZError {
+          if error.code == .virtualMachineLimitExceeded {
+            var hint = ""
+
+            do {
+              let runningVMs: [String] = try localStorage.list().compactMap { (name, vmDir) in
+                if try !vmDir.running() {
+                  return nil
+                }
+
+                return name
+              }
+
+              if !runningVMs.isEmpty {
+                let runningVMsJoined = runningVMs.joined(separator: ", ")
+
+                hint = " (other running VMs: \(runningVMsJoined))"
+              }
+            } catch {
+              // we can't provide any hint
+            }
+
+            throw RuntimeError.VirtualMachineLimitExceeded(hint)
+          }
+
+          throw error
+        }

        if let vncImpl = vncImpl {
-          let vncURL = try await vncImpl.waitForURL()
+          let vncURL = try await vncImpl.waitForURL(netBridged: !netBridged.isEmpty)

          if noGraphics || ProcessInfo.processInfo.environment["CI"] != nil {
            print("VNC server is running at \(vncURL)")
@@ -349,9 +482,24 @@ struct Run: AsyncParsableCommand {
    }
    sigusr1Src.activate()

-    if noGraphics {
-      // enter the main even loop, without bringing up any UI,
-      // and just wait for the VM to exit.
+    // Gracefull shutdown support. For macOS this brings up a dialog,
+    // asking the user if they are sure they want to shut down.
+    signal(SIGUSR2, SIG_IGN)
+    let sigusr2Src = DispatchSource.makeSignalSource(signal: SIGUSR2)
+    sigusr2Src.setEventHandler {
+      Task {
+        print("Requesting guest OS to stop...")
+        try vm!.virtualMachine.requestStop()
+      }
+    }
+    sigusr2Src.activate()
+
+    let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
+    if noGraphics || useVNCWithoutGraphics {
+      // Enter the main event loop without bringing up any UI,
+      // waiting for the VM to exit.
+      NSApplication.shared.setActivationPolicy(.prohibited)
+
      NSApplication.shared.run()
    } else {
      runUI(suspendable, captureSystemKeys)
@@ -373,21 +521,22 @@ struct Run: AsyncParsableCommand {
  }

  func userSpecifiedNetwork(vmDir: VMDirectory) throws -> Network? {
+    var softnetExtraArguments: [String] = []
+
+    if let netSoftnetAllow = netSoftnetAllow {
+      softnetExtraArguments += ["--allow", netSoftnetAllow]
+    }
+
    if netSoftnet {
      let config = try VMConfig.init(fromURL: vmDir.configURL)

-      var extraArguments: [String] = []
-
-      if let netSoftnetAllow = netSoftnetAllow {
-        extraArguments += ["--allow", netSoftnetAllow]
-      }
-
-      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: extraArguments)
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: softnetExtraArguments)
    }

    if netHost {
      let config = try VMConfig.init(fromURL: vmDir.configURL)
-      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"])
+
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"] + softnetExtraArguments)
    }

    if netBridged.count > 0 {
@@ -421,46 +570,9 @@ struct Run: AsyncParsableCommand {
  }

  func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
-    var result: [VZStorageDeviceConfiguration] = []
-    let readOnlySuffix = ":ro"
-    let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
-
-    for rawDisk in expandedDiskPaths {
-      let diskReadOnly = rawDisk.hasSuffix(readOnlySuffix)
-      let diskPath = diskReadOnly ? String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count)) : rawDisk
-      let diskURL = URL(fileURLWithPath: diskPath)
-
-      // check if `diskPath` is a block device or a directory
-      if pathHasMode(diskPath, mode: S_IFBLK) || pathHasMode(diskPath, mode: S_IFDIR) {
-        print("Using block device\n")
-        guard #available(macOS 14, *) else {
-          throw UnsupportedOSError("attaching block devices", "are")
-        }
-        let fileHandle = FileHandle(forUpdatingAtPath: diskPath)
-        guard fileHandle != nil else {
-          if ProcessInfo.processInfo.userName != "root" {
-            throw RuntimeError.VMConfigurationError("need to run as root to work with block devices")
-          }
-          throw RuntimeError.VMConfigurationError("block device \(diskURL.url.path) seems to be already in use, unmount it first via 'diskutil unmount'")
-        }
-        let attachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: fileHandle!, readOnly: diskReadOnly, synchronizationMode: .full)
-        result.append(VZVirtioBlockDeviceConfiguration(attachment: attachment))
-      } else {
-        // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-        // see https://github.com/cirruslabs/tart/issues/323 for more details.
-        if try !diskReadOnly && !FileLock(lockURL: diskURL).trylock() {
-          throw RuntimeError.DiskAlreadyInUse("disk \(diskURL.url.path) seems to be already in use, unmount it first in Finder")
-        }
-
-        let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
-          url: diskURL,
-          readOnly: diskReadOnly
-        )
-        result.append(VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
-      }
+    try disk.map {
+      try AdditionalDisk(parseFrom: $0).configuration
    }
-
-    return result
  }

  func directoryShares() throws -> [VZDirectorySharingDeviceConfiguration] {
@@ -533,80 +645,79 @@ struct Run: AsyncParsableCommand {
  }

  private func runUI(_ suspendable: Bool, _ captureSystemKeys: Bool) {
-    let nsApp = NSApplication.shared
-    nsApp.setActivationPolicy(.regular)
-    nsApp.activate(ignoringOtherApps: true)
-
-    struct MainApp: App {
-      static var disappearSignal: Int32 = SIGINT
-      static var capturesSystemKeys: Bool = false
-
-      @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
-
-      var body: some Scene {
-        WindowGroup(vm!.name) {
-          Group {
-            VMView(vm: vm!, capturesSystemKeys: MainApp.capturesSystemKeys).onAppear {
-              NSWindow.allowsAutomaticWindowTabbing = false
-            }.onDisappear {
-              let ret = kill(getpid(), MainApp.disappearSignal)
-              if ret != 0 {
-                // Fallback to the old termination method that doesn't
-                // propagate the cancellation to Task's in case graceful
-                // termination via kill(2) is not successful
-                NSApplication.shared.terminate(self)
-              }
-            }
-          }.frame(
-            minWidth: CGFloat(vm!.config.display.width),
-            idealWidth: CGFloat(vm!.config.display.width),
-            maxWidth: .infinity,
-            minHeight: CGFloat(vm!.config.display.height),
-            idealHeight: CGFloat(vm!.config.display.height),
-            maxHeight: .infinity
-          )
-        }.commands {
-          // Remove some standard menu options
-          CommandGroup(replacing: .help, addition: {})
-          CommandGroup(replacing: .newItem, addition: {})
-          CommandGroup(replacing: .pasteboard, addition: {})
-          CommandGroup(replacing: .textEditing, addition: {})
-          CommandGroup(replacing: .undoRedo, addition: {})
-          CommandGroup(replacing: .windowSize, addition: {})
-          // Replace some standard menu options
-          CommandGroup(replacing: .appInfo) { AboutTart(config: vm!.config) }
-          CommandMenu("Control") {
-            Button("Start") {
-              Task { try await vm!.virtualMachine.start() }
-            }
-            Button("Stop") {
-              Task { try await vm!.virtualMachine.stop() }
-            }
-            Button("Request Stop") {
-              Task { try vm!.virtualMachine.requestStop() }
-            }
-            if #available(macOS 14, *) {
-              Button("Suspend") {
-                kill(getpid(), SIGUSR1)
-              }
-            }
-          }
-        }
-      }
-    }
-
-    MainApp.disappearSignal = suspendable ? SIGUSR1 : SIGINT
+    MainApp.suspendable = suspendable
    MainApp.capturesSystemKeys = captureSystemKeys
    MainApp.main()
  }
 }

-// The only way to fully remove Edit menu item.
-class MinimalMenuAppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
-  let indexOfEditMenu = 2
+struct MainApp: App {
+  static var suspendable: Bool = false
+  static var capturesSystemKeys: Bool = false

-  func applicationDidFinishLaunching(_ : Notification) {
-    NSApplication.shared.mainMenu?.removeItem(at: indexOfEditMenu)
+  @NSApplicationDelegateAdaptor private var appDelegate: AppDelegate
+
+  var body: some Scene {
+    WindowGroup(vm!.name) {
+      Group {
+        VMView(vm: vm!, capturesSystemKeys: MainApp.capturesSystemKeys).onAppear {
+          NSWindow.allowsAutomaticWindowTabbing = false
+        }.onDisappear {
+          let ret = kill(getpid(), MainApp.suspendable ? SIGUSR1 : SIGINT)
+          if ret != 0 {
+            // Fallback to the old termination method that doesn't
+            // propagate the cancellation to Task's in case graceful
+            // termination via kill(2) is not successful
+            NSApplication.shared.terminate(self)
+          }
+        }
+      }.frame(
+        minWidth: CGFloat(vm!.config.display.width),
+        idealWidth: CGFloat(vm!.config.display.width),
+        maxWidth: .infinity,
+        minHeight: CGFloat(vm!.config.display.height),
+        idealHeight: CGFloat(vm!.config.display.height),
+        maxHeight: .infinity
+      )
+    }.commands {
+      // Remove some standard menu options
+      CommandGroup(replacing: .help, addition: {})
+      CommandGroup(replacing: .newItem, addition: {})
+      CommandGroup(replacing: .pasteboard, addition: {})
+      CommandGroup(replacing: .textEditing, addition: {})
+      CommandGroup(replacing: .undoRedo, addition: {})
+      CommandGroup(replacing: .windowSize, addition: {})
+      // Replace some standard menu options
+      CommandGroup(replacing: .appInfo) { AboutTart(config: vm!.config) }
+      CommandMenu("Control") {
+        Button("Start") {
+          Task { try await vm!.virtualMachine.start() }
+        }
+        Button("Stop") {
+          Task { try await vm!.virtualMachine.stop() }
+        }
+        Button("Request Stop") {
+          Task { try vm!.virtualMachine.requestStop() }
+        }
+        if #available(macOS 14, *) {
+          if (MainApp.suspendable) {
+            Button("Suspend") {
+              kill(getpid(), SIGUSR1)
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
+  func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply {
+    if (kill(getpid(), MainApp.suspendable ? SIGUSR1 : SIGINT) == 0) {
+      return .terminateLater
+    } else {
+      return .terminateNow
+    }
  }
 }

@@ -653,12 +764,12 @@ struct VMView: NSViewRepresentable {

    machineView.capturesSystemKeys = capturesSystemKeys

-    // Enable automatic display reconfiguration
-    // for guests that support it
+    // If not specified, enable automatic display
+    // reconfiguration for guests that support it
    //
    // This is disabled for Linux because of poor HiDPI
    // support, which manifests in fonts being too small
-    if #available(macOS 14.0, *), vm.config.os != .linux {
+    if #available(macOS 14.0, *), vm.config.displayRefit ?? (vm.config.os != .linux) {
      machineView.automaticallyReconfiguresDisplay = true
    }

@@ -670,6 +781,138 @@ struct VMView: NSViewRepresentable {
  }
 }

+struct AdditionalDisk {
+  let configuration: VZStorageDeviceConfiguration
+
+  init(parseFrom: String) throws {
+    let (diskPath, readOnly, syncModeRaw, cachingModeRaw) = Self.parseOptions(parseFrom)
+
+    self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw, cachingModeRaw: cachingModeRaw)
+  }
+
+  static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String, cachingModeRaw: String) throws -> VZStorageDeviceConfiguration {
+    let diskURL = URL(string: diskPath)
+
+    if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
+      guard #available(macOS 14, *) else {
+        throw UnsupportedOSError("attaching Network Block Devices", "are")
+      }
+
+      let nbdAttachment = try VZNetworkBlockDeviceStorageDeviceAttachment(
+        url: diskURL!,
+        timeout: 30,
+        isForcedReadOnly: diskReadOnly,
+        synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)
+      )
+
+      return VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment)
+    }
+
+    // Expand the tilde (~) since at this point we're dealing with a local path,
+    // and "expandingTildeInPath" seems to corrupt the remote URLs like nbd://
+    let diskPath = NSString(string: diskPath).expandingTildeInPath
+
+    let diskFileURL = URL(fileURLWithPath: diskPath)
+
+    if pathHasMode(diskPath, mode: S_IFBLK) {
+      guard #available(macOS 14, *) else {
+        throw UnsupportedOSError("attaching block devices", "are")
+      }
+
+      let fd = open(diskPath, diskReadOnly ? O_RDONLY : O_RDWR)
+      if fd == -1 {
+        let details = Errno(rawValue: CInt(errno))
+
+        switch details.rawValue {
+        case EBUSY:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "already in use, try umounting it via \"diskutil unmountDisk\" (when the whole disk) or \"diskutil umount\" (when mounting a single partition)")
+        case EACCES:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "permission denied, consider changing the disk's owner using \"sudo chown $USER \(diskFileURL.url.path)\" or run Tart as a superuser (see --disk help for more details on how to do that correctly)")
+        default:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "\(details)")
+        }
+      }
+
+      let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
+                                                                         readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw))
+
+      return VZVirtioBlockDeviceConfiguration(attachment: blockAttachment)
+    }
+
+    // Support remote VM names in --disk command-line argument
+    if let remoteName = try? RemoteName(diskPath) {
+      let vmDir = try VMStorageOCI().open(remoteName)
+
+      // Unfortunately, VZDiskImageStorageDeviceAttachment does not support
+      // FileHandle, so we can't easily clone the disk, open it and unlink(2)
+      // to simplify the garbage collection, so use an intermediate directory.
+      let clonedDiskURL = try Config().tartTmpDir.appendingPathComponent("run-disk-\(UUID().uuidString)")
+
+      try FileManager.default.copyItem(at: vmDir.diskURL, to: clonedDiskURL)
+
+      let lock = try FileLock(lockURL: clonedDiskURL)
+      try lock.lock()
+
+      let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: clonedDiskURL, readOnly: diskReadOnly)
+
+      return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+    }
+
+    // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
+    // see https://github.com/cirruslabs/tart/issues/323 for more details.
+    if try !diskReadOnly && !FileLock(lockURL: diskFileURL).trylock() {
+      throw RuntimeError.DiskAlreadyInUse("disk \(diskFileURL.url.path) seems to be already in use, unmount it first in Finder")
+    }
+
+    let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
+      url: diskFileURL,
+      readOnly: diskReadOnly,
+      cachingMode: try VZDiskImageCachingMode(cachingModeRaw) ?? .automatic,
+      synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
+    )
+
+    return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+  }
+
+  static func parseOptions(_ parseFrom: String) -> (String, Bool, String, String) {
+    var arguments = parseFrom.split(separator: ":")
+
+    let options = DiskOptions(String(arguments.last!))
+    if options.foundAtLeastOneOption {
+      arguments.removeLast()
+    }
+
+    return (arguments.joined(separator: ":"), options.readOnly, options.syncModeRaw, options.cachingModeRaw)
+  }
+}
+
+struct DiskOptions {
+  var readOnly: Bool = false
+  var syncModeRaw: String = ""
+  var cachingModeRaw: String = ""
+  var foundAtLeastOneOption: Bool = false
+
+  init(_ parseFrom: String) {
+    let options = parseFrom.split(separator: ",")
+
+    for option in options {
+      switch true {
+      case option == "ro":
+        self.readOnly = true
+        self.foundAtLeastOneOption = true
+      case option.hasPrefix("sync="):
+        self.syncModeRaw = String(option.dropFirst("sync=".count))
+        self.foundAtLeastOneOption = true
+      case option.hasPrefix("caching="):
+        self.cachingModeRaw = String(option.dropFirst("caching=".count))
+        self.foundAtLeastOneOption = true
+      default:
+        continue
+      }
+    }
+  }
+}
+
 struct DirectoryShare {
  let name: String?
  let path: URL
@@ -816,5 +1059,5 @@ func pathHasMode(_ path: String, mode: mode_t) -> Bool {
  guard statRes != -1 else {
    return false
  }
-  return (Int32(st.st_mode) & Int32(mode)) == Int32(mode)
+  return (st.st_mode & S_IFMT) == mode
 }
@@ -1,10 +1,11 @@
 import ArgumentParser
 import Foundation
+import Virtualization

 struct Set: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "set", abstract: "Modify VM's configuration")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
  var name: String

  @Option(help: "Number of VM CPUs")
@@ -16,6 +17,20 @@ struct Set: AsyncParsableCommand {
  @Option(help: "VM display resolution in a format of <width>x<height>. For example, 1200x800")
  var display: VMDisplayConfig?

+  @Flag(inversion: .prefixedNo, help: ArgumentHelp("Whether to automatically reconfigure the VM's display to fit the window"))
+  var displayRefit: Bool? = nil
+
+  @Flag(help: ArgumentHelp("Generate a new random MAC address for the VM."))
+  var randomMAC: Bool = false
+
+  #if arch(arm64)
+    @Flag(help: ArgumentHelp("Generate a new random serial number for the macOS VM."))
+  #endif
+  var randomSerial: Bool = false
+
+  @Option(help: ArgumentHelp("Replace the VM's disk contents with the disk contents at path.", valueName: "path"))
+  var disk: String?
+
  @Option(help: ArgumentHelp("Resize the VMs disk to the specified size in GB (note that the disk size can only be increased to avoid losing data)",
                             discussion: """
                             Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images
@@ -51,8 +66,29 @@ struct Set: AsyncParsableCommand {
      }
    }

+    vmConfig.displayRefit = displayRefit
+
+    if randomMAC {
+      vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
+    }
+
+    #if arch(arm64)
+      if randomSerial {
+        let oldPlatform = vmConfig.platform as! Darwin
+        vmConfig.platform = Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: oldPlatform.hardwareModel)
+      }
+    #endif
+
    try vmConfig.save(toURL: vmDir.configURL)

+    if let disk = disk {
+      let temporaryDiskURL = try Config().tartTmpDir.appendingPathComponent("set-disk-\(UUID().uuidString)")
+
+      try FileManager.default.copyItem(atPath: disk, toPath: temporaryDiskURL.path())
+
+      _ = try FileManager.default.replaceItemAt(vmDir.diskURL, withItemAt: temporaryDiskURL)
+    }
+
    if diskSize != nil {
      try vmDir.resizeDisk(diskSize!)
    }
@@ -6,7 +6,7 @@ import SwiftDate
 struct Stop: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "stop", abstract: "Stop a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeRunningMachines))
  var name: String

  @Option(name: [.short, .long], help: "Seconds to wait for graceful termination before forcefully terminating the VM")
@@ -15,12 +15,12 @@ struct Stop: AsyncParsableCommand {
  func run() async throws {
    let vmDir = try VMStorageLocal().open(name)
    switch try vmDir.state() {
-    case "suspended":
+    case .Suspended:
      try stopSuspended(vmDir)
-    case "running":
+    case .Running:
      try await stopRunning(vmDir)
-    default:
-      return
+    case .Stopped:
+      throw RuntimeError.VMNotRunning(name)
    }
  }

@@ -6,7 +6,7 @@ import SwiftDate
 struct Suspend: AsyncParsableCommand {
  static var configuration = CommandConfiguration(commandName: "suspend", abstract: "Suspend a VM")

-  @Argument(help: "VM name")
+  @Argument(help: "VM name", completion: .custom(completeRunningMachines))
  var name: String

  func run() async throws {
@@ -14,7 +14,7 @@ struct Suspend: AsyncParsableCommand {
    let lock = try vmDir.lock()

    // Find the VM's PID
-    var pid = try lock.pid()
+    let pid = try lock.pid()
    if pid == 0 {
      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
    }
@@ -39,9 +39,10 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
    inPipe.fileHandleForWriting.write("\(host)\n".data(using: .utf8)!)
    inPipe.fileHandleForWriting.closeFile()

+    let outputData = try outPipe.fileHandleForReading.readToEnd()
+
    process.waitUntilExit()

-    let outputData = try outPipe.fileHandleForReading.readToEnd()
    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
      if let outputData = outputData {
        print(String(decoding: outputData, as: UTF8.self))
@@ -41,7 +41,7 @@ class KeychainCredentialsProvider: CredentialsProvider {
                              kSecAttrLabel as String: "Tart Credentials",
    ]
    let value: [String: Any] = [kSecAttrAccount as String: user,
-                                kSecValueData as String: passwordData,
+                                kSecValueData as String: passwordData as Any,
    ]

    let status = SecItemCopyMatching(key as CFDictionary, nil)
@@ -1,56 +1,6 @@
 import Foundation
-import AsyncAlgorithms

-fileprivate let urlSession = createURLSession()
-
-class Fetcher {
-  static func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    if viaFile {
-      return try await fetchViaFile(request)
-    }
-
-    return try await fetchViaMemory(request)
-  }
-
-  private static func fetchViaMemory(_ request: URLRequest) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let dataCh = AsyncThrowingChannel<Data, Error>()
-
-    let (data, response) = try await urlSession.data(for: request)
-
-    Task {
-      await dataCh.send(data)
-
-      dataCh.finish()
-    }
-
-    return (dataCh, response as! HTTPURLResponse)
-  }
-
-  private static func fetchViaFile(_ request: URLRequest) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let dataCh = AsyncThrowingChannel<Data, Error>()
-
-    let (fileURL, response) = try await urlSession.download(for: request)
-
-    // Acquire a handle to the downloaded file and then remove it.
-    //
-    // This keeps a working reference to that file, yet we don't
-    // have to deal with the cleanup any more.
-    let mappedFile = try Data(contentsOf: fileURL, options: [.alwaysMapped])
-    try FileManager.default.removeItem(at: fileURL)
-
-    Task {
-      for chunk in (0 ..< mappedFile.count).chunks(ofCount: 64 * 1024 * 1024) {
-        await dataCh.send(mappedFile.subdata(in: chunk))
-      }
-
-      dataCh.finish()
-    }
-
-    return (dataCh, response as! HTTPURLResponse)
-  }
-}
-
-fileprivate func createURLSession() -> URLSession {
+fileprivate var urlSessionConfiguration: URLSessionConfiguration {
  let config = URLSessionConfiguration.default

  // Harbor expects a CSRF token to be present if the HTTP client
@@ -63,5 +13,84 @@ fileprivate func createURLSession() -> URLSession {
  // [2]: https://github.com/cirruslabs/tart/issues/295
  config.httpShouldSetCookies = false

-  return URLSession(configuration: config)
+  return config
+}
+
+class Fetcher {
+  static func fetch(_ request: URLRequest, viaFile: Bool = false, progress: Progress? = nil) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
+    let delegate = Delegate()
+    let session = URLSession(configuration: urlSessionConfiguration, delegate: delegate, delegateQueue: nil)
+    let task = session.dataTask(with: request)
+
+    let stream = AsyncThrowingStream<Data, Error> { continuation in
+      delegate.streamContinuation = continuation
+    }
+
+    let response = try await withCheckedThrowingContinuation { continuation in
+      delegate.responseContinuation = continuation
+      task.resume()
+    }
+
+    return (stream, response as! HTTPURLResponse)
+  }
+}
+
+fileprivate class Delegate: NSObject, URLSessionDataDelegate {
+  var responseContinuation: CheckedContinuation<URLResponse, Error>?
+  var streamContinuation: AsyncThrowingStream<Data, Error>.Continuation?
+
+  private var buffer: Data = Data()
+  private let bufferFlushSize = 16 * 1024 * 1024
+
+  func urlSession(
+    _ session: URLSession,
+    dataTask: URLSessionDataTask,
+    didReceive response: URLResponse,
+    completionHandler: @escaping (URLSession.ResponseDisposition) -> Void
+  ) {
+    // Soft-limit for the maximum buffer capacity
+    let capacity = min(response.expectedContentLength, Int64(bufferFlushSize))
+
+    // Pre-initialize buffer as we now know the capacity
+    buffer = Data(capacity: Int(capacity))
+
+    responseContinuation?.resume(returning: response)
+    responseContinuation = nil
+    completionHandler(.allow)
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    dataTask: URLSessionDataTask,
+    didReceive data: Data
+  ) {
+    buffer.append(data)
+
+    if buffer.count >= bufferFlushSize {
+      streamContinuation?.yield(buffer)
+      buffer.removeAll(keepingCapacity: true)
+    }
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    task: URLSessionTask,
+    didCompleteWithError error: Error?
+  ) {
+    if let error = error {
+      responseContinuation?.resume(throwing: error)
+      responseContinuation = nil
+
+      streamContinuation?.finish(throwing: error)
+      streamContinuation = nil
+    } else {
+      if !buffer.isEmpty {
+        streamContinuation?.yield(buffer)
+        buffer.removeAll(keepingCapacity: true)
+      }
+
+      streamContinuation?.finish()
+      streamContinuation = nil
+    }
+  }
 }
@@ -0,0 +1,57 @@
+import Foundation
+
+struct LocalLayerCache {
+  struct DigestInfo {
+    let range: Range<Data.Index>
+    let compressedDigest: String
+    let uncompressedContentDigest: String?
+  }
+
+  let name: String
+  let deduplicatedBytes: UInt64
+  let diskURL: URL
+
+  private let mappedDisk: Data
+  private var digestToRange: [String: DigestInfo] = [:]
+  private var offsetToRange: [UInt64: DigestInfo] = [:]
+
+  init?(_ name: String, _ deduplicatedBytes: UInt64, _ diskURL: URL, _ manifest: OCIManifest) throws {
+    self.name = name
+    self.deduplicatedBytes = deduplicatedBytes
+    self.diskURL = diskURL
+
+    // mmap(2) the disk that contains the layers from the manifest
+    self.mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+
+    // Record the ranges of the disk layers listed in the manifest
+    var offset: UInt64 = 0
+
+    for layer in manifest.layers.filter({ $0.mediaType == diskV2MediaType }) {
+      guard let uncompressedSize = layer.uncompressedSize() else {
+        return nil
+      }
+
+      let info = DigestInfo(
+        range: Int(offset)..<Int(offset + uncompressedSize),
+        compressedDigest: layer.digest,
+        uncompressedContentDigest: layer.uncompressedContentDigest()!
+      )
+      self.digestToRange[layer.digest] = info
+      self.offsetToRange[offset] = info
+
+      offset += uncompressedSize
+    }
+  }
+
+  func findInfo(digest: String, offsetHint: UInt64) -> DigestInfo? {
+    // Layers can have the same digests, for example, empty ones. Let's use the offset hint to make a better guess.
+    if let info = self.offsetToRange[offsetHint], info.compressedDigest == digest {
+      return info
+    }
+    return self.digestToRange[digest]
+  }
+
+  func subdata(_ range: Range<Data.Index>) -> Data {
+    return self.mappedDisk.subdata(in: range)
+  }
+}
@@ -52,6 +52,11 @@ struct ARPCache {
    process.standardInput = FileHandle.nullDevice

    try process.run()
+
+    guard let arpCommandOutput = try pipe.fileHandleForReading.readToEnd() else {
+      throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
+    }
+
    process.waitUntilExit()

    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
@@ -60,10 +65,6 @@ struct ARPCache {
        terminationStatus: process.terminationStatus)
    }

-    guard let arpCommandOutput = try pipe.fileHandleForReading.readToEnd() else {
-      throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
-    }
-
    self.arpCommandOutput = arpCommandOutput
  }

@@ -1,7 +1,8 @@
 import Virtualization
+import Semaphore

 protocol Network {
  func attachments() -> [VZNetworkDeviceAttachment]
-  func run(_ sema: DispatchSemaphore) throws
+  func run(_ sema: AsyncSemaphore) throws
  func stop() async throws
 }
@@ -1,4 +1,5 @@
 import Foundation
+import Semaphore
 import Virtualization

 class NetworkBridged: Network {
@@ -12,7 +13,7 @@ class NetworkBridged: Network {
    interfaces.map { VZBridgedNetworkDeviceAttachment(interface: $0) }
  }

-  func run(_ sema: DispatchSemaphore) throws {
+  func run(_ sema: AsyncSemaphore) throws {
    // no-op, only used for Softnet
  }

@@ -1,4 +1,5 @@
 import Foundation
+import Semaphore
 import Virtualization

 class NetworkShared: Network {
@@ -6,7 +7,7 @@ class NetworkShared: Network {
    [VZNATNetworkDeviceAttachment()]
  }

-  func run(_ sema: DispatchSemaphore) throws {
+  func run(_ sema: AsyncSemaphore) throws {
    // no-op, only used for Softnet
  }

@@ -1,7 +1,8 @@
-import Foundation
-import Virtualization
 import Atomics
+import Foundation
+import Semaphore
 import System
+import Virtualization

 enum SoftnetError: Error {
  case InitializationFailed(why: String)
@@ -44,7 +45,7 @@ class Softnet: Network {
    return executableURL
  }

-  func run(_ sema: DispatchSemaphore) throws {
+  func run(_ sema: AsyncSemaphore) throws {
    try process.run()

    monitorTask = Task {
@@ -1,6 +1,6 @@
 import Foundation

 protocol Disk {
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
 }
@@ -5,7 +5,7 @@ class DiskV1: Disk {
  private static let bufferSizeBytes = 4 * 1024 * 1024
  private static let layerLimitBytes = 500 * 1000 * 1000

-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
    var pushedLayers: [OCIManifestLayer] = []

    // Open the disk file
@@ -45,7 +45,7 @@ class DiskV1: Disk {
    return pushedLayers
  }

-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
      throw OCIError.FailedToCreateVmFile
    }
@@ -1,48 +1,104 @@
 import Foundation
 import Compression
+import System
+import Retry

 class DiskV2: Disk {
  private static let bufferSizeBytes = 4 * 1024 * 1024
-  private static let layerLimitBytes = 500 * 1000 * 1000
-  private static let holeGranularityBytes = 64 * 1024
+  private static let layerLimitBytes = 512 * 1024 * 1024

-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
-    var pushedLayers: [OCIManifestLayer] = []
+  // A zero chunk for faster than byte-by-byte comparisons
+  //
+  // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
+  // on the last block since it costs only 4 MiB of excess data per 512 MiB layer.
+  //
+  // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
+  // +--------------------------------------+---------------------------------------------------+
+  // | Operation                            | time(1) result                                    |
+  // +--------------------------------------+---------------------------------------------------+
+  // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
+  // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
+  // +--------------------------------------+---------------------------------------------------+
+  private static let holeGranularityBytes = 4 * 1024 * 1024
+  private static let zeroChunk = Data(count: holeGranularityBytes)
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []

    // Open the disk file
-    var mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])

    // Compress the disk file as multiple individually decompressible streams,
-    // each equal ``Self.layerLimitBytes`` bytes or slightly larger due to the
-    // internal compressor's buffer
-    var offset: UInt64 = 0
+    // each equal ``Self.layerLimitBytes`` bytes or less due to LZ4 compression
+    try await withThrowingTaskGroup(of: (Int, OCIManifestLayer).self) { group in
+      for (index, data) in mappedDisk.chunks(ofCount: layerLimitBytes).enumerated() {
+        // Respect the concurrency limit
+        if index >= concurrency {
+          if let (index, pushedLayer) = try await group.next() {
+            pushedLayers.append((index, pushedLayer))
+          }
+        }

-    while let (compressedData, uncompressedSize, uncompressedDigest) = try compressNextLayerOfLimitBytesOrMore(mappedDisk: mappedDisk, offset: offset) {
-      offset += uncompressedSize
+        // Launch a disk layer pushing task
+        group.addTask {
+          let compressedData = try (data as NSData).compressed(using: .lz4) as Data
+          let compressedDataDigest = Digest.hash(compressedData)

-      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+          try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+            if try await !registry.blobExists(compressedDataDigest) {
+              _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error: \(error.localizedDescription)")
+              print("Attempting to re-try...")

-      pushedLayers.append(OCIManifestLayer(
-        mediaType: diskV2MediaType,
-        size: compressedData.count,
-        digest: layerDigest,
-        uncompressedSize: uncompressedSize,
-        uncompressedContentDigest: uncompressedDigest
-      ))
+              return .retry
+            }

-      // Update progress using a relative value
-      progress.completedUnitCount += Int64(uncompressedSize)
+            return .throw
+          }
+
+          // Update progress using a relative value
+          progress.completedUnitCount += Int64(data.count)
+
+          return (index, OCIManifestLayer(
+            mediaType: diskV2MediaType,
+            size: compressedData.count,
+            digest: compressedDataDigest,
+            uncompressedSize: UInt64(data.count),
+            uncompressedContentDigest: Digest.hash(data)
+          ))
+        }
+      }
+
+      for try await pushedLayer in group {
+        pushedLayers.append(pushedLayer)
+      }
    }

-    return pushedLayers
+    return pushedLayers.sorted {
+      $0.index < $1.index
+    }.map {
+      $0.pushedLayer
+    }
  }

-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
    // Support resumable pulls
    let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)

-    if !pullResumed && !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
-      throw OCIError.FailedToCreateVmFile
+    if !pullResumed {
+      if deduplicate, let localLayerCache = localLayerCache {
+        // Clone the local layer cache's disk and use it as a base, potentially
+        // reducing the space usage since some blocks won't be written at all
+        try FileManager.default.copyItem(at: localLayerCache.diskURL, to: diskURL)
+      } else {
+        // Otherwise create an empty disk
+        if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+          throw OCIError.FailedToCreateVmFile
+        }
+      }
    }

    // Calculate the uncompressed disk size
@@ -62,6 +118,15 @@ class DiskV2: Disk {
    try disk.truncate(atOffset: uncompressedDiskSize)
    try disk.close()

+    // Determine the file system block size
+    var st = stat()
+    if stat(diskURL.path, &st) == -1 {
+      let details = Errno(rawValue: errno)
+
+      throw RuntimeError.PullFailed("failed to stat(2) disk \(diskURL.path): \(details)")
+    }
+    let fsBlockSize = UInt64(st.st_blksize)
+
    // Concurrently fetch and decompress layers
    try await withThrowingTaskGroup(of: Void.self) { group in
      var globalDiskWritingOffset: UInt64 = 0
@@ -86,29 +151,52 @@ class DiskV2: Disk {
        // Launch a fetching and decompression task
        group.addTask {
          // No need to fetch and decompress anything if we've already done so
-          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+          if pullResumed {
+            // do not check hash in the condition above to make it lazy e.g. only do expensive calculations if needed
+            if try Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+              // Update the progress
+              progress.completedUnitCount += Int64(diskLayer.size)
+
+              return
+            }
+          }
+
+          // Open the disk file for writing
+          let disk = try FileHandle(forWritingTo: diskURL)
+
+          // Also open the disk file for reading and verifying
+          // its contents in case the local layer cache is used
+          let rdisk: FileHandle? = if deduplicate && localLayerCache != nil {
+            try FileHandle(forReadingFrom: diskURL)
+          } else {
+            nil
+          }
+
+          // Check if we already have this layer contents in the local layer cache,
+          // or perhaps even on the cloned disk (when the deduplication is enabled)
+          if let localLayerCache = localLayerCache,
+             let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset),
+             localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest {
+            if deduplicate && localLayerInfo.range.lowerBound == diskWritingOffset {
+              // Do nothing, because the data is already on the disk that we've inherited from
+            } else {
+              // Fulfil the layer contents from the local blob cache
+              let data = localLayerCache.subdata(localLayerInfo.range)
+              _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
+            }
+
+            try disk.close()
+
+            if let rdisk = rdisk {
+              try rdisk.close()
+            }
+
            // Update the progress
            progress.completedUnitCount += Int64(diskLayer.size)

            return
          }

-          // Open the disk file
-          let disk = try FileHandle(forWritingTo: diskURL)
-
-          // A zero chunk for faster than byte-by-byte comparisons
-          //
-          // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
-          // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
-          //
-          // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
-          // +--------------------------------------+---------------------------------------------------+
-          // | Operation                            | time(1) result                                    |
-          // +--------------------------------------+---------------------------------------------------+
-          // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
-          // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
-          // +--------------------------------------+---------------------------------------------------+
-          let zeroChunk = Data(count: holeGranularityBytes)
          var diskWritingOffset = diskWritingOffset

          // Pull and decompress a single layer into the specific offset on disk
@@ -117,15 +205,7 @@ class DiskV2: Disk {
              return
            }

-            for chunk in data.chunks(ofCount: holeGranularityBytes) {
-              // Only write chunks that are not zero
-              if chunk != zeroChunk {
-                try disk.seek(toOffset: diskWritingOffset)
-                disk.write(chunk)
-              }
-
-              diskWritingOffset += UInt64(chunk.count)
-            }
+            diskWritingOffset = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
          }

          try await registry.pullBlob(diskLayer.digest) { data in
@@ -138,6 +218,10 @@ class DiskV2: Disk {
          try filter.finalize()

          try disk.close()
+
+          if let rdisk = rdisk {
+            try rdisk.close()
+          }
        }

        globalDiskWritingOffset += uncompressedLayerSize
@@ -145,44 +229,50 @@ class DiskV2: Disk {
    }
  }

-  private static func compressNextLayerOfLimitBytesOrMore(mappedDisk: Data, offset: UInt64) throws -> (Data, UInt64, String)? {
-    var compressedData = Data()
-    var bytesRead: UInt64 = 0
-    let digest = Digest()
+  private static func zeroSkippingWrite(_ disk: FileHandle, _ rdisk: FileHandle?, _ fsBlockSize: UInt64, _ offset: UInt64, _ data: Data) throws -> UInt64 {
+    var offset = offset

-    // Create a compressing filter that we will terminate upon
-    // reaching ``Self.layerLimitBytes`` of compressed data
-    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: bufferSizeBytes) { (length: Int) -> Data? in
-      if compressedData.count >= Self.layerLimitBytes {
-        return nil
+    for chunk in data.chunks(ofCount: holeGranularityBytes) {
+      // If the local layer cache is used, only write chunks that differ
+      // since the base disk can contain anything at any position
+      if let rdisk = rdisk {
+        // F_PUNCHHOLE requires the holes to be aligned to file system block boundaries
+        let isHoleAligned = (offset % fsBlockSize) == 0 && (UInt64(chunk.count) % fsBlockSize) == 0
+
+        if isHoleAligned && chunk == zeroChunk {
+          var arg = fpunchhole_t(fp_flags: 0, reserved: 0, fp_offset: off_t(offset), fp_length: off_t(chunk.count))
+
+          if fcntl(disk.fileDescriptor, F_PUNCHHOLE, &arg) == -1 {
+            let details = Errno(rawValue: errno)
+
+            throw RuntimeError.PullFailed("failed to punch hole: \(details)")
+          }
+        } else {
+          try rdisk.seek(toOffset: offset)
+          let actualContentsOnDisk = try rdisk.read(upToCount: chunk.count)
+
+          if chunk != actualContentsOnDisk {
+            try disk.seek(toOffset: offset)
+            disk.write(chunk)
+          }
+        }
+
+        offset += UInt64(chunk.count)
+
+        continue
      }

-      let readFromByte = Int(offset + bytesRead)
-
-      let numBytesToRead = min(mappedDisk.count - readFromByte, bufferSizeBytes)
-      if numBytesToRead == 0 {
-        return nil
+      // Otherwise, only write chunks that are not zero
+      // since the base disk is created from scratch and
+      // is zeroed via truncate(2)
+      if chunk != zeroChunk {
+        try disk.seek(toOffset: offset)
+        disk.write(chunk)
      }

-      let uncompressedChunk = mappedDisk.subdata(in: readFromByte ..< (readFromByte + numBytesToRead))
-
-      bytesRead += UInt64(uncompressedChunk.count)
-      digest.update(uncompressedChunk)
-
-      return uncompressedChunk
+      offset += UInt64(chunk.count)
    }

-    // Retrieve compressed data chunks, but normally no more than ``Self.layerLimitBytes`` bytes
-    while let compressedChunk = try compressingFilter.readData(ofLength: Self.bufferSizeBytes) {
-      compressedData.append(compressedChunk)
-    }
-
-    // Nothing was read this time from the disk,
-    // signal that to the consumer
-    if bytesRead == 0 {
-      return nil
-    }
-
-    return (compressedData, bytesRead, digest.finalize())
+    return offset
  }
 }
@@ -78,7 +78,7 @@ struct OCIManifestConfig: Codable, Equatable {
  var digest: String
 }

-struct OCIManifestLayer: Codable, Equatable {
+struct OCIManifestLayer: Codable, Equatable, Hashable {
  var mediaType: String
  var size: Int
  var digest: String
@@ -113,6 +113,14 @@ struct OCIManifestLayer: Codable, Equatable {
  func uncompressedContentDigest() -> String? {
    annotations?[uncompressedContentDigestAnnotation]
  }
+
+  static func == (lhs: Self, rhs: Self) -> Bool {
+    return lhs.digest == rhs.digest
+  }
+
+  func hash(into hasher: inout Hasher) {
+    hasher.combine(digest)
+  }
 }

 struct Descriptor: Equatable {
@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2

 import Antlr4

@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4

 open class ReferenceLexer: Lexer {
@@ -49,7 +49,7 @@ open class ReferenceLexer: Lexer {

 	public
 	required init(_ input: CharStream) {
-	    RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION)
+	    RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION)
 		super.init(input)
 		_interp = LexerATNSimulator(self, ReferenceLexer._ATN, ReferenceLexer._decisionToDFA, ReferenceLexer._sharedContextCache)
 	}
@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4

 /**
@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4

 open class ReferenceParser: Parser {
@@ -41,7 +41,7 @@ open class ReferenceParser: Parser {
 	static let VOCABULARY = Vocabulary(_LITERAL_NAMES, _SYMBOLIC_NAMES)

 	override open
-	func getGrammarFileName() -> String { return "java-escape" }
+	func getGrammarFileName() -> String { return "Reference.g4" }

 	override open
 	func getRuleNames() -> [String] { return ReferenceParser.ruleNames }
@@ -60,7 +60,7 @@ open class ReferenceParser: Parser {

 	override public
 	init(_ input:TokenStream) throws {
-	    RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION)
+	    RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION)
 		try super.init(input)
 		_interp = ParserATNSimulator(self,ReferenceParser._ATN,ReferenceParser._decisionToDFA, ReferenceParser._sharedContextCache)
 	}
@@ -460,7 +460,7 @@ open class ReferenceParser: Parser {
 		 		setState(63)
 		 		try _errHandler.sync(self)
 		 		_la = try _input.LA(1)
-		 		if ((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0) {
+		 		if (((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
 		 			setState(62)
 		 			try separator()

@@ -611,7 +611,7 @@ open class ReferenceParser: Parser {
 		 	setState(84)
 		 	try _errHandler.sync(self)
 		 	_la = try _input.LA(1)
-		 	while ((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0) {
+		 	while (((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
 		 		setState(79)
 		 		try separator()
 		 		setState(80)
@@ -664,7 +664,7 @@ open class ReferenceParser: Parser {
 		 	try enterOuterAlt(_localctx, 1)
 		 	setState(87)
 		 	_la = try _input.LA(1)
-		 	if (!((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
+		 	if (!(((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0))) {
 		 	try _errHandler.recoverInline(self)
 		 	}
 		 	else {
@@ -1,6 +1,5 @@
 import Foundation
 import Algorithms
-import AsyncAlgorithms

 enum RegistryError: Error {
  case UnexpectedHTTPStatusCode(when: String, code: Int, details: String = "")
@@ -10,6 +9,7 @@ enum RegistryError: Error {
 }

 enum HTTPMethod: String {
+  case HEAD = "HEAD"
  case GET = "GET"
  case POST = "POST"
  case PUT = "PUT"
@@ -21,6 +21,7 @@ enum HTTPCode: Int {
  case Created = 201
  case Accepted = 202
  case Unauthorized = 401
+  case NotFound = 404
 }

 extension Data {
@@ -29,7 +30,7 @@ extension Data {
  }
 }

-extension AsyncThrowingChannel<Data, Error> {
+extension AsyncThrowingStream<Data, Error> {
  func asData() async throws -> Data {
    var result = Data()

@@ -42,9 +43,6 @@ extension AsyncThrowingChannel<Data, Error> {
 }

 struct TokenResponse: Decodable, Authentication {
-  let defaultIssuedAt = Date()
-  let defaultExpiresIn = 60
-
  var token: String?
  var accessToken: String?
  var expiresIn: Int?
@@ -66,7 +64,8 @@ struct TokenResponse: Decodable, Authentication {
      return dateFormatter.date(from: dateString) ?? Date()
    }

-    let response = try decoder.decode(TokenResponse.self, from: fromData)
+    var response = try decoder.decode(TokenResponse.self, from: fromData)
+    response.issuedAt = response.issuedAt ?? Date()

    guard response.token != nil || response.accessToken != nil else {
      throw DecodingError.keyNotFound(CodingKeys.token, .init(codingPath: [], debugDescription: "Missing token or access_token. One must be present."))
@@ -85,7 +84,7 @@ struct TokenResponse: Decodable, Authentication {
      //
      // [1]: https://docs.docker.com/registry/spec/auth/token/#requesting-a-token

-      (issuedAt ?? defaultIssuedAt) + TimeInterval(expiresIn ?? defaultExpiresIn)
+      (issuedAt ?? Date()) + TimeInterval(expiresIn ?? 60)
    }
  }

@@ -114,11 +113,11 @@ class Registry {
    return host
  }

-  init(urlComponents: URLComponents,
+  init(baseURL: URL,
       namespace: String,
       credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
  ) throws {
-    baseURL = urlComponents.url!
+    self.baseURL = baseURL
    self.namespace = namespace
    self.credentialsProviders = credentialsProviders
  }
@@ -132,7 +131,17 @@ class Registry {
    let proto = insecure ? "http" : "https"
    let baseURLComponents = URLComponents(string: proto + "://" + host + "/v2/")!

-    try self.init(urlComponents: baseURLComponents, namespace: namespace, credentialsProviders: credentialsProviders)
+    guard let baseURL = baseURLComponents.url else {
+      var hint = ""
+
+      if host.hasPrefix("http://") || host.hasPrefix("https://") {
+        hint += ", make sure that it doesn't start with http:// or https://"
+      }
+
+      throw RuntimeError.ImproperlyFormattedHost(host, hint)
+    }
+
+    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders)
  }

  func ping() async throws {
@@ -181,7 +190,7 @@ class Registry {
    return URLComponents(url: uploadLocation.absolutize(baseURL), resolvingAgainstBaseURL: true)!
  }

-  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0) async throws -> String {
+  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0, digest: String? = nil) async throws -> String {
    // Initiate a blob upload
    let (data, postResponse) = try await dataRequest(.POST, endpointURL("\(namespace)/blobs/uploads/"),
                                                     headers: ["Content-Length": "0"])
@@ -193,7 +202,7 @@ class Registry {
    // Figure out where to upload the blob
    var uploadLocation = try uploadLocationFromResponse(postResponse)

-    let digest = Digest.hash(fromData)
+    let digest = digest ?? Digest.hash(fromData)

    if chunkSizeMb == 0 {
      // monolithic upload
@@ -241,6 +250,19 @@ class Registry {
    return digest
  }

+  public func blobExists(_ digest: String) async throws -> Bool {
+    let (data, response) = try await dataRequest(.HEAD, endpointURL("\(namespace)/blobs/\(digest)"))
+
+    switch response.statusCode {
+    case HTTPCode.Ok.rawValue:
+      return true
+    case HTTPCode.NotFound.rawValue:
+      return false
+    default:
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asText())
+    }
+  }
+
  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
    if response.statusCode != HTTPCode.Ok.rawValue {
@@ -284,7 +306,7 @@ class Registry {
    body: Data? = nil,
    doAuth: Bool = true,
    viaFile: Bool = false
-  ) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  ) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
    var urlComponents = urlComponents

    if urlComponents.queryItems == nil && !parameters.isEmpty {
@@ -304,12 +326,12 @@ class Registry {
      request.httpBody = body
    }

-    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)

    if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
      _ = try await channel.asData()
      try await auth(response: response)
-      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
    }

    return (channel, response)
@@ -390,11 +412,13 @@ class Registry {
    return nil
  }

-  private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  private func authAwareRequest(request: URLRequest, viaFile: Bool = false, doAuth: Bool) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
    var request = request

-    if let (name, value) = await authenticationKeeper.header() {
-      request.addValue(value, forHTTPHeaderField: name)
+    if doAuth {
+      if let (name, value) = await authenticationKeeper.header() {
+        request.addValue(value, forHTTPHeaderField: name)
+      }
    }

    request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",
@@ -11,7 +11,7 @@ class PIDLock {
    if fd == -1 {
      let details = Errno(rawValue: CInt(errno))

-      throw RuntimeError.PIDLockFailed("failed to open lock file \(url): \(details)")
+      throw RuntimeError.PIDLockMissing("failed to open lock file \(url): \(details)")
    }
  }

@@ -38,7 +38,7 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
      }
      guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
-        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+        throw UnsupportedHostOSError()
      }
      self.hardwareModel = hardwareModel
    }
@@ -58,7 +58,11 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
      VZMacOSBootLoader()
    }

-    func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
+    func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+      if needsNestedVirtualization {
+        throw RuntimeError.VMConfigurationError("macOS virtual machines do not support nested virtualization")
+      }
+
      let result = VZMacPlatformConfiguration()

      result.machineIdentifier = ecid
@@ -14,8 +14,12 @@ struct Linux: Platform {
    return result
  }

-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
-    VZGenericPlatformConfiguration()
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+    let config = VZGenericPlatformConfiguration()
+    if #available(macOS 15, *) {
+      config.isNestedVirtualizationEnabled = needsNestedVirtualization
+    }
+    return config
  }

  func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
@@ -3,7 +3,7 @@ import Virtualization
 protocol Platform: Codable {
  func os() -> OS
  func bootLoader(nvramURL: URL) throws -> VZBootLoader
-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration
  func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
  func keyboards() -> [VZKeyboardConfiguration]
  func pointingDevices() -> [VZPointingDeviceConfiguration]
@@ -50,6 +50,10 @@ struct Root: AsyncParsableCommand {
    }
    defer { SentrySDK.flush(timeout: 2.seconds.timeInterval) }

+    SentrySDK.configureScope { scope in
+      scope.setExtra(value: ProcessInfo.processInfo.arguments, key: "Command-line arguments")
+    }
+
    // Enrich future events with Cirrus CI-specific tags
    if let tags = ProcessInfo.processInfo.environment["CIRRUS_SENTRY_TAGS"] {
      SentrySDK.configureScope { scope in
@@ -0,0 +1,28 @@
+import Foundation
+
+fileprivate func normalizeName(_ name: String) -> String {
+  // Colons are misinterpreted by Zsh completion
+  return name.replacingOccurrences(of: ":", with: "\\:")
+}
+
+func completeMachines(_ arguments: [String]) -> [String] {
+  let localVMs = (try? VMStorageLocal().list().map { name, _ in
+    normalizeName(name)
+  }) ?? []
+  let ociVMs = (try? VMStorageOCI().list().map { name, _, _ in
+    normalizeName(name)
+  }) ?? []
+  return (localVMs + ociVMs)
+}
+
+func completeLocalMachines(_ arguments: [String]) -> [String] {
+  let localVMs = (try? VMStorageLocal().list()) ?? []
+  return localVMs.map { name, _ in normalizeName(name) }
+}
+
+func completeRunningMachines(_ arguments: [String]) -> [String] {
+  let localVMs = (try? VMStorageLocal().list()) ?? []
+  return localVMs
+    .filter { _, vmDir in (try? vmDir.state() == .Running) ?? false}
+    .map { name, _ in normalizeName(name) }
+}
@@ -1,4 +1,5 @@
 import Foundation
+import System

 extension URL {
  func accessDate() throws -> Date {
@@ -13,7 +14,9 @@ extension URL {
    let times = [accessDate.asTimeval(), modificationDate.asTimeval()]
    let ret = utimes(path, times)
    if ret != 0 {
-      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(ret.explanation())")
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(details)")
    }
  }
 }
@@ -1,4 +1,5 @@
 import Foundation
+import XAttr

 extension URL: Prunable {
  var url: URL {
@@ -13,7 +14,31 @@ extension URL: Prunable {
    try resourceValues(forKeys: [.totalFileAllocatedSizeKey]).totalFileAllocatedSize!
  }

+  func deduplicatedSizeBytes() throws -> Int {
+    let values = try resourceValues(forKeys: [.totalFileAllocatedSizeKey, .mayShareFileContentKey])
+    // make sure the file's origin file is there and duplication works
+    if values.mayShareFileContent == true {
+      return Int(deduplicatedBytes())
+    }
+    return 0
+  }
+
  func sizeBytes() throws -> Int {
    try resourceValues(forKeys: [.totalFileSizeKey]).totalFileSize!
  }
+
+  func setDeduplicatedBytes(_ size: UInt64) {
+    let data = "\(size)".data(using: .utf8)!
+    try! self.setExtendedAttribute(name: "run.tart.deduplicated-bytes", value: data)
+  }
+
+  func deduplicatedBytes() -> UInt64 {
+    guard let data = try? self.extendedAttributeValue(forName: "run.tart.deduplicated-bytes") else {
+      return 0
+    }
+    if let strValue = String(data: data, encoding: .utf8) {
+      return UInt64(strValue) ?? 0
+    }
+    return 0
+  }
 }
@@ -1,6 +1,6 @@
 import Foundation
 import Virtualization
-import AsyncAlgorithms
+import Semaphore

 struct UnsupportedRestoreImageError: Error {
 }
@@ -30,7 +30,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
  var configuration: VZVirtualMachineConfiguration

  // Semaphore used to communicate with the VZVirtualMachineDelegate
-  var sema = DispatchSemaphore(value: 0)
+  var sema = AsyncSemaphore(value: 0)

  // VM's config
  var name: String
@@ -46,7 +46,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
       serialPorts: [VZSerialPortConfiguration] = [],
       suspendable: Bool = false,
-       audio: Bool = true
+       nested: Bool = false,
+       audio: Bool = true,
+       clipboard: Bool = true,
+       sync: VZDiskImageSynchronizationMode = .full,
+       caching: VZDiskImageCachingMode? = nil
  ) throws {
    name = vmDir.name
    config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -63,7 +67,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
                                                directorySharingDevices: directorySharingDevices,
                                                serialPorts: serialPorts,
                                                suspendable: suspendable,
-                                                audio: audio
+                                                nested: nested,
+                                                audio: audio,
+                                                clipboard: clipboard,
+                                                sync: sync,
+                                                caching: caching
    )
    virtualMachine = VZVirtualMachine(configuration: configuration)

@@ -91,12 +99,17 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    // Download the IPSW
    defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")

-    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+    let downloadProgress = Progress(totalUnitCount: 100)
+    ProgressObserver(downloadProgress).log(defaultLogger)

-    let progress = Progress(totalUnitCount: response.expectedContentLength)
-    ProgressObserver(progress).log(defaultLogger)
+    let request = URLRequest(url: remoteURL)
+    let (channel, response) = try await Fetcher.fetch(request, viaFile: true, progress: downloadProgress)

    let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".ipsw")
+    defaultLogger.appendNewLine("Computing digest for \(temporaryLocation.path)...")
+    let digestProgress = Progress(totalUnitCount: response.expectedContentLength)
+    ProgressObserver(digestProgress).log(defaultLogger)
+
    FileManager.default.createFile(atPath: temporaryLocation.path, contents: nil)
    let lock = try FileLock(lockURL: temporaryLocation)
    try lock.lock()
@@ -108,7 +121,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
      let chunkAsData = Data(chunk)
      fileHandle.write(chunkAsData)
      digest.update(chunkAsData)
-      progress.completedUnitCount += Int64(chunk.count)
+      digestProgress.completedUnitCount += Int64(chunk.count)
    }

    try fileHandle.close()
@@ -191,18 +204,24 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
      virtualMachine.delegate = self

      // Run automated installation
-      try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
-        DispatchQueue.main.async { [ipswURL] in
-          let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: ipswURL)
+      try await install(ipswURL)
+    }

-          defaultLogger.appendNewLine("Installing OS...")
-          ProgressObserver(installer.progress).log(defaultLogger)
+    @MainActor
+    private func install(_ url: URL) async throws {
+      let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: url)
+      defaultLogger.appendNewLine("Installing OS...")
+      ProgressObserver(installer.progress).log(defaultLogger)

+      try await withTaskCancellationHandler(operation: {
+        try await withCheckedThrowingContinuation { continuation in
          installer.install { result in
            continuation.resume(with: result)
          }
        }
-      }
+      }, onCancel: {
+        installer.progress.cancel()
+      })
    }
  #endif

@@ -232,16 +251,18 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
  }

  func run() async throws {
-    await withTaskCancellationHandler(operation: {
-      // Wait for the VM to finish running
-      // or for the exit condition
-      sema.wait()
-    }, onCancel: {
-      sema.signal()
-    })
+    do {
+      try await sema.waitUnlessCancelled()
+    } catch is CancellationError {
+      // Triggered by "tart stop", Ctrl+C, or closing the
+      // VM window, so shut down the VM gracefully below.
+    }

    if Task.isCancelled {
-      try await stop()
+      if (self.virtualMachine.state == VZVirtualMachine.State.running) {
+        print("Stopping VM...")
+        try await stop()
+      }
    }

    try await network.stop()
@@ -277,7 +298,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
    serialPorts: [VZSerialPortConfiguration],
    suspendable: Bool = false,
-    audio: Bool = true
+    nested: Bool = false,
+    audio: Bool = true,
+    clipboard: Bool = true,
+    sync: VZDiskImageSynchronizationMode = .full,
+    caching: VZDiskImageCachingMode? = nil
  ) throws -> VZVirtualMachineConfiguration {
    let configuration = VZVirtualMachineConfiguration()

@@ -289,22 +314,29 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    configuration.memorySize = vmConfig.memorySize

    // Platform
-    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL)
+    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL, needsNestedVirtualization: nested)

    // Display
    configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]

    // Audio
+    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+
    if audio && !suspendable {
-      let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+
+      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
      outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+
      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
-      configuration.audioDevices = [soundDeviceConfiguration]
+    } else {
+      // just a null speaker
+      soundDeviceConfiguration.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()]
    }

+    configuration.audioDevices = [soundDeviceConfiguration]
+
    // Keyboard and mouse
    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
@@ -322,19 +354,28 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
      return vio
    }

-    // Storage
-    let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
-      // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: .full) :
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
-
-    var device: VZStorageDeviceConfiguration
-    if #available(macOS 14, *), vmConfig.os == .linux {
-      device = VZNVMExpressControllerDeviceConfiguration(attachment: attachment)
-    } else {
-      device = VZVirtioBlockDeviceConfiguration(attachment: attachment)
+    // Clipboard sharing via Spice agent
+    if clipboard && vmConfig.os == .linux {
+      let spiceAgentConsoleDevice = VZVirtioConsoleDeviceConfiguration()
+      let spiceAgentPort = VZVirtioConsolePortConfiguration()
+      spiceAgentPort.name = VZSpiceAgentPortAttachment.spiceAgentPortName
+      spiceAgentPort.attachment = VZSpiceAgentPortAttachment()
+      spiceAgentConsoleDevice.ports[0] = spiceAgentPort
+      configuration.consoleDevices.append(spiceAgentConsoleDevice)
    }
-    var devices: [VZStorageDeviceConfiguration] = [device]
+
+    // Storage
+    let attachment: VZDiskImageStorageDeviceAttachment =  try VZDiskImageStorageDeviceAttachment(
+      url: diskURL,
+      readOnly: false,
+      // When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1]
+      //
+      // [1]: https://github.com/cirruslabs/tart/pull/675
+      cachingMode: caching ?? (vmConfig.os == .linux ? .cached : .automatic),
+      synchronizationMode: sync
+    )
+
+    var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)]
    devices.append(contentsOf: additionalStorageDevices)
    configuration.storageDevices = devices

@@ -24,6 +24,7 @@ enum CodingKeys: String, CodingKey {
  case memorySize
  case macAddress
  case display
+  case displayRefit

  // macOS-specific keys
  case ecid
@@ -52,6 +53,7 @@ struct VMConfig: Codable {
  private(set) var memorySize: UInt64
  var macAddress: VZMACAddress
  var display: VMDisplayConfig = VMDisplayConfig()
+  var displayRefit: Bool?

  init(
    platform: Platform,
@@ -121,6 +123,7 @@ struct VMConfig: Codable {
    self.macAddress = macAddress

    display = try container.decodeIfPresent(VMDisplayConfig.self, forKey: .display) ?? VMDisplayConfig()
+    displayRefit = try container.decodeIfPresent(Bool.self, forKey: .displayRefit)
  }

  func encode(to encoder: Encoder) throws {
@@ -136,6 +139,9 @@ struct VMConfig: Codable {
    try container.encode(memorySize, forKey: .memorySize)
    try container.encode(macAddress.string, forKey: .macAddress)
    try container.encode(display, forKey: .display)
+    if let displayRefit = displayRefit {
+      try container.encode(displayRefit, forKey: .displayRefit)
+    }
  }

  mutating func setCPU(cpuCount: Int) throws {
@@ -1,3 +1,4 @@
+import Compression
 import Foundation
 import Sentry

@@ -10,18 +11,7 @@ enum OCIError: Error {
 }

 extension VMDirectory {
-  private static let bufferSizeBytes = 64 * 1024 * 1024
-  private static let layerLimitBytes = 500 * 1000 * 1000
-
-  func pullFromRegistry(registry: Registry, reference: String, concurrency: UInt) async throws {
-    defaultLogger.appendNewLine("pulling manifest...")
-
-    let (manifest, _) = try await registry.pullManifest(reference: reference)
-
-    return try await pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
-  }
-
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
    // Pull VM's config file layer and re-serialize it into a config file
    let configLayers = manifest.layers.filter {
      $0.mediaType == configMediaType
@@ -61,7 +51,19 @@ extension VMDirectory {
    let progress = Progress(totalUnitCount: diskCompressedSize)
    ProgressObserver(progress).log(defaultLogger)

-    try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL, concurrency: concurrency, progress: progress)
+    do {
+      try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
+                                  concurrency: concurrency, progress: progress,
+                                  localLayerCache: localLayerCache,
+                                  deduplicate: deduplicate)
+    } catch let error where error is FilterError {
+      throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
+    }
+
+    if deduplicate, let llc = localLayerCache {
+      // set custom attribute to remember deduplicated bytes
+      diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
+    }

    // Pull VM's NVRAM file layer and store it in an NVRAM file
    defaultLogger.appendNewLine("pulling NVRAM...")
@@ -80,9 +82,12 @@ extension VMDirectory {
      nvram.write(data)
    }
    try nvram.close()
+
+    // Serialize VM's manifest to enable better deduplication on subsequent "tart pull"'s
+    try manifest.toJSON().write(to: manifestURL)
  }

-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String, concurrency: UInt) async throws -> RemoteName {
    var layers = Array<OCIManifestLayer>()

    // Read VM's config and push it as blob
@@ -101,9 +106,9 @@ extension VMDirectory {

    switch diskFormat {
    case "v1":
-      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
    case "v2":
-      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
    default:
      throw RuntimeError.OCIUnsupportedDiskFormat(diskFormat)
    }
@@ -3,6 +3,12 @@ import Virtualization
 import CryptoKit

 struct VMDirectory: Prunable {
+  enum State: String {
+    case Running = "running"
+    case Suspended = "suspended"
+    case Stopped = "stopped"
+  }
+
  var baseURL: URL

  var configURL: URL {
@@ -17,6 +23,9 @@ struct VMDirectory: Prunable {
  var stateURL: URL {
    baseURL.appendingPathComponent("state.vzvmsave")
  }
+  var manifestURL: URL {
+    baseURL.appendingPathComponent("manifest.json")
+  }

  var explicitlyPulledMark: URL {
    baseURL.appendingPathComponent(".explicitly-pulled")
@@ -47,13 +56,13 @@ struct VMDirectory: Prunable {
    return try lock.pid() != 0
  }

-  func state() throws -> String {
+  func state() throws -> State {
    if try running() {
-      return "running"
+      return State.Running
    } else if FileManager.default.fileExists(atPath: stateURL.path) {
-      return "suspended"
+      return State.Suspended
    } else {
-      return "stopped"
+      return State.Stopped
    }
  }

@@ -173,6 +182,14 @@ struct VMDirectory: Prunable {
    try allocatedSizeBytes() / 1000 / 1000 / 1000
  }

+  func deduplicatedSizeBytes() throws -> Int {
+    try configURL.deduplicatedSizeBytes() + diskURL.deduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes()
+  }
+
+  func deduplicatedSizeGB() throws -> Int {
+    try deduplicatedSizeBytes() / 1000 / 1000 / 1000
+  }
+
  func sizeBytes() throws -> Int {
    try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes()
  }
@@ -24,6 +24,8 @@ class VMStorageHelper {
  private static func missingVMWrap<R: Any>(_ name: String, closure: () throws -> R) throws -> R {
    do {
      return try closure()
+    } catch RuntimeError.PIDLockMissing {
+      throw RuntimeError.VMDoesNotExist(name: name)
    } catch {
      if error.isFileNotFound() {
        throw RuntimeError.VMDoesNotExist(name: name)
@@ -55,11 +57,14 @@ enum RuntimeError : Error {
  case VMAlreadyRunning(_ message: String)
  case NoIPAddressFound(_ message: String)
  case DiskAlreadyInUse(_ message: String)
+  case FailedToOpenBlockDevice(_ path: String, _ explanation: String)
  case InvalidDiskSize(_ message: String)
  case FailedToUpdateAccessDate(_ message: String)
  case PIDLockFailed(_ message: String)
+  case PIDLockMissing(_ message: String)
  case FailedToParseRemoteName(_ message: String)
  case VMTerminationFailed(_ message: String)
+  case ImproperlyFormattedHost(_ host: String, _ hint: String)
  case InvalidCredentials(_ message: String)
  case VMDirectoryAlreadyInitialized(_ message: String)
  case ExportFailed(_ message: String)
@@ -68,6 +73,8 @@ enum RuntimeError : Error {
  case OCIStorageError(_ message: String)
  case OCIUnsupportedDiskFormat(_ format: String)
  case SuspendFailed(_ message: String)
+  case PullFailed(_ message: String)
+  case VirtualMachineLimitExceeded(_ hint: String)
 }

 protocol HasExitCode {
@@ -93,16 +100,22 @@ extension RuntimeError : CustomStringConvertible {
      return message
    case .DiskAlreadyInUse(let message):
      return message
+    case .FailedToOpenBlockDevice(let path, let explanation):
+      return "failed to open block device \(path): \(explanation)"
    case .InvalidDiskSize(let message):
      return message
    case .FailedToUpdateAccessDate(let message):
      return message
    case .PIDLockFailed(let message):
      return message
+    case .PIDLockMissing(let message):
+      return message
    case .FailedToParseRemoteName(let cause):
      return "failed to parse remote name: \(cause)"
    case .VMTerminationFailed(let message):
      return message
+    case .ImproperlyFormattedHost(let host, let hint):
+      return "improperly formatted host \"\(host)\" was provided\(hint)"
    case .InvalidCredentials(let message):
      return message
    case .VMDirectoryAlreadyInitialized(let message):
@@ -119,6 +132,10 @@ extension RuntimeError : CustomStringConvertible {
      return "OCI disk format \(format) is not supported by this version of Tart"
    case .SuspendFailed(let message):
      return "Failed to suspend the VM: \(message)"
+    case .PullFailed(let message):
+      return message
+    case .VirtualMachineLimitExceeded(let hint):
+      return "The number of VMs exceeds the system limit\(hint)"
    }
  }
 }
@@ -66,7 +66,7 @@ class VMStorageLocal: PrunableStorage {
  }

  func prunables() throws -> [Prunable] {
-    try list().map { (_, vmDir) in vmDir }
+    try list().map { (_, vmDir) in vmDir }.filter { try !$0.running() }
  }

  func hasVMsWithMACAddress(macAddress: String) throws -> Bool {
@@ -1,5 +1,6 @@
 import Foundation
 import Sentry
+import Retry

 class VMStorageOCI: PrunableStorage {
  let baseURL = try! Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
@@ -112,6 +113,10 @@ class VMStorageOCI: PrunableStorage {
        continue
      }

+      // Split the relative VM's path at the last component
+      // and figure out which character should be used
+      // to join them together, either ":" for tags or
+      // "@" for hashes
      let parts = [foundURL.deletingLastPathComponent().relativePath, foundURL.lastPathComponent]
      var name: String

@@ -122,6 +127,9 @@ class VMStorageOCI: PrunableStorage {
        name = parts.joined(separator: "@")
      }

+      // Remove the percent-encoding, if any
+      name = percentDecode(name)
+
      result.append((name, vmDir, isSymlink))
    }

@@ -132,9 +140,9 @@ class VMStorageOCI: PrunableStorage {
    try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
  }

-  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
    SentrySDK.configureScope { scope in
-      scope.setContext(value: ["imageName": name], key: "OCI")
+      scope.setContext(value: ["imageName": name.description], key: "OCI")
    }

    defaultLogger.appendNewLine("pulling manifest...")
@@ -188,7 +196,31 @@ class VMStorageOCI: PrunableStorage {
      }

      try await withTaskCancellationHandler(operation: {
-        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
+        try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
+          // Choose the best base image which has the most deduplication ratio
+          let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
+
+          if let llc = localLayerCache {
+            let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)
+
+            if deduplicate {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+            } else {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to avoid fetching \(deduplicatedHuman), will try use it...")
+            }
+          }
+
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate)
+        } recoverFromFailure: { error in
+          if error is URLError {
+            print("Error: \(error.localizedDescription)")
+            print("Attempting to re-try...")
+
+            return .retry
+          }
+
+          return .throw
+        }
        try move(digestName, from: tmpVMDir)
        transaction.finish()
      }, onCancel: {
@@ -207,6 +239,9 @@ class VMStorageOCI: PrunableStorage {
      // are excluded from garbage collection
      VMDirectory(baseURL: vmURL(name)).markExplicitlyPulled()
    }
+
+    // to explicitly set the image as being accessed so it won't get pruned immediately
+    _ = try VMStorageOCI().open(name)
  }

  func linked(from: RemoteName, to: RemoteName) -> Bool {
@@ -225,13 +260,66 @@ class VMStorageOCI: PrunableStorage {

    try gc()
  }
+
+  func chooseLocalLayerCache(_ name: RemoteName, _ manifest: OCIManifest, _ registry: Registry) async throws -> LocalLayerCache? {
+    // Establish a closure that will calculate how much bytes
+    // we'll deduplicate if we re-use the given manifest
+    let target = Swift.Set(manifest.layers)
+
+    let calculateDeduplicatedBytes = { (manifest: OCIManifest) -> UInt64 in
+      target.intersection(manifest.layers).map({ UInt64($0.size) }).reduce(0, +)
+    }
+
+    // Load OCI VM images and their manifests (if present)
+    var candidates: [(name: String, vmDir: VMDirectory, manifest: OCIManifest, deduplicatedBytes: UInt64)] = []
+
+    for (name, vmDir, isSymlink) in try list() {
+      if isSymlink {
+        continue
+      }
+
+      guard let manifestJSON = try? Data(contentsOf: vmDir.manifestURL) else {
+        continue
+      }
+
+      guard let manifest = try? OCIManifest(fromJSON: manifestJSON) else {
+        continue
+      }
+
+      candidates.append((name, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
+    }
+
+    // Previously we haven't stored the OCI VM image manifests, but still fetched the VM image manifest if
+    // what the user was trying to pull was a tagged image, and we already had that image in the OCI VM cache
+    //
+    // Keep supporting this behavior for backwards comaptibility, but only communicate
+    // with the registry if we haven't already retrieved the manifest for that OCI VM image.
+    if name.reference.type == .Tag,
+       let vmDir = try? open(name),
+       let digest = try? digest(name),
+       try !candidates.contains(where: {try $0.manifest.digest() == digest}),
+       let (manifest, _) = try? await registry.pullManifest(reference: digest) {
+      candidates.append((name.description, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
+    }
+
+    // Now, find the best match based on how many bytes we'll deduplicate
+    let choosen = candidates.filter {
+      $0.deduplicatedBytes > 1024 * 1024 * 1024 // save at least 1GB
+    }.max { left, right in
+      return left.deduplicatedBytes < right.deduplicatedBytes
+    }
+
+    return try choosen.flatMap({ choosen in
+      try LocalLayerCache(choosen.name, choosen.deduplicatedBytes, choosen.vmDir.diskURL, choosen.manifest)
+    })
+  }
 }

 extension URL {
  func appendingRemoteName(_ name: RemoteName) -> URL {
    var result: URL = self

-    for pathComponent in (name.host + "/" + name.namespace + "/" + name.reference.value).split(separator: "/") {
+    for pathComponent in (percentEncode(name.host) + "/" + name.namespace + "/" + name.reference.value).split(separator: "/") {
      result = result.appendingPathComponent(String(pathComponent))
    }

@@ -239,6 +327,23 @@ extension URL {
  }

  func appendingHost(_ name: RemoteName) -> URL {
-    self.appendingPathComponent(name.host, isDirectory: true)
+    self.appendingPathComponent(percentEncode(name.host), isDirectory: true)
  }
 }
+
+// Work around a pretty inane Swift's URL behavior where calling
+// appendingPathComponent() or deletingLastPathComponent() on a
+// URL like URL(filePath: "example.com:8080") (note the "filePath")
+// will flip its isFileURL from "true" to "false" and discard its
+// absolute path infromation (if any).
+//
+// The same kind of operations won't do anything to a URL like
+// URL(filePath: "127.0.0.1:8080"), which makes things even more
+// ridiculous.
+private func percentEncode(_ s: String) -> String {
+  return s.addingPercentEncoding(withAllowedCharacters: CharacterSet(charactersIn: ":").inverted)!
+}
+
+private func percentDecode(_ s: String) -> String {
+  s.removingPercentEncoding!
+}
@@ -15,7 +15,7 @@ class FullFledgedVNC: VNC {
    vnc.start()
  }

-  func waitForURL() async throws -> URL {
+  func waitForURL(netBridged: Bool) async throws -> URL {
    while true {
      // Port is 0 shortly after start(),
      // but will be initialized later
@@ -9,9 +9,9 @@ class ScreenSharingVNC: VNC {
    self.vmConfig = vmConfig
  }

-  func waitForURL() async throws -> URL {
+  func waitForURL(netBridged: Bool) async throws -> URL {
    let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!
-    let ip = try await IP.resolveIP(vmMACAddress, secondsToWait: 60)
+    let ip = try await IP.resolveIP(vmMACAddress, resolutionStrategy: netBridged ? .arp : .dhcp, secondsToWait: 60)

    if let ip = ip {
      return URL(string: "vnc://\(ip)")!
@@ -1,6 +1,6 @@
 import Foundation

 protocol VNC {
-  func waitForURL() async throws -> URL
+  func waitForURL(netBridged: Bool) async throws -> URL
  func stop() throws
 }
@@ -36,7 +36,7 @@ final class LayerizerTests: XCTestCase {
    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)

    print("pushing disk...")
-    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())

    print("pulling disk...")
    try await DiskV1.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
@@ -57,7 +57,7 @@ final class LayerizerTests: XCTestCase {
    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)

    print("pushing disk...")
-    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())

    print("pulling disk...")
    try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
@@ -3,24 +3,26 @@ import XCTest

 final class TokenResponseTests: XCTestCase {
  func testBasic() throws {
+    var expectedTokenExpiresAtRange = DateInterval()
    let tokenResponseRaw = Data("{\"token\":\"some token\"}".utf8)
    let tokenResponse = try TokenResponse.parse(fromData: tokenResponseRaw)

    XCTAssertEqual(tokenResponse.token, "some token")

-    let expectedTokenExpiresAtRange = Date()...Date().addingTimeInterval(60)
+    expectedTokenExpiresAtRange.end = Date().addingTimeInterval(60)
    XCTAssertTrue(expectedTokenExpiresAtRange.contains(tokenResponse.tokenExpiresAt))

    XCTAssertTrue(tokenResponse.isValid())
  }

  func testExpirationBasic() throws {
+    var expectedTokenExpiresAtRange = DateInterval()
    let tokenResponseRaw = Data("{\"token\":\"some token\",\"expires_in\":2}".utf8)
    let tokenResponse = try TokenResponse.parse(fromData: tokenResponseRaw)

    XCTAssertEqual(tokenResponse.expiresIn, 2)

-    let expectedTokenExpiresAtRange = Date()...Date().addingTimeInterval(2)
+    expectedTokenExpiresAtRange.end = Date().addingTimeInterval(2)
    XCTAssertTrue(expectedTokenExpiresAtRange.contains(tokenResponse.tokenExpiresAt))

    XCTAssertTrue(tokenResponse.isValid())
@@ -31,7 +31,7 @@ class RegistryRunner {

  init() async throws {
    // Start container
-    let container = try Self.dockerCmd("run", "-d", "--rm", "-p", "5000", "registry:2")
+    let container = try Self.dockerCmd("run", "-d", "--rm", "-p", "127.0.0.1:0:5000", "registry:2")
      .trimmingCharacters(in: CharacterSet.newlines)
    containerID = container

@@ -39,7 +39,7 @@ class RegistryRunner {
    let port = try Self.dockerCmd("inspect", containerID, "--format", "{{(index (index .NetworkSettings.Ports \"5000/tcp\") 0).HostPort}}")
      .trimmingCharacters(in: CharacterSet.newlines)

-    registry = try Registry(urlComponents: URLComponents(string: "http://127.0.0.1:\(port)/v2/")!,
+    registry = try Registry(baseURL: URL(string: "http://127.0.0.1:\(port)/v2/")!,
                            namespace: "vm-image")

    // Wait for the Docker Registry to start
@@ -0,0 +1 @@
+root = true
@@ -0,0 +1,92 @@
+run:
+  timeout: 5m
+
+linters:
+  enable-all: true
+
+  disable:
+    # Messages like "struct of size 104 bytes could be of size 96 bytes" from a package
+    # that was last updated 2 years ago[1] are barely helpful.
+    #
+    # After all, we're writing the code for other people, so let's trust the compiler here (that's
+    # constantly evolving compared to this linter) and revisit this if memory usage becomes a problem.
+    #
+    # [1]: https://github.com/mdempsky/maligned/commit/6e39bd26a8c8b58c5a22129593044655a9e25959
+    - maligned
+
+    # We don't have high-performance requirements at this moment, so sacrificing
+    # the code readability for marginal performance gains is not worth it.
+    - prealloc
+
+    # New linters that require a lot of codebase churn and noise, but perhaps we can enable them in the future.
+    - nlreturn
+    - wrapcheck
+    - errorlint
+
+    # Unfortunately, we use globals due to how spf13/cobra works.
+    - gochecknoglobals
+
+    # That's fine that some Proto objects don't have all fields initialized
+    - exhaustivestruct
+
+    # Style linters that are total nuts.
+    - wsl
+    - gofumpt
+    - goimports
+    - funlen
+
+    # This conflicts with the Protocol Buffers Version 3 design,
+    # which is largely based on default values for struct fields.
+    - exhaustivestruct
+
+    # Enough parallelism for now.
+    - paralleltest
+
+    # Ill-based assumptions about identifiers like fmt.Println without taking context into account.
+    - forbidigo
+
+    # Advantages of using t.Helper() are too small to waste developer's cognitive stamina on it.
+    - thelper
+
+    # Too restrictive defaults, plus there's already a gocyclo linter in place.
+    - cyclop
+
+    # Gives false positives for textbook examples[1][2]
+    # [1]: https://github.com/charithe/durationcheck/issues/7
+    # [2]: https://golang.org/pkg/time/ (see "To convert an integer number of units to a Duration, multiply:")
+    - durationcheck
+
+    # No way to disable the "exported" check for the whole project[1]
+    # [1]: https://github.com/mgechev/revive/issues/244#issuecomment-560512162
+    - revive
+
+    # Unfortunately too much false-positives, e.g. for a 0700 umask or number 10 when using strconv.FormatInt()
+    - gomnd
+
+    # Needs package whitelists
+    - depguard
+
+    # Generates absolutely useless errors, e.g.
+    # "string `.yml` has 3 occurrences, make it a constant"
+    - goconst
+
+    # It's OK to not sort imports
+    - gci
+
+    # It's OK to not initialize some struct fields
+    - exhaustruct
+
+    # This is not a library, so it's OK to use dynamic errors
+    - goerr113
+
+    # fmt.Sprintf() looks a bit nicer than string addition
+    - perfsprint
+
+    # We can control this ourselves
+    - varnamelen
+    - contextcheck
+
+issues:
+  # Don't hide multiple issues that belong to one class since GitHub annotations can handle them all nicely.
+  max-issues-per-linter: 0
+  max-same-issues: 0
@@ -0,0 +1,187 @@
+# Benchmark
+
+Tart comes with a Golang-based benchmarking utility that allows one to easily compare host and guest performance.
+
+Currently, only Flexible I/O tester workloads are supported. To run them, first make sure that [passwordless sudo](https://serverfault.com/questions/160581/how-to-setup-passwordless-sudo-on-linux) is configured.
+
+Then, [install Golang](https://go.dev/). The easiest way is through [Homebrew](https://brew.sh/):
+
+```shell
+brew install go
+```
+
+Finally, run the following command from this (`benchmark/`) directory:
+
+```shell
+go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sonoma-base:latest --prepare 'sudo purge && sync'
+```
+
+You can also enable the debugging output to diagnose issues:
+
+```shell
+go run cmd/main.go fio --debug
+```
+
+## Results
+
+### Mar 27, 2024
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sonoma 14.4.1
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.4.1
+
+```
+Name                    	Executor	Bandwidth	I/O operations
+Random writing of 1MB   	local   	2.6 GB/s 	649.35 kIOPS  
+Random writing of 1MB   	Tart    	2.5 GB/s 	620.22 kIOPS  
+Random writing of 10MB  	local   	2.6 GB/s 	651.74 kIOPS  
+Random writing of 10MB  	Tart    	2.5 GB/s 	615.52 kIOPS  
+Random writing of 100MB 	local   	1.9 GB/s 	481.51 kIOPS  
+Random writing of 100MB 	Tart    	2.0 GB/s 	493.31 kIOPS  
+Random writing of 1000MB	local   	1.7 GB/s 	414.89 kIOPS  
+Random writing of 1000MB	Tart    	1.1 GB/s 	287.4 kIOPS  
+```
+
+### Dec 2, 2024
+
+Host:
+
+* Hardware: MacBook Pro (Apple M1 Pro, 8 performance and 2 efficiency cores, 32 GB RAM, `MacBookPro18,3`)
+* OS: macOS Sequoia 15.1.1
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)          	Latency (write)         	Latency (sync)      
+Single 4KiB random write process        	local                                             	0 B/s     	19 MB/s    	0 IOPS     	4.81 kIOPS  	0s ± 0s                 	203.418µs ± 155.865µs   	0s ± 0s             
+Single 4KiB random write process        	Tart                                              	0 B/s     	18 MB/s    	0 IOPS     	4.54 kIOPS  	0s ± 0s                 	213.655µs ± 188.822µs   	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	19 MB/s    	0 IOPS     	4.68 kIOPS  	0s ± 0s                 	208.413µs ± 183.45µs    	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	24 MB/s    	0 IOPS     	6.11 kIOPS  	0s ± 0s                 	158.07µs ± 2.294654ms   	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	22 MB/s    	0 IOPS     	5.49 kIOPS  	0s ± 0s                 	173.414µs ± 310.213µs   	0s ± 0s             
+16 parallel 64KiB random write processes	local                                             	0 B/s     	18 GB/s    	0 IOPS     	273.76 kIOPS	0s ± 0s                 	323.423µs ± 604.999µs   	0s ± 0s             
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	16 GB/s    	0 IOPS     	273.48 kIOPS	0s ± 0s                 	335.086µs ± 7.591748ms  	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	18 GB/s    	0 IOPS     	281.49 kIOPS	0s ± 0s                 	326.655µs ± 7.485473ms  	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	17 GB/s    	0 IOPS     	266.79 kIOPS	0s ± 0s                 	340µs ± 7.868384ms      	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	16 GB/s    	0 IOPS     	251.02 kIOPS	0s ± 0s                 	355.077µs ± 8.354218ms  	0s ± 0s             
+Single 1MiB random write process        	local                                             	0 B/s     	1.3 GB/s   	0 IOPS     	1.31 kIOPS  	0s ± 0s                 	751.716µs ± 370.731µs   	0s ± 0s             
+Single 1MiB random write process        	Tart                                              	0 B/s     	1.1 GB/s   	0 IOPS     	1.1 kIOPS   	0s ± 0s                 	885.833µs ± 3.572539ms  	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.1 GB/s   	0 IOPS     	1.08 kIOPS  	0s ± 0s                 	898.427µs ± 3.464261ms  	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	1000 MB/s  	0 IOPS     	976.47 IOPS 	0s ± 0s                 	972.491µs ± 6.87654ms   	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	1.1 GB/s   	0 IOPS     	1.03 kIOPS  	0s ± 0s                 	925.545µs ± 4.261693ms  	0s ± 0s             
+Random reads/writes (4k)                	local                                             	62 MB/s   	62 MB/s    	15.37 kIOPS	15.37 kIOPS 	2.059453ms ± 1.431822ms 	2.098761ms ± 1.445082ms 	0s ± 0s             
+Random reads/writes (4k)                	Tart                                              	38 MB/s   	38 MB/s    	9.6 kIOPS  	9.61 kIOPS  	3.30369ms ± 1.500464ms  	3.350589ms ± 1.512986ms 	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	39 MB/s   	39 MB/s    	9.82 kIOPS 	9.83 kIOPS  	3.228106ms ± 1.367512ms 	3.27626ms ± 1.385964ms  	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	35 MB/s   	35 MB/s    	8.74 kIOPS 	8.76 kIOPS  	3.640772ms ± 15.472355ms	3.661779ms ± 15.264288ms	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	24 MB/s   	24 MB/s    	5.98 kIOPS 	5.99 kIOPS  	5.31188ms ± 4.55205ms   	5.375047ms ± 5.113847ms 	0s ± 0s             
+Random reads/writes (64k)               	local                                             	435 MB/s  	436 MB/s   	6.79 kIOPS 	6.8 kIOPS   	4.955892ms ± 2.066685ms 	4.440414ms ± 1.860036ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart                                              	352 MB/s  	353 MB/s   	5.5 kIOPS  	5.51 kIOPS  	5.946067ms ± 2.041124ms 	5.658948ms ± 1.928372ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	331 MB/s  	332 MB/s   	5.16 kIOPS 	5.17 kIOPS  	6.330765ms ± 1.726782ms 	6.033862ms ± 1.671028ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	428 MB/s  	428 MB/s   	6.68 kIOPS 	6.69 kIOPS  	4.661666ms ± 18.342779ms	4.904961ms ± 18.396772ms	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	297 MB/s  	298 MB/s   	4.64 kIOPS 	4.65 kIOPS  	6.591009ms ± 2.827053ms 	7.166883ms ± 3.001036ms 	0s ± 0s             
+sync test                               	local                                             	0 B/s     	48 MB/s    	0 IOPS     	21.15 kIOPS 	0s ± 0s                 	23.471µs ± 81.868µs     	23.374µs ± 6.255µs  
+sync test                               	Tart                                              	0 B/s     	24 MB/s    	0 IOPS     	10.72 kIOPS 	0s ± 0s                 	24.983µs ± 61.761µs     	67.575µs ± 76.196µs 
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	21 MB/s    	0 IOPS     	9.5 kIOPS   	0s ± 0s                 	26.973µs ± 63.935µs     	77.388µs ± 47.103µs 
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	30 MB/s    	0 IOPS     	13.19 kIOPS 	0s ± 0s                 	11.923µs ± 25.225µs     	62.894µs ± 208.933µs
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	38 MB/s    	0 IOPS     	17.02 kIOPS 	0s ± 0s                 	10.124µs ± 21.868µs     	47.803µs ± 33.706µs
+```
+
+### Dec 4, 2024
+
+Host:
+
+* AWS instance: `mac2.metal` + `gp3` EBS volume
+* Hardware: Mac mini (Apple M1, 4 performance and 4 efficiency cores, 16 GB RAM, `Macmini9,1`)
+* OS: macOS Sequoia 15.0
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)            	Latency (write)           	Latency (sync)         
+Single 4KiB random write process        	local                                             	0 B/s     	4.4 MB/s   	0 IOPS     	1.1 kIOPS   	0s ± 0s                   	702.357µs ± 359.925µs     	0s ± 0s                
+Single 4KiB random write process        	Tart                                              	0 B/s     	2.6 MB/s   	0 IOPS     	656.37 IOPS 	0s ± 0s                   	1.140086ms ± 1.450472ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	2.7 MB/s   	0 IOPS     	677.07 IOPS 	0s ± 0s                   	1.179872ms ± 1.219626ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	3.3 MB/s   	0 IOPS     	832.66 IOPS 	0s ± 0s                   	948.648µs ± 94.141338ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	15 MB/s    	0 IOPS     	3.65 kIOPS  	0s ± 0s                   	260.717µs ± 19.977757ms   	0s ± 0s                
+16 parallel 64KiB random write processes	local                                             	0 B/s     	9.5 GB/s   	0 IOPS     	147.89 kIOPS	0s ± 0s                   	753.289µs ± 8.028974ms    	0s ± 0s                
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	10 GB/s    	0 IOPS     	176.96 kIOPS	0s ± 0s                   	429.83µs ± 33.792264ms    	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	12 GB/s    	0 IOPS     	180.89 kIOPS	0s ± 0s                   	383.524µs ± 17.524971ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	336 MB/s   	0 IOPS     	5.24 kIOPS  	0s ± 0s                   	9.970844ms ± 365.808663ms 	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	9.4 GB/s   	0 IOPS     	147.04 kIOPS	0s ± 0s                   	524.139µs ± 34.100009ms   	0s ± 0s                
+Single 1MiB random write process        	local                                             	0 B/s     	178 MB/s   	0 IOPS     	173.36 IOPS 	0s ± 0s                   	3.835103ms ± 2.917977ms   	0s ± 0s                
+Single 1MiB random write process        	Tart                                              	0 B/s     	140 MB/s   	0 IOPS     	136.48 IOPS 	0s ± 0s                   	4.721178ms ± 7.744965ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	144 MB/s   	0 IOPS     	140.63 IOPS 	0s ± 0s                   	4.443507ms ± 11.572454ms  	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	47 MB/s    	0 IOPS     	45.55 IOPS  	0s ± 0s                   	13.267881ms ± 358.283094ms	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	196 MB/s   	0 IOPS     	191.4 IOPS  	0s ± 0s                   	4.102516ms ± 73.117503ms  	0s ± 0s                
+Random reads/writes (4k)                	local                                             	8.7 MB/s  	8.7 MB/s   	2.16 kIOPS 	2.16 kIOPS  	193.370794ms ± 42.593607ms	222.272016ms ± 56.586971ms	0s ± 0s                
+Random reads/writes (4k)                	Tart                                              	4.1 MB/s  	4.1 MB/s   	1.02 kIOPS 	1.03 kIOPS  	31.038867ms ± 13.508668ms 	31.184305ms ± 14.032766ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	4.2 MB/s  	4.2 MB/s   	1.04 kIOPS 	1.05 kIOPS  	30.368422ms ± 13.505627ms 	30.595412ms ± 13.840944ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	2.2 MB/s  	2.2 MB/s   	545.33 IOPS	548.86 IOPS 	59.31316ms ± 716.351086ms 	57.647852ms ± 711.503882ms	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	6.0 MB/s  	6.0 MB/s   	1.5 kIOPS  	1.5 kIOPS   	21.244222ms ± 47.808399ms 	21.39459ms ± 44.716307ms  	0s ± 0s                
+Random reads/writes (64k)               	local                                             	121 MB/s  	121 MB/s   	1.89 kIOPS 	1.89 kIOPS  	61.894699ms ± 21.353345ms 	73.176462ms ± 13.02948ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart                                              	72 MB/s   	72 MB/s    	1.12 kIOPS 	1.12 kIOPS  	27.842263ms ± 15.320781ms 	29.161858ms ± 15.765314ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	71 MB/s   	72 MB/s    	1.11 kIOPS 	1.11 kIOPS  	28.009493ms ± 16.333136ms 	29.285868ms ± 16.540589ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	28 MB/s   	28 MB/s    	441.85 IOPS	444.81 IOPS 	71.726725ms ± 633.215756ms	72.597238ms ± 630.969305ms	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	81 MB/s   	81 MB/s    	1.26 kIOPS 	1.26 kIOPS  	24.872043ms ± 36.980111ms 	25.568559ms ± 37.027145ms 	0s ± 0s                
+sync test                               	local                                             	0 B/s     	1.9 MB/s   	0 IOPS     	868.08 IOPS 	0s ± 0s                   	92.08µs ± 233.598µs       	1.059033ms ± 98.751µs  
+sync test                               	Tart                                              	0 B/s     	1.5 MB/s   	0 IOPS     	649.42 IOPS 	0s ± 0s                   	146.737µs ± 434.261µs     	1.391898ms ± 699.148µs 
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.3 MB/s   	0 IOPS     	568.82 IOPS 	0s ± 0s                   	158.736µs ± 504.002µs     	1.59798ms ± 14.161331ms
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	13 MB/s    	0 IOPS     	5.77 kIOPS  	0s ± 0s                   	26.596µs ± 832.169µs      	145.785µs ± 2.864048ms 
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	19 MB/s    	0 IOPS     	8.37 kIOPS  	0s ± 0s                   	20.135µs ± 108.817µs      	98.274µs ± 239.631µs   
+```
+
+Host:
+
+* AWS instance: `mac2.metal` + `gp3` EBS volume
+* Hardware: Mac mini (Apple M1, 4 performance and 4 efficiency cores, 16 GB RAM, `Macmini9,1`)
+* OS: macOS Sequoia 15.0
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sequoia 15.1
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)            	Latency (write)           	Latency (sync)         
+Single 4KiB random write process        	local                                             	0 B/s     	4.8 MB/s   	0 IOPS     	1.19 kIOPS  	0s ± 0s                   	690.818µs ± 326.595µs     	0s ± 0s                
+Single 4KiB random write process        	Tart                                              	0 B/s     	2.8 MB/s   	0 IOPS     	700.94 IOPS 	0s ± 0s                   	1.090362ms ± 918.444µs    	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	3.0 MB/s   	0 IOPS     	746.23 IOPS 	0s ± 0s                   	1.028192ms ± 974.533µs    	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	4.2 MB/s   	0 IOPS     	1.04 kIOPS  	0s ± 0s                   	916.36µs ± 105.318323ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	14 MB/s    	0 IOPS     	3.57 kIOPS  	0s ± 0s                   	269.796µs ± 22.419599ms   	0s ± 0s                
+16 parallel 64KiB random write processes	local                                             	0 B/s     	9.5 GB/s   	0 IOPS     	148.74 kIOPS	0s ± 0s                   	753.46µs ± 8.06509ms      	0s ± 0s                
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	5.2 GB/s   	0 IOPS     	81.46 kIOPS 	0s ± 0s                   	778.624µs ± 11.705178ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	5.3 GB/s   	0 IOPS     	83.47 kIOPS 	0s ± 0s                   	865.448µs ± 38.369176ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	116 MB/s   	0 IOPS     	1.8 kIOPS   	0s ± 0s                   	37.601112ms ± 727.319309ms	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	5.3 GB/s   	0 IOPS     	83.19 kIOPS 	0s ± 0s                   	900.751µs ± 51.223205ms   	0s ± 0s                
+Single 1MiB random write process        	local                                             	0 B/s     	177 MB/s   	0 IOPS     	173.27 IOPS 	0s ± 0s                   	3.833194ms ± 2.873871ms   	0s ± 0s                
+Single 1MiB random write process        	Tart                                              	0 B/s     	151 MB/s   	0 IOPS     	147.44 IOPS 	0s ± 0s                   	4.925853ms ± 7.793808ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	151 MB/s   	0 IOPS     	147.87 IOPS 	0s ± 0s                   	4.884797ms ± 7.563512ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	72 MB/s    	0 IOPS     	69.9 IOPS   	0s ± 0s                   	8.909771ms ± 214.311644ms 	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	159 MB/s   	0 IOPS     	155.69 IOPS 	0s ± 0s                   	4.863448ms ± 88.965211ms  	0s ± 0s                
+Random reads/writes (4k)                	local                                             	8.7 MB/s  	8.7 MB/s   	2.16 kIOPS 	2.16 kIOPS  	193.353233ms ± 42.728494ms	222.325905ms ± 56.901372ms	0s ± 0s                
+Random reads/writes (4k)                	Tart                                              	3.5 MB/s  	3.5 MB/s   	862.89 IOPS	865.54 IOPS 	36.893229ms ± 12.644216ms 	37.143334ms ± 12.772017ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	3.6 MB/s  	3.6 MB/s   	907.4 IOPS 	911.55 IOPS 	35.048969ms ± 10.559354ms 	35.3046ms ± 10.67824ms    	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	2.7 MB/s  	2.8 MB/s   	684.11 IOPS	688.05 IOPS 	48.815322ms ± 687.806727ms	44.395556ms ± 635.532064ms	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	7.0 MB/s  	7.0 MB/s   	1.74 kIOPS 	1.74 kIOPS  	18.00448ms ± 93.784447ms  	18.617037ms ± 107.423001ms	0s ± 0s                
+Random reads/writes (64k)               	local                                             	121 MB/s  	121 MB/s   	1.89 kIOPS 	1.89 kIOPS  	61.983727ms ± 21.324782ms 	73.228597ms ± 12.730945ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart                                              	75 MB/s   	75 MB/s    	1.17 kIOPS 	1.17 kIOPS  	26.830538ms ± 7.643051ms  	27.709602ms ± 7.830965ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	76 MB/s   	77 MB/s    	1.19 kIOPS 	1.19 kIOPS  	26.255337ms ± 7.302592ms  	27.256805ms ± 7.388266ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	32 MB/s   	33 MB/s    	505.26 IOPS	508.66 IOPS 	65.170269ms ± 747.794957ms	61.062186ms ± 695.614904ms	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	79 MB/s   	79 MB/s    	1.23 kIOPS 	1.23 kIOPS  	25.861503ms ± 171.669777ms	25.992302ms ± 164.647788ms	0s ± 0s                
+sync test                               	local                                             	0 B/s     	1.9 MB/s   	0 IOPS     	865.16 IOPS 	0s ± 0s                   	100.95µs ± 268.722µs      	1.054051ms ± 365.377µs 
+sync test                               	Tart                                              	0 B/s     	1.6 MB/s   	0 IOPS     	704.13 IOPS 	0s ± 0s                   	133.886µs ± 390.263µs     	1.285085ms ± 575.27µs  
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.6 MB/s   	0 IOPS     	728.26 IOPS 	0s ± 0s                   	129.246µs ± 472.724µs     	1.242713ms ± 1.281286ms
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	35 MB/s    	0 IOPS     	15.67 kIOPS 	0s ± 0s                   	11.319µs ± 24.771µs       	51.731µs ± 42.208µs    
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	17 MB/s    	0 IOPS     	7.39 kIOPS  	0s ± 0s                   	21.23µs ± 81.749µs        	113.239µs ± 191.266µs  
+```
@@ -0,0 +1,23 @@
+package main
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/command"
+	"log"
+	"os"
+	"os/signal"
+)
+
+func main() {
+	// Set up a signal-interruptible context
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+
+	// Run the root command
+	if err := command.NewCommand().ExecuteContext(ctx); err != nil {
+		cancel()
+
+		log.Fatal(err)
+	}
+
+	cancel()
+}
@@ -0,0 +1,29 @@
+module github.com/cirruslabs/tart/benchmark
+
+go 1.22.1
+
+require (
+	github.com/avast/retry-go/v4 v4.5.1
+	github.com/dustin/go-humanize v1.0.1
+	github.com/google/uuid v1.6.0
+	github.com/gosuri/uitable v0.0.4
+	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.9.0
+	go.uber.org/zap v1.27.0
+	golang.org/x/crypto v0.21.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/color v1.16.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
@@ -0,0 +1,52 @@
+github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
+github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
+github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -0,0 +1,69 @@
+package fio
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		// Ars Technica's "Single 4KiB random write process" test[1]
+		// with JSON output and created file cleanup
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "Single 4KiB random write process",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=4k --size=4g --numjobs=1 --iodepth=1 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// Ars Technica's "16 parallel 64KiB random write processes" test[1]
+		// with JSON outpu, created file cleanup and group reporting (for
+		// easier analysis)
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "16 parallel 64KiB random write processes",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=64k --size=256m --numjobs=16 --iodepth=16 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1 --group_reporting",
+	},
+	{
+		// Ars Technica's "16 parallel 64KiB random write processes" test[1]
+		// with JSON output, created file cleanup and reduced file I/O size
+		// from 16 to 10 GB to avoid "No space left on device".
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "Single 1MiB random write process",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=1m --size=10g --numjobs=1 --iodepth=1 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// Oracle's "Test random read/writes" (in IOPS Performance Tests[1]) category
+		// with JSON output, created file cleanup, without ETA newline, without custom
+		// file path, with file I/O size reduced from 500GB to 2GB to prevent
+		// "No space left on device" and with posixaio instead of libaio.
+		//
+		// [1]: https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm#FIO_Commands
+		Name: "Random reads/writes (4k)",
+		Command: "fio --name=benchmark --size=2GB --direct=1 --rw=randrw --bs=4k --ioengine=posixaio --iodepth=256 --runtime=120 --numjobs=4 --time_based --group_reporting" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// Oracle's "Test random read/writes" (in Throughput Performance Tests[1]) category
+		// with JSON output, created file cleanup, without ETA newline, without custom
+		// file path, with file I/O size reduced from 500GB to 2GB to prevent
+		// "No space left on device" and with posixaio instead of libaio.
+		//
+		// [1]: https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm#Throughput_Performance_Tests
+		Name: "Random reads/writes (64k)",
+		Command: "fio --name=benchmark --size=2GB --direct=1 --rw=randrw --bs=64k --ioengine=posixaio --iodepth=64 --runtime=120 --numjobs=4 --time_based --group_reporting" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// RedHat's "How can I test to see if my environment is fast enough for etcd"[1]
+		// with custom name
+		//
+		// [1]: https://access.redhat.com/solutions/5726511
+		Name: "sync test",
+		Command: "mkdir -p test-data && fio --name=benchmark --rw=write --ioengine=sync --fdatasync=1 --directory=test-data --size=22m --bs=2300" +
+			" --output-format json --unlink 1",
+	},
+}
@@ -0,0 +1,185 @@
+package fio
+
+import (
+	"encoding/json"
+	"fmt"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"github.com/dustin/go-humanize"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
+)
+
+var debug bool
+var image string
+var prepare string
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "fio",
+		Short: "run Flexible I/O tester (fio) benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-base:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	var executorInitializers = []struct {
+		Name string
+		Fn   func() (executorpkg.Executor, error)
+	}{
+		{
+			Name: "local",
+			Fn: func() (executorpkg.Executor, error) {
+				return local.New(logger)
+			},
+		},
+		{
+			Name: "Tart",
+			Fn: func() (executorpkg.Executor, error) {
+				return tart.New(cmd.Context(), image, nil, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none\")",
+			Fn: func() (executorpkg.Executor, error) {
+				return tart.New(cmd.Context(), image, []string{
+					"--root-disk-opts",
+					"sync=none",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"caching=cached\")",
+			Fn: func() (executorpkg.Executor, error) {
+				return tart.New(cmd.Context(), image, []string{
+					"--root-disk-opts",
+					"caching=cached",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
+			Fn: func() (executorpkg.Executor, error) {
+				return tart.New(cmd.Context(), image, []string{
+					"--root-disk-opts",
+					"sync=none,caching=cached",
+				}, logger)
+			},
+		},
+	}
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)",
+		"Latency (read)", "Latency (write)", "Latency (sync)")
+
+	for _, benchmark := range benchmarks {
+		for _, executorInitializer := range executorInitializers {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("installing Flexible I/O tester (fio) on executor %s",
+				executorInitializer.Name)
+
+			if _, err := executor.Run(cmd.Context(), "brew install fio"); err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			var fioResult Result
+
+			if err := json.Unmarshal(stdout, &fioResult); err != nil {
+				return err
+			}
+
+			if len(fioResult.Jobs) != 1 {
+				return fmt.Errorf("expected exactly 1 job from fio's JSON output, got %d",
+					len(fioResult.Jobs))
+			}
+
+			job := fioResult.Jobs[0]
+
+			readBandwidth := humanize.Bytes(uint64(job.Read.BW)*humanize.KByte) + "/s"
+			readIOPS := humanize.SIWithDigits(job.Read.IOPS, 2, "IOPS")
+
+			logger.Sugar().Infof("read bandwidth: %s, read IOPS: %s, read latency: %s",
+				readBandwidth, readIOPS, job.Read.LatencyNS.String())
+
+			writeBandwidth := humanize.Bytes(uint64(job.Write.BW)*humanize.KByte) + "/s"
+			writeIOPS := humanize.SIWithDigits(job.Write.IOPS, 2, "IOPS")
+
+			logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s, write latency: %s",
+				writeBandwidth, writeIOPS, job.Write.LatencyNS.String())
+
+			logger.Sugar().Infof("sync latency: %s", job.Sync.LatencyNS.String())
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, readBandwidth, writeBandwidth,
+				readIOPS, writeIOPS, job.Read.LatencyNS.String(), job.Write.LatencyNS.String(),
+				job.Sync.LatencyNS.String())
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}
@@ -0,0 +1,35 @@
+package fio
+
+import (
+	"fmt"
+	"time"
+)
+
+type Result struct {
+	Jobs []Job `json:"jobs"`
+}
+
+type Job struct {
+	Name  string `json:"jobname"`
+	Read  Stats  `json:"read"`
+	Write Stats  `json:"write"`
+	Sync  Stats  `json:"sync"`
+}
+
+type Stats struct {
+	BW        float64 `json:"bw"`
+	IOPS      float64 `json:"iops"`
+	LatencyNS Latency `json:"lat_ns"`
+}
+
+type Latency struct {
+	Mean   float64 `json:"mean"`
+	Stddev float64 `json:"stddev"`
+}
+
+func (latency Latency) String() string {
+	meanDuration := time.Duration(latency.Mean) * time.Nanosecond
+	stddevDuration := time.Duration(latency.Stddev) * time.Nanosecond
+
+	return fmt.Sprintf("%v ± %v", meanDuration, stddevDuration)
+}
@@ -0,0 +1,20 @@
+package command
+
+import (
+	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/spf13/cobra"
+)
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:           "benchmark",
+		SilenceUsage:  true,
+		SilenceErrors: true,
+	}
+
+	cmd.AddCommand(
+		fio.NewCommand(),
+	)
+
+	return cmd
+}
@@ -0,0 +1,10 @@
+package executor
+
+import (
+	"context"
+)
+
+type Executor interface {
+	Run(ctx context.Context, command string) ([]byte, error)
+	Close() error
+}
@@ -0,0 +1,42 @@
+package local
+
+import (
+	"bytes"
+	"context"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"io"
+	"os/exec"
+)
+
+type Local struct {
+	logger *zap.Logger
+}
+
+func New(logger *zap.Logger) (*Local, error) {
+	return &Local{
+		logger: logger,
+	}, nil
+}
+
+func (local *Local) Name() string {
+	return "local"
+}
+
+func (local *Local) Run(ctx context.Context, command string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, "zsh", "-c", command)
+
+	loggerWriter := &zapio.Writer{Log: local.logger, Level: zap.DebugLevel}
+	stdoutBuf := &bytes.Buffer{}
+
+	cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	cmd.Stderr = loggerWriter
+
+	err := cmd.Run()
+
+	return stdoutBuf.Bytes(), err
+}
+
+func (local *Local) Close() error {
+	return nil
+}
@@ -0,0 +1,20 @@
+package local_test
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"testing"
+)
+
+func TestLocal(t *testing.T) {
+	local, err := local.New(zap.NewNop())
+	require.NoError(t, err)
+
+	output, err := local.Run(context.Background(), "echo \"this is a test\"")
+	require.NoError(t, err)
+	require.Equal(t, "this is a test\n", string(output))
+
+	require.NoError(t, local.Close())
+}
@@ -0,0 +1,35 @@
+package tart
+
+import (
+	"bytes"
+	"context"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"io"
+	"os/exec"
+	"strings"
+)
+
+const tartBinaryName = "tart"
+
+func Cmd(ctx context.Context, logger *zap.Logger, args ...string) error {
+	_, err := CmdWithOutput(ctx, logger, args...)
+
+	return err
+}
+
+func CmdWithOutput(ctx context.Context, logger *zap.Logger, args ...string) (string, error) {
+	logger.Sugar().Debugf("running %s %s", tartBinaryName, strings.Join(args, " "))
+
+	cmd := exec.CommandContext(ctx, tartBinaryName, args...)
+
+	loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+	stdoutBuf := &bytes.Buffer{}
+
+	cmd.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	cmd.Stderr = loggerWriter
+
+	err := cmd.Run()
+
+	return stdoutBuf.String(), err
+}
@@ -0,0 +1,139 @@
+package tart
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"github.com/avast/retry-go/v4"
+	"github.com/google/uuid"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"golang.org/x/crypto/ssh"
+	"io"
+	"net"
+	"strings"
+	"time"
+)
+
+type Tart struct {
+	vmRunCancel context.CancelFunc
+	vmName      string
+	sshClient   *ssh.Client
+	logger      *zap.Logger
+}
+
+func New(ctx context.Context, image string, runArgsExtra []string, logger *zap.Logger) (*Tart, error) {
+	tart := &Tart{
+		vmName: fmt.Sprintf("tart-benchmark-%s", uuid.NewString()),
+		logger: logger,
+	}
+
+	if err := Cmd(ctx, tart.logger, "pull", image); err != nil {
+		return nil, err
+	}
+
+	if err := Cmd(ctx, tart.logger, "clone", image, tart.vmName); err != nil {
+		return nil, err
+	}
+
+	vmRunCtx, vmRunCancel := context.WithCancel(ctx)
+	tart.vmRunCancel = vmRunCancel
+
+	go func() {
+		runArgs := []string{"run", "--no-graphics", tart.vmName}
+
+		runArgs = append(runArgs, runArgsExtra...)
+
+		_ = Cmd(vmRunCtx, tart.logger, runArgs...)
+	}()
+
+	ip, err := CmdWithOutput(ctx, tart.logger, "ip", "--wait", "60", tart.vmName)
+	if err != nil {
+		return nil, tart.Close()
+	}
+
+	err = retry.Do(func() error {
+		dialer := net.Dialer{
+			Timeout: 1 * time.Second,
+		}
+
+		addr := fmt.Sprintf("%s:22", strings.TrimSpace(ip))
+
+		netConn, err := dialer.DialContext(ctx, "tcp", addr)
+		if err != nil {
+			return err
+		}
+
+		sshConn, chans, reqs, err := ssh.NewClientConn(netConn, addr, &ssh.ClientConfig{
+			User: "admin",
+			Auth: []ssh.AuthMethod{
+				ssh.Password("admin"),
+			},
+			HostKeyCallback: func(_ string, _ net.Addr, _ ssh.PublicKey) error {
+				return nil
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		tart.sshClient = ssh.NewClient(sshConn, chans, reqs)
+
+		return nil
+	}, retry.RetryIf(func(err error) bool {
+		return !errors.Is(err, context.Canceled)
+	}))
+	if err != nil {
+		return nil, tart.Close()
+	}
+
+	return tart, nil
+}
+
+func (tart *Tart) Name() string {
+	return "Tart"
+}
+
+func (tart *Tart) Run(ctx context.Context, command string) ([]byte, error) {
+	sshSession, err := tart.sshClient.NewSession()
+	if err != nil {
+		return nil, err
+	}
+
+	// Work around x/crypto/ssh not being context.Context-friendly (e.g. https://github.com/golang/go/issues/20288)
+	monitorCtx, monitorCancel := context.WithCancel(ctx)
+	go func() {
+		<-monitorCtx.Done()
+		_ = sshSession.Close()
+	}()
+	defer monitorCancel()
+
+	loggerWriter := &zapio.Writer{Log: tart.logger, Level: zap.DebugLevel}
+	stdoutBuf := &bytes.Buffer{}
+
+	sshSession.Stdin = bytes.NewBufferString(command)
+	sshSession.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	sshSession.Stderr = loggerWriter
+
+	if err := sshSession.Shell(); err != nil {
+		return nil, err
+	}
+
+	if err := sshSession.Wait(); err != nil {
+		return nil, err
+	}
+
+	return stdoutBuf.Bytes(), nil
+}
+
+func (tart *Tart) Close() error {
+	if tart.sshClient != nil {
+		_ = tart.sshClient.Close()
+	}
+
+	tart.vmRunCancel()
+	_ = Cmd(context.Background(), tart.logger, "delete", tart.vmName)
+
+	return nil
+}
@@ -0,0 +1,22 @@
+package tart_test
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"testing"
+)
+
+func TestTart(t *testing.T) {
+	ctx := context.Background()
+
+	tart, err := tart.New(ctx, "ghcr.io/cirruslabs/macos-sonoma-base:latest", nil, zap.NewNop())
+	require.NoError(t, err)
+
+	output, err := tart.Run(ctx, "echo \"this is a test\"")
+	require.NoError(t, err)
+	require.Equal(t, "this is a test\n", string(output))
+
+	require.NoError(t, tart.Close())
+}
@@ -0,0 +1,62 @@
+---
+draft: false
+date: 2024-06-20
+search:
+  exclude: true
+authors:
+  - edigaryev
+categories:
+  - orchard
+---
+
+# Jumping through the hoops: SSH jump host functionality in Orchard
+
+Almost a year ago, when we started building [Orchard](https://github.com/cirruslabs/orchard), an orchestration system for Tart, we quickly realized that most worker machines will be in a private network, and that VMs will be only reachable from the worker machines themselves. Thus, one of our goals became to simplify accessing the compute resources in a cluster through a centralized controller host.
+
+This effort resulted in commands like `orchard port-forward` and `orchard ssh`, which were later improved to support connecting not just to the VMs, but to the worker machines themselves.
+
+Today, we’re making an even further step in this effort: with a trivial configuration, an Orchard controller can act as an SSH jump host to allow connecting to the VMs using just the `ssh` command like `ssh -J <service account name>@orchard-controller.example.com <VM name>`!
+
+<!-- more -->
+
+## Implementation
+
+In a typical cluster there’s one controller, to which workers connect by calling various REST API endpoints to synchronize the worker & VMs state. Each worker also maintains a persistent bi-directional gRPC connection with the controller, with the goal of improving the overall reactivity and making the port-forwarding work.
+
+The gRPC service definition that the controller offers is pretty minimalistic:
+
+```protobuf
+service Controller {
+  rpc Watch(google.protobuf.Empty) returns (stream WatchInstruction);
+  rpc PortForward(stream PortForwardData) returns (stream PortForwardData);
+}
+```
+
+Each watch instruction corresponds a single action to be done by the worker, which can either be a request for establishing a port-forwarding stream or a request for VMs re-syncing:
+
+```protobuf
+oneof action {
+  PortForward port_forward_action = 1;
+  SyncVMs sync_vms_action = 2;
+}
+```
+
+Now, when the user invokes `orchard port-forward` or `orchard ssh`, controller effectively becomes a rendezvous point by accepting the WebSocket connection from the user, and then asking the worker associated with the requested VM to establish a port-forwarding stream, and finally proxying the two streams together.
+
+![An illustration showing the Orchard controller and worker proxying the SSH connection](../images/jumping-through-the-hoops.png)
+
+SSH protocol works the same way, multiplexing multiple channels in a single transport connection, where each channel can be upgraded either to an interactive session (that’s what you get when you `ssh` to the server) or X11 channel (for X11 forwarding using `-X`), direct or forward TCP/IP channels (these are used for local and remote port-forwarding when using `-L` and `-R` options correspondingly) and so on.
+
+In fact, `ssh -J` jump host functionality also uses the direct TCP/IP channel, which is [just a single port-forwarding request](https://datatracker.ietf.org/doc/html/rfc4254#section-7.2) that needs to be implemented. We’ve used [Golang's SSH library](https://pkg.go.dev/golang.org/x/crypto/ssh) as the most mature choice for this task, and it’s been pleasant to work with so far.
+
+The support for `ssh -J` has landed in Orchard version 0.19.0. To configure the SSH jump host, simply add the `--listen-ssh` command-line argument to your `orchard controller run` invocation.
+
+Once running, you can connect to any VM in the cluster using the `ssh -J <service account name>@orchard-controller.example.com <VM name>`. The password for the jump host is the corresponding service account’s token.
+
+## Future plans
+
+First of all, we’d like to thank our paid clients, without which this feature wouldn’t be possible. [Become one now](../../licensing.md) and get the benefit of higher Tart VMs and Orchard workers allowances and making sure that the roadmap for Tart and Orchard is aligned with your company's needs.
+
+In the near future we plan to implement a mechanism similar to `authorized_keys` file that will allow attaching public SSH keys to the Orchard controller’s service accounts, and thus avoid the need to type the passwords.
+
+Stay tuned and don’t hesitate to send us your feedback on [GitHub](https://github.com/cirruslabs/orchard) and [Twitter](https://x.com/cirrus_labs)!
@@ -1,6 +1,8 @@
 ---
 hide:
  - navigation
+title: Frequently Asked Questions
+description: Advanced configuration and troubleshooting tips for advanced configurations.
 ---

 ## VM location on disk
@@ -55,7 +57,15 @@ This issue is worked around automatically [when using Softnet](http://github.com
 sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.InternetSharing.default.plist bootpd -dict DHCPLeaseTimeSecs -int 600
 ```

-Note that this tweak persists across reboots, so normally you'll only need to do it once per new host.
+This tweak persists across reboots, so normally you'll only need to do it once per new host.
+
+If that doesn't help after starting a new VM, it's possible that the `/var/db/dhcpd_leases` file is already overfilled with 86,400-second leases. You can remove it with the following command and try starting a new VM again:
+
+```shell
+sudo rm /var/db/dhcpd_leases
+```
+
+And no worries, this file will be re-created on the next `tart run`.

 ## Running login/clone/pull/push commands over SSH

@@ -88,3 +98,21 @@ Instead of Anka Registry, Tart can work with any OCI-compatible container regist
 and scalable experience for distributing virtual machines.

 Tart doesn't yet have an analogue of Anka Controller for managing long living VMs but [soon will be](https://github.com/cirruslabs/tart/issues/372).
+
+## Automatic pruning
+
+`tart pull` and `tart clone` commands check the remaining space available on the volume associated with `TART_HOME` directory (defaults to `~/.tart`) before pulling or cloning anything.
+
+In case there's not enough space to fit the newly pulled or cloned VM image, Tart will remove the least recently accessed VMs from OCI cache and `.ipsw` files from IPSW cache until enough free space is available.
+
+To disable this functionality, set the `TART_NO_AUTO_PRUNE` environment variable either globally:
+
+```shell
+export TART_NO_AUTO_PRUNE=
+```
+
+...or per `tart pull` and `tart clone` invocation as follows:
+
+```shell
+TART_NO_AUTO_PRUNE= tart pull ...
+```
@@ -1,4 +1,5 @@
 ---
 template: overrides/home.html
-title: Tart
+title: Toolset to build, run and manage macOS and Linux VMs
+description: Native performance. Remote storage for Virtual Machines. Many integrations including GitHub, GitLab and more.
 ---
@@ -1,3 +1,8 @@
+---
+title: Buildkite Integration
+description: Run pipeline steps in isolated ephemeral Tart Virtual Machines.
+---
+
 # Buildkite

 It is possible to run [Buildkite](https://buildkite.com/) pipeline steps in isolated ephemeral Tart Virtual Machines with the help of [Tart Buildkite Plugin](https://github.com/cirruslabs/tart-buildkite-plugin):
@@ -1,3 +1,8 @@
+---
+title: Cirrus CLI
+description: Tool for running isolated tasks reproducibly in any environment with a simple YAML configuration.
+---
+
 # Cirrus CLI

 Tart itself is only responsible for managing virtual machines, but we've built Tart support into a tool called Cirrus CLI
@@ -1,128 +0,0 @@
-# Cirrus Runners for GitHub Actions
-
-*Cirrus Runners* is the fastest and most cost-efficient way to get your current CI workflows to benefit from Apple Silicon hardware. No need to manage infrastructure or migrate to another CI provider.
-Your actions will be executed in clean macOS virtual machines with 4 Apple M2 cores.
-
-## Testimonials from customers
-
-Mitchell Hashimoto, HashiCorp co-founder:
-
-> I've been using "Cirrus Runners" since [that tweet](https://twitter.com/mitchellh/status/1731071326201561194) and it has been fantastic. Huge speed increase, huge cost decrease, zero maintenance, exactly what I wanted.
-
-Max Lapides, Senior Mobile Engineer at [Tonal](https://www.tonal.com/):
-
-> Previously, we were using the GitHub‑hosted macOS runners and our iOS build took ~30 minutes. Now with Cirrus Runners, the iOS build only takes ~12 minutes. That’s a huge boost to our productivity, and for only $150/month per runner it is much less expensive too.
-
-John A., Software Engineer at [GitKraken](https://www.gitkraken.com/):
-
-> GitHub Actions MacOS-x86 runners have become increasingly unreliable, so we're moving our Mac builds over to arm64 because Cirrus Labs' M1 runners are not only ~3 times faster, they've also been far more stable.
-
-Sebastian Jachec, Mobile Engineer at [Daybridge](https://www.daybridge.com/):
-
-> It’s been plain-sailing with the Cirrus Runners — they’ve been great! They’re consistently 60+% faster on workflows that we previously used Github Actions’ macOS runners for.
-
-## Pricing
-
-Each Cirrus Runner costs $150 a month and there is no limit on the amount of minutes for your actions.
-We recommend to purchase several Cirrus Runners depending on your team size, so you can run actions in
-parallel. Note that you can change your subscription at any time via [this page](https://billing.stripe.com/p/login/3cs7vNbzo92p7fy3cc)
-or by emailing [support@cirruslabs.org](mailto:support@cirruslabs.org).
-
-### Discounts
-
-We offer two mutually exclusive discounts:
-
- 10% "Volume Discount" for subscriptions of 10 or more Cirrus Runners.
- 15% "Annual Discount" for 12 months subscription commitment of any amount of Cirrus Runners.
-
-Please contact [support@cirruslabs.org](mailto:support@cirruslabs.org) after activating the subscription in order to get the discount applied.
-
-### Priority Support
-
-Subscriptions of 20 or more Cirrus Runners include access to [Priority Support](../licensing.md#priority-support).
-Please contact [sales@cirruslabs.org](mailto:sales@cirruslabs.org) in order to get all the details.
-
-### CPU and Memory resources of Cirrus Runners
-
-By default, a single Cirrus Runner is allocated with 4 M2 cores and 12 GB of unified memory which is enough for most of the workloads.
-For workloads that require more resources it is possible to use XL Cirrus Runners which have twice the resources: a full M2 chip with 8 cores
-and 24 GB of unified memory. Note that a single XL Cirrus Runner also uses twice the concurrency.
-
-In order to use an XL Cirrus Runner for a job please append `-xl` suffix to your `runs-on` property. More on that down below.
-
-## Installation
-
-Once you configure [Cirrus Runners App](https://github.com/apps/cirrus-runners) for your organization, you'll be redirected
-to a checkout page powered by Stripe. During the checkout process you'll be able to configure a subscription for
-a desired amount of parallel Cirrus Runners and try it for free for 10 days.
-
-Once configured, please follow instruction below. If you have any questions please contact [support@cirruslabs.org](mailto:support@cirruslabs.org).
-Subscriptions with more than 10 runners also include Priority Support 
-
-## Configuring Cirrus Runners
-
-In order for Cirrus Runners to be used by your GitHub Actions workflow jobs, specify a desired image in the `runs-on` property.
-
-=== "Default Cirrus Runner"
-
-    ```yaml
-    name: Tests
-    jobs:
-      test:
-        runs-on: ghcr.io/cirruslabs/macos-sonoma-xcode:latest
-    ```
-
-=== "XL Cirrus Runner"
-
-    ```yaml
-    name: Integration Tests
-    jobs:
-      test:
-        runs-on: ghcr.io/cirruslabs/macos-sonoma-xcode:latest-xl
-    ```
-
-List of all available images can be found in [this repository](https://github.com/cirruslabs/macos-image-templates).
-
-Note that Tart VM images don't have the same set of pre-installed packages as the official Intel GitHub runners.
-If something is missing please [create an issue within this repository](https://github.com/cirruslabs/macos-image-templates/issues/new).
-
-When workflows are executing you'll see Cirrus on-demand runners on your organization's settings page at `https://github.com/organizations/<ORGANIZATION>/settings/actions/runners`.
-Note that Cirrus Runners will get added to the default runner group.
-
-!!! tip "Using Cirrus Runners with public repositories"
-
-    By default, only private repositories can access runners in a default runner group, but you can override this in your organization's settings:
-
-    ```https://github.com/organizations/<YOUR ORGANIZATION NAME>/settings/actions/runner-groups/1```
-
-![](/assets/images/TartGHARunners.png)
-
-### Dashboard
-
-You can also see the status of your runners on the [Cirrus Runners Dashboard](https://cirrus-runners.app/). This dashboard
-also provides insights into price performance of your Cirrus Runners. Please check out [this blog post](/blog/2023/11/03/new-dashboard-with-insights-into-performance-of-cirrus-runners/)
-to learn more about what this dashboard can do for you.
-
-![](/assets/images/RunnersDashboard.png)
-
-## Data handling flow
-
-By design Cirrus Runners service never sees any of your secrets or source code and acts as compute platform with the lastest
-Apple Silicon hardware that can quickly allocate CPU/Memory resources for your jobs.
-
-Here is a high-level overview of how Cirrus Runners service manages runners for your organization:
-
- Cirrus Runner GitHub App is subscribed to [`workflow_job`](https://docs.github.com/en/webhooks/webhook-events-and-payloads#workflow_job).
- Upon receiving a new event targeting Cirrus Runners via `runs-on` property the following steps take place:
-
-    - Non-personal information about your job is saved to perform health checking of Cirrus Runners execution.
-    - Cirrus Runners GitHub App has only one permission that allows generating temporary registration tokens for
-      self-hosted GitHub Actions Runners. Note that Cirrus Runners GitHub App itself doesn't have access to contents of
-      repositories in your organization.
-    - Cirrus Runners Service creates a new single use Tart VM, generates a temporary registration tokens for self-hosted runners
-      and passes it without storing inside the VM for the GitHub Actions Runner service to [start a ephemeral runner](https://github.blog/changelog/2021-09-20-github-actions-ephemeral-self-hosted-runners-new-webhooks-for-auto-scaling/).
-
- Cirrus Runners service continuously monitors health of the Tart VM executing your job to make sure it runs to completion.
- After the job finishes the ephemeral Tart VM is getting destroyed with all the information of the job run.
-
-If you have any questions or concerns please feel free to reach out to [support@cirruslabs.org](mailto:support@cirruslabs.org).
@@ -1,3 +1,8 @@
+---
+title: GitLab Runner Executor
+description: Run jobs in isolated ephemeral Tart Virtual Machines.
+---
+
 # GitLab Runner Executor

 It is possible to run GitLab jobs in isolated ephemeral Tart Virtual Machines via [Tart Executor](https://github.com/cirruslabs/gitlab-tart-executor).
@@ -1,3 +1,8 @@
+---
+title: Managing Virtual Machine
+description: Use Packer to build custom VM images, configure VMs and work with remote OCI registries.
+---
+
 # Managing Virtual Machine

 ## Creating from scratch
@@ -15,7 +20,7 @@ tart create --from-ipsw=latest sonoma-vanilla
 tart run sonoma-vanilla
 ```

-After the initial booting of the VM you'll need to manually go through the macOS installation process. As a convention we recommend creating an `admin` user with an `admin` password. After the regular installation please do some additional modifications in the VM:
+After the initial booting of the VM, you'll need to manually go through the macOS installation process. As a convention we recommend creating an `admin` user with an `admin` password. After the regular installation please do some additional modifications in the VM:

 1. Enable Auto-Login. Users & Groups -> Login Options -> Automatic login -> admin.
 2. Allow SSH. Sharing -> Remote Login
@@ -48,7 +53,7 @@ sudo ufw allow ssh

 ## Configuring a VM

-By default, a tart VM uses 2 CPUs and 4 GB of memory with a `1024x768` display. This can be changed with `tart set` command.
+By default, a Tart VM uses 2 CPUs and 4 GB of memory with a `1024x768` display. This can be changed after VM creation with `tart set` command.
 Please refer to `tart set --help` for additional details.

 ## Building with Packer
@@ -92,18 +97,19 @@ Here is a [repository with Packer templates](https://github.com/cirruslabs/macos

 ## Working with a Remote OCI Container Registry

-<!-- markdownlint-disable MD034 -->
-For example, let's say you want to push/pull images to a registry hosted at https://acme.io/.
-<!-- markdownlint-enable MD034 -->
+Tart supports interacting with Open Container Initiative (OCI) registries, but only runs images created and pushed by Tart. This means images created for container engines, like Docker, can't be pulled. Instead, create a custom image as documented above.
+
+For example, let's say you want to push/pull images to an OCI registry hosted at `https://acme.io/`.

 ### Registry Authorization

-First, you need to log in and save credential for `acme.io` host via `tart login` command:
+First, you need to login to `acme.io` with the `tart login` command:

 ```bash
 tart login acme.io
 ```

+If you login to your registry with OAuth, you may need to create an access token to use as the password.
 Credentials are securely stored in Keychain.

 In addition, Tart supports [Docker credential helpers](https://docs.docker.com/engine/reference/commandline/login/#credential-helpers)
@@ -128,10 +134,10 @@ You can either pull an image:
 tart pull acme.io/remoteorg/name:latest
 ```

-...or instantiate a VM from a remote image:
+or create a VM from a remote image:

 ```bash
 tart clone acme.io/remoteorg/name:latest my-local-vm-name
 ```

-This invocation calls the `tart pull` implicitly (if the image is not being present) before doing the actual cloning.
+If the specified image is not already present, this invocation calls the `tart pull` implicitly before cloning.
@@ -0,0 +1,260 @@
+# Copyright (c) 2016-2024 Martin Donath <martin.donath@squidfunk.com>
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# -----------------------------------------------------------------------------
+# Configuration
+# -----------------------------------------------------------------------------
+
+# Definitions
+definitions:
+
+  # Background image
+  - &background_image >-
+    {{ layout.background_image | x }}
+
+  # Background color (default: indigo)
+  - &background_color >-
+    {%- if layout.background_color -%}
+      {{ layout.background_color }}
+    {%- else -%}
+      {%- set palette = config.theme.palette or {} -%}
+      {%- if not palette is mapping -%}
+        {%- set list = palette | selectattr("accent") | list + palette -%}
+        {%- set palette = list | first -%}
+      {%- endif -%}
+      {%- set accent = palette.get("accent", "indigo") -%}
+      {%- set accent = accent.replace(" ", "-") -%}
+      {{ {
+        "red":         "#ff1a47",
+        "pink":        "#f50056",
+        "purple":      "#df41fb",
+        "deep-purple": "#7c4dff",
+        "indigo":      "#526cfe",
+        "blue":        "#4287ff",
+        "light-blue":  "#0091eb",
+        "cyan":        "#00bad6",
+        "teal":        "#00bda4",
+        "green":       "#00c753",
+        "light-green": "#63de17",
+        "lime":        "#b0eb00",
+        "yellow":      "#ffd500",
+        "amber":       "#ffaa00",
+        "orange":      "#ff9100",
+        "deep-orange": "#ff6e42"
+      }[accent] or "#4051b5" }}
+    {%- endif -%}
+
+  # Text color (default: white)
+  - &color >-
+    {%- if layout.color -%}
+      {{ layout.color }}
+    {%- else -%}
+      {%- set palette = config.theme.palette or {} -%}
+      {%- if not palette is mapping -%}
+        {%- set list = palette | selectattr("accent") | list + palette -%}
+        {%- set palette = list | first -%}
+      {%- endif -%}
+      {%- set accent = palette.get("accent", "indigo") -%}
+      {%- set accent = accent.replace(" ", "-") -%}
+      {{ {
+        "red":         "#ffffff",
+        "pink":        "#ffffff",
+        "purple":      "#ffffff",
+        "deep-purple": "#ffffff",
+        "indigo":      "#ffffff",
+        "blue":        "#ffffff",
+        "light-blue":  "#ffffff",
+        "cyan":        "#ffffff",
+        "teal":        "#ffffff",
+        "green":       "#ffffff",
+        "light-green": "#ffffff",
+        "lime":        "#000000",
+        "yellow":      "#000000",
+        "amber":       "#000000",
+        "orange":      "#000000",
+        "deep-orange": "#ffffff"
+      }[accent] or "#ffffff" }}
+    {%- endif -%}
+
+  # Font family (default: Roboto)
+  - &font_family >-
+    {%- if layout.font_family -%}
+      {{ layout.font_family }}
+    {%- elif config.theme.font != false -%}
+      {{ config.theme.font.get("text", "Roboto") }}
+    {%- else -%}
+      Roboto
+    {%- endif -%}
+
+  # Font variant
+  - &font_variant >-
+    {%- if layout.font_variant -%}
+      {{ layout.font_variant }}
+    {%- endif -%}
+
+  # Site name
+  - &site_name >-
+    {{ config.site_name }}
+
+  # Page title
+  - &page_title >-
+    {%- if page.meta.no_title_in_card -%}
+      {# do not show anything #}
+    {%- elif layout.title -%}
+      {{ layout.title }}
+    {%- else -%}
+      {{ page.meta.get("title", page.title) }}
+    {%- endif -%}
+
+  # Page title with site name
+  - &page_title_with_site_name >-
+    {%- if not page.is_homepage -%}
+      {{ page.meta.get("title", page.title) }} - {{ config.site_name }}
+    {%- else -%}
+      {{ page.meta.get("title", page.title) }}
+    {%- endif -%}
+
+  # Page description
+  - &page_description >-
+    {%- if layout.description -%}
+      {{ layout.description }}
+    {%- else -%}
+      {{ page.meta.get("description", config.site_description) | x }}
+    {%- endif -%}
+
+  # Page description for social card
+  - &page_description_social_card >-
+    {%- if layout.description -%}
+      {{ layout.description }}
+    {%- else -%}
+      {{ page.meta.get("description", config.site_description_social_card) | x }}
+    {%- endif -%}
+
+  # Logo
+  - &logo >-
+    {%- if layout.logo -%}
+      {{ layout.logo }}
+    {%- elif config.theme.logo -%}
+      {{ config.docs_dir }}/{{ config.theme.logo }}
+    {%- endif -%}
+
+  # Logo (icon)
+  - &logo_icon >-
+    {%- if not layout.logo -%}
+      {{ config.theme.icon.logo | x }}
+    {%- endif -%}
+
+# Meta tags
+tags:
+
+  # Open Graph
+  og:type: website
+  og:title: *page_title_with_site_name
+  og:description: *page_description
+  og:image: "{{ image.url }}"
+  og:image:type: "{{ image.type }}"
+  og:image:width: "{{ image.width }}"
+  og:image:height: "{{ image.height }}"
+  og:url: "{{ page.canonical_url }}"
+
+  # Twitter
+  twitter:card: summary_large_image
+  twitter:title: *page_title_with_site_name
+  twitter:description: *page_description
+  twitter:image: "{{ image.url }}"
+
+# -----------------------------------------------------------------------------
+# Specification
+# -----------------------------------------------------------------------------
+
+# Card size and layers
+size: { width: 1200, height: 630 }
+layers:
+
+  # Background
+  - background:
+      image: *background_image
+      color: *background_color
+
+  # Logo
+  - size: { width: 170, height: 192 }
+    offset: { x: 966, y: 64 }
+    background:
+      image: *logo
+    icon:
+      value: *logo_icon
+      color: *color
+
+  # Site name
+  - size: { width: 832, height: 42 }
+    offset: { x: 64, y: 64 }
+    typography:
+      content: *site_name
+      align: start center
+      color: *color
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Bold
+
+  # Motto
+  - size: { width: 832, height: 150 }
+    offset: { x: 64, y: 106 }
+    typography:
+      content: "{{ config.motto }}"
+      align: start center
+      color: *color
+      line:
+        amount: 2.5
+        height: 1.25
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Bold
+
+  # Page title
+  - size: { width: 1072, height: 256 }
+    offset: { x: 64, y: 256 }
+    typography:
+      content: *page_title
+      align: start center
+      color: *color
+      line:
+        amount: 3
+        height: 1.25
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Bold
+
+  # Page description
+  - size: { width: 832, height: 64 }
+    offset: { x: 64, y: 512 }
+    typography:
+      content: *page_description_social_card
+      align: start center
+      color: *color
+      line:
+        amount: 2
+        height: 1.5
+      font:
+        family: *font_family
+        variant: *font_variant
+        style: Regular
@@ -11,7 +11,7 @@ This page covers Terms of Service only for Cirrus Runners and Tart Documentation

 ### The Gist

-Cirrus Labs Inc ("Cirrus Labs") operates the [Cirrus Runners service](../integrations/github-actions.md) which we hope you use.
+Cirrus Labs Inc ("Cirrus Labs") operates the [Cirrus Runners service](https://cirrus-runners.app/) which we hope you use.
 If you use it, please use it responsibly. If you don't, we'll have to terminate your subscription.

 For paid plans, you'll be charged on a monthly basis. You can cancel anytime, but there are no refunds.
@@ -1,6 +1,8 @@
 ---
 hide:
  - navigation
+title: Licensing and Support
+description: Free Tier with 100 CPU core limit. Very affordable Tiers for larger enterprises.
 ---

 Both [Tart Virtualization](https://github.com/cirruslabs/tart) and [Orchard Orchestration](https://github.com/cirruslabs/orchard)
--- a/Show More
+++ b/Show More