Avoid duplicate progress updates in CI logs (#1140)

* Avoid duplicate progress updates in CI logs

* Update Sources/tart/Logging/ProgressObserver.swift

Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com>

---------

Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com>

.ci/set-version.sh

@@ -3,3 +3,5 @@
 TMPFILE=$(mktemp)
 envsubst < Sources/tart/CI/CI.swift > $TMPFILE
 mv $TMPFILE Sources/tart/CI/CI.swift
+
+/usr/libexec/PlistBuddy -c "Add :CFBundleShortVersionString string ${CIRRUS_TAG}" Resources/Info.plist

.cirrus.yml

@@ -1,24 +1,23 @@
 use_compute_credits: true
 
 task:
-  name: Test on Sonoma
+  name: Test on Sequoia
   alias: test
   persistent_worker:
     labels:
       name: dev-mini
     resources:
       tart-vms: 1
+  build_script:
+    - swift build
   test_script:
     - swift test
   integration_test_script:
-    # Build Tart
-    - swift build
     - codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
     - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH
     # Run integration tests
     - cd integration-tests
-    - HOMEBREW_NO_AUTO_UPDATE=1 brew install virtualenv
-    - virtualenv venv
+    - python3 -m venv --symlinks venv
     - source venv/bin/activate
     - pip install -r requirements.txt
     - pytest --verbose --junit-xml=pytest-junit.xml
@@ -38,7 +37,7 @@ task:
   name: Lint
   alias: lint
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
   lint_script:
     - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
   always:
@@ -55,7 +54,7 @@ task:
   name: Build ($BUILD_ARCH)
   alias: build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
   build_script: swift build --arch $BUILD_ARCH --product tart
   sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/$BUILD_ARCH-apple-macosx/debug/tart
   binary_artifacts:
@@ -68,7 +67,7 @@ task:
     - lint
     - build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
   env:
     MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -84,13 +83,14 @@ task:
     - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
     - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
   install_script:
-    - brew install go goreleaser/tap/goreleaser-pro
+    - brew install go
     - brew install mitchellh/gon/gon
+    - brew install --cask goreleaser/tap/goreleaser-pro
   info_script:
     - security find-identity -v
     - xcodebuild -version
     - swift -version
-  goreleaser_script: goreleaser release --skip-publish --snapshot --clean
+  goreleaser_script: goreleaser release --skip=publish --snapshot --clean
   always:
     dist_artifacts:
       path: "dist/*"
@@ -103,7 +103,7 @@ task:
     - test
     - build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-runner:sonoma
+    image: ghcr.io/cirruslabs/macos-runner:sequoia
   env:
     MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -111,7 +111,7 @@ task:
     GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!]
     SENTRY_ORG: cirrus-labs
     SENTRY_PROJECT: persistent-workers
-    SENTRY_AUTH_TOKEN: ENCRYPTED[!c16a5cf7da5f856b4bc2f21fe8cb7aa2a6c981f851c094ed4d3025fd02ea59a58a86cee8b193a69a1fc20fa217e56ac3!]
+    SENTRY_AUTH_TOKEN: ENCRYPTED[!9eaf2875d51b113e2f68598441ff8e6b2e53242e48fcb93633bd75a373fbe2e7caa900d837cc92f0b142b65579731644!]
   setup_script:
     - cd $HOME
     - echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
@@ -121,9 +121,10 @@ task:
     - security import certificate.p12 -k build.keychain -P password101 -T /usr/bin/codesign -T /usr/bin/pkgbuild
     - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
     - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
-  install_script: 
-    - brew install go goreleaser/tap/goreleaser-pro getsentry/tools/sentry-cli
+  install_script:
+    - brew install go getsentry/tools/sentry-cli
     - brew install mitchellh/gon/gon
+    - brew install --cask goreleaser/tap/goreleaser-pro
   info_script:
     - security find-identity -v
     - xcodebuild -version
@@ -153,7 +154,7 @@ task:
     registry_config: ENCRYPTED[!cf1a0f25325aa75bad3ce6ebc890bc53eb0044c02efa70d8cefb83ba9766275a994b4831706c52630a0692b2fa9cfb9e!]
   env:
     DEPLOY_TOKEN: ENCRYPTED[!45ed45666558902ed1c2400add734ec063103bec31841847e8c8764802fca229bfa6d85c690e16ad159e047574b48793!]
-  deploy_script: 
+  deploy_script:
     - git config --global user.name "Cirrus CI"
     - git config --global user.name "hello@cirruslabs.org"
     - git remote set-url origin https://$DEPLOY_TOKEN@github.com/cirruslabs/tart/

.gitignore

@@ -8,6 +8,9 @@ tart.xcodeproj/
 # AppCode
 .idea/
 
+# VS Code
+.vscode/
+
 # Swift
 .build/
 
@@ -16,3 +19,6 @@ dist/
 
 # mkdocs
 .cache
+
+# mkdocs-material
+site

.goreleaser.yml

@@ -1,11 +1,12 @@
+version: 2
+
 project_name: tart
 
 before:
   hooks:
     - .ci/set-version.sh
-    - swift build --arch x86_64 -c release --product tart
-    - swift build --arch arm64 -c release --product tart
-    - gon gon.hcl
+    - swift build --arch arm64 --configuration release --product tart
+    - swift build --arch x86_64 --configuration release --product tart
 
 builds:
   - id: tart
@@ -20,8 +21,14 @@ builds:
     prebuilt:
       path: '.build/{{- if eq .Arch "arm64" }}arm64{{- else }}x86_64{{ end }}-apple-macosx/release/tart'
 
+universal_binaries:
+  - name_template: tart.app/Contents/MacOS/tart
+    replace: true
+    hooks:
+      post: gon gon.hcl
+
 archives:
-  - name_template: "{{ .ProjectName }}-{{ .Arch }}"
+  - name_template: "{{ .ProjectName }}"
     files:
       - src: Resources/embedded.provisionprofile
         dst: tart.app/Contents
@@ -42,7 +49,13 @@ brews:
     repository:
       owner: cirruslabs
       name: homebrew-cli
-    caveats: See the GitHub repository for more information
+    caveats: |
+      Tart has been installed. You might want to reduce the default DHCP lease time
+      from 86,400 to 600 seconds to avoid DHCP shortage when running lots of VMs daily:
+
+        sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.InternetSharing.default.plist bootpd -dict DHCPLeaseTimeSecs -int 600
+
+      See https://tart.run/faq/#changing-the-default-dhcp-lease-time for more details.
     homepage: https://github.com/cirruslabs/tart
     license: "Fair Source"
     description: Run macOS and Linux VMs on Apple Hardware

CONTRIBUTING.md

@@ -29,6 +29,7 @@ Table of Contents
 
 1. Code should follow camel case
 2. Code should follow [SwiftFormat](https://github.com/nicklockwood/SwiftFormat#swift-package-manager-plugin) guidelines. You can auto-format the code by running the following command:
+
 ```bash
 swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore .
 ```

PROFILING.md

@@ -0,0 +1,64 @@
+# Profiling Tart
+
+## Using `time(1)`
+
+Perhaps, the easiest, but not the most comprehensive way to tell what's going on with Tart is to use the [`time(1)`](https://ss64.com/mac/time.html) command.
+
+In the example below, you will run `tart pull` via `time(1)` to gather generalized CPU, I/O and memory usage metrics:
+
+```shell
+/usr/bin/time -l tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+**Note:** you need to specify a full path to `time(1)` binary, otherwise the shell's built-in `time` command will be invoked, which doesn't have the `-l` command-line argument.
+
+**Note:** The `-l` command-line argument makes `time(1)` return much more useful information, for example, maximum memory usage.
+
+When running the command above, you'll see the `tart pull` output first as it pulls the image, and then the `time(1)` output, which will be printed once the Tart process finishes:
+
+```
+      172.17 real        10.29 user         8.36 sys
+           353796096  maximum resident set size
+                   0  average shared memory size
+                   0  average unshared data size
+                   0  average unshared stack size
+               23838  page reclaims
+                  35  page faults
+                   0  swaps
+                   0  block input operations
+                   0  block output operations
+                   8  messages sent
+                   8  messages received
+                   0  signals received
+                 146  voluntary context switches
+              222950  involuntary context switches
+         39683070975  instructions retired
+         27562035252  cycles elapsed
+           170920448  peak memory footprint
+```
+
+From the output above, you can tell that `tart pull` spent nearly 90% of time off-CPU (`real` > `user` + `sys`), which means that Tart was mostly waiting for the I/O (be it a network or disk), instead of decompressing disk layers or doing other useful computations.
+
+## Using `xctrace(1)`
+
+[`xctrace(1)`](https://keith.github.io/xcode-man-pages/xctrace.1.html) is a `.trace` format recorder for the [Instruments](https://en.wikipedia.org/wiki/Instruments_(software)) app, which yields much more powerful insights compared to `time(1)`. For example, it can tell which Tart functions spent the most time on the CPU, thus allowing the Tart developers to further optimize these functions.
+
+To use it, make sure that [Xcode](https://developer.apple.com/xcode/resources/) is installed. If you're installing Xcode for the first time on the machine, you'll need to launch it once and click the blue "Install" button. There's no need to choose any platforms except for the macOS.
+
+Once done, you can create a CPU profile of `tart pull`:
+
+```shell
+xctrace record --template "CPU Profiler" --target-stdout - --launch -- /opt/homebrew/bin/tart pull ghcr.io/cirruslabs/macos-sequoia-base:latest
+```
+
+Now that `xctrace(1)` is running, you'll see the `tart pull`-related output first, and once finished, the following line will appear:
+
+```
+Output file saved as: Launch_[...].trace
+```
+
+To view this trace in the Instruments app, simply find this directory in Finder and double-click it. Instruments app will appear:
+
+![](Resources/Instruments.png)
+
+To send this trace, right-click its directory in Finder and choose "Compress [...]". This will result in a similarly named file with a `.zip` at the end, which can now be conveniently sent via email or uploaded.

Package.resolved

@@ -1,13 +1,31 @@
 {
-  "originHash" : "6d48639bc0ea02002de0b4f38fe3fce0ddc9d174f2e56180c2ffcbedb7391ef8",
+  "originHash" : "668bad809d4882f75f097e66a12a6dbc8e61ec998f1800a7e09439c854fadda1",
   "pins" : [
     {
       "identity" : "antlr4",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/antlr/antlr4",
       "state" : {
-        "branch" : "dev",
-        "revision" : "2703a8516c0fb7fe92db6b9c40e0113f577646d2"
+        "revision" : "cc82115a4e7f53d71d9d905caa2c2dfa4da58899",
+        "version" : "4.13.2"
+      }
+    },
+    {
+      "identity" : "cirruslabs_tart-guest-agent_apple_swift",
+      "kind" : "remoteSourceControl",
+      "location" : "https://buf.build/gen/swift/git/1.28.2-00000000000000-17d7dedafb88.1/cirruslabs_tart-guest-agent_apple_swift.git",
+      "state" : {
+        "revision" : "ccfae5de1917cdb0d7c5000008fa5ed0bad032bf",
+        "version" : "1.28.2-00000000000000-17d7dedafb88.1"
+      }
+    },
+    {
+      "identity" : "cirruslabs_tart-guest-agent_grpc_swift",
+      "kind" : "remoteSourceControl",
+      "location" : "https://buf.build/gen/swift/git/1.24.2-00000000000000-17d7dedafb88.1/cirruslabs_tart-guest-agent_grpc_swift.git",
+      "state" : {
+        "branch" : "1.24.2-00000000000000-17d7dedafb88.1",
+        "revision" : "b8421f137325fe8de737ff5b61238f6f2131b2a8"
       }
     },
     {
@@ -19,13 +37,22 @@
         "revision" : "772883073d044bc754d401cabb6574624eb3778f"
       }
     },
+    {
+      "identity" : "grpc-swift",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/grpc/grpc-swift.git",
+      "state" : {
+        "revision" : "8c5e99d0255c373e0330730d191a3423c57373fb",
+        "version" : "1.24.2"
+      }
+    },
     {
       "identity" : "semaphore",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/groue/Semaphore",
       "state" : {
-        "revision" : "f1c4a0acabeb591068dea6cffdd39660b86dec28",
-        "version" : "0.0.8"
+        "revision" : "2543679282aa6f6c8ecf2138acd613ed20790bc2",
+        "version" : "0.1.0"
       }
     },
     {
@@ -33,8 +60,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/getsentry/sentry-cocoa",
       "state" : {
-        "revision" : "ef4fec9dfb8dd5027b09a4a5c9362feafd118e1a",
-        "version" : "8.24.0"
+        "revision" : "65b3d2a7608685e8d4a37c68fa2c64f28d0b537e",
+        "version" : "8.51.1"
       }
     },
     {
@@ -51,17 +78,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-argument-parser",
       "state" : {
-        "revision" : "46989693916f56d1186bd59ac15124caef896560",
-        "version" : "1.3.1"
-      }
-    },
-    {
-      "identity" : "swift-async-algorithms",
-      "kind" : "remoteSourceControl",
-      "location" : "https://github.com/apple/swift-async-algorithms",
-      "state" : {
-        "branch" : "main",
-        "revision" : "f05e450f0b909c0e80670a47516c4b9700b9e5da"
+        "revision" : "309a47b2b1d9b5e991f36961c983ecec72275be3",
+        "version" : "1.6.1"
       }
     },
     {
@@ -78,8 +96,26 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-collections.git",
       "state" : {
-        "revision" : "f504716c27d2e5d4144fa4794b12129301d17729",
-        "version" : "1.0.3"
+        "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7",
+        "version" : "1.1.4"
+      }
+    },
+    {
+      "identity" : "swift-http-structured-headers",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-http-structured-headers.git",
+      "state" : {
+        "revision" : "db6eea3692638a65e2124990155cd220c2915903",
+        "version" : "1.3.0"
+      }
+    },
+    {
+      "identity" : "swift-http-types",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-http-types.git",
+      "state" : {
+        "revision" : "a0a57e949a8903563aba4615869310c0ebf14c03",
+        "version" : "1.4.0"
       }
     },
     {
@@ -87,8 +123,53 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-log.git",
       "state" : {
-        "revision" : "e97a6fcb1ab07462881ac165fdbb37f067e205d5",
-        "version" : "1.5.4"
+        "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537",
+        "version" : "1.6.1"
+      }
+    },
+    {
+      "identity" : "swift-nio",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-nio.git",
+      "state" : {
+        "revision" : "34d486b01cd891297ac615e40d5999536a1e138d",
+        "version" : "2.83.0"
+      }
+    },
+    {
+      "identity" : "swift-nio-extras",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-nio-extras.git",
+      "state" : {
+        "revision" : "f1f6f772198bee35d99dd145f1513d8581a54f2c",
+        "version" : "1.26.0"
+      }
+    },
+    {
+      "identity" : "swift-nio-http2",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-nio-http2.git",
+      "state" : {
+        "revision" : "4281466512f63d1bd530e33f4aa6993ee7864be0",
+        "version" : "1.36.0"
+      }
+    },
+    {
+      "identity" : "swift-nio-ssl",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-nio-ssl.git",
+      "state" : {
+        "revision" : "4b38f35946d00d8f6176fe58f96d83aba64b36c7",
+        "version" : "2.31.0"
+      }
+    },
+    {
+      "identity" : "swift-nio-transport-services",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-nio-transport-services.git",
+      "state" : {
+        "revision" : "cd1e89816d345d2523b11c55654570acd5cd4c56",
+        "version" : "1.24.0"
       }
     },
     {
@@ -100,13 +181,22 @@
         "version" : "1.0.2"
       }
     },
+    {
+      "identity" : "swift-protobuf",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-protobuf.git",
+      "state" : {
+        "revision" : "ebc7251dd5b37f627c93698e4374084d98409633",
+        "version" : "1.28.2"
+      }
+    },
     {
       "identity" : "swift-retry",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/fumoboy007/swift-retry",
       "state" : {
-        "revision" : "9f133487ffc2ab4539688c29efe57bb1ba31d7b0",
-        "version" : "0.2.3"
+        "revision" : "df9d7b185d2e433147ec0083a73c257e665eea0d",
+        "version" : "0.2.4"
       }
     },
     {
@@ -118,6 +208,24 @@
         "version" : "1.8.0"
       }
     },
+    {
+      "identity" : "swift-system",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-system.git",
+      "state" : {
+        "revision" : "a34201439c74b53f0fd71ef11741af7e7caf01e1",
+        "version" : "1.4.2"
+      }
+    },
+    {
+      "identity" : "swift-xattr",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/jozefizso/swift-xattr",
+      "state" : {
+        "revision" : "f8605af7b3290dbb235fb182ec6e9035d0c8c3ac",
+        "version" : "3.0.0"
+      }
+    },
     {
       "identity" : "swiftdate",
       "kind" : "remoteSourceControl",
@@ -132,8 +240,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/nicklockwood/SwiftFormat",
       "state" : {
-        "revision" : "9df3b01f477163b33d5e63c5e2e5b9f946a49c56",
-        "version" : "0.53.6"
+        "revision" : "ab6844edb79a7b88dc6320e6cee0a0db7674dac3",
+        "version" : "0.54.5"
       }
     },
     {

Package.swift

@@ -10,25 +10,26 @@ let package = Package(
     .executable(name: "tart", targets: ["tart"])
   ],
   dependencies: [
-    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.1"),
+    .package(url: "https://github.com/apple/swift-argument-parser", from: "1.6.1"),
     .package(url: "https://github.com/mhdhejazi/Dynamic", branch: "master"),
     .package(url: "https://github.com/apple/swift-algorithms", from: "1.2.0"),
-    .package(url: "https://github.com/apple/swift-async-algorithms", branch: "main"),
     .package(url: "https://github.com/malcommac/SwiftDate", from: "7.0.0"),
-    .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
+    .package(url: "https://github.com/antlr/antlr4", exact: "4.13.2"),
     .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.2.0")),
     .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.53.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.24.0"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.51.1"),
     .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
     .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.8.0"),
     .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.1"),
     .package(url: "https://github.com/groue/Semaphore", from: "0.0.8"),
     .package(url: "https://github.com/fumoboy007/swift-retry", from: "0.2.3"),
+    .package(url: "https://github.com/jozefizso/swift-xattr", from: "3.0.0"),
+    .package(url: "https://github.com/grpc/grpc-swift.git", .upToNextMajor(from: "1.24.2")),
+    .package(url: "https://buf.build/gen/swift/git/1.24.2-00000000000000-17d7dedafb88.1/cirruslabs_tart-guest-agent_grpc_swift.git", revision: "1.24.2-00000000000000-17d7dedafb88.1"),
   ],
   targets: [
     .executableTarget(name: "tart", dependencies: [
       .product(name: "Algorithms", package: "swift-algorithms"),
-      .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
       .product(name: "ArgumentParser", package: "swift-argument-parser"),
       .product(name: "Dynamic", package: "Dynamic"),
       .product(name: "SwiftDate", package: "SwiftDate"),
@@ -40,6 +41,9 @@ let package = Package(
       .product(name: "SwiftRadix", package: "SwiftRadix"),
       .product(name: "Semaphore", package: "Semaphore"),
       .product(name: "DMRetry", package: "swift-retry"),
+      .product(name: "XAttr", package: "swift-xattr"),
+      .product(name: "GRPC", package: "grpc-swift"),
+      .product(name: "Cirruslabs_TartGuestAgent_Grpc_Swift", package: "cirruslabs_tart-guest-agent_grpc_swift"),
     ], exclude: [
       "OCI/Reference/Makefile",
       "OCI/Reference/Reference.g4",

README.md

@@ -17,23 +17,23 @@ service — a drop-in replacement for the standard GitHub-hosted runners, offeri
   </a>
 </p>
 
-Many companies are using Tart in their internal setups. Here are a few of them:
+Many companies are using Tart in their internal setups. Here are just a few of them:
 
 <p align="center">
-  <a href="https://ahrefs.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/ahrefs.png" height="65"/>
+  <a href="https://atlassian.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Atlassian.png" height="65"/>
   </a>
-  <a href="https://krisp.ai/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
+  <a href="https://www.figma.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Figma.png" height="65"/>
   </a>
   <a href="https://mullvad.net/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Mullvad.png" height="65"/>
   </a>
-  <a href="https://shape.dk/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/shape.png" height="65"/>
+  <a href="https://krisp.ai/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
   </a>
-  <a href="https://suran.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Suran.png" height="65"/>
+  <a href="https://testingbot.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/TestingBot.png" height="65"/>
   </a>
   <a href="https://symflower.com/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Symflower.png" height="65"/>
@@ -41,15 +41,18 @@ Many companies are using Tart in their internal setups. Here are a few of them:
   <a href="https://transloadit.com/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png" height="65"/>
   </a>
-  <a href="https://uphold.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png" height="65"/>
+  <a href="https://cirrus-ci.org/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/CirrusCI.png" height="65"/>
   </a>
   <a href="https://www.pitsdatarecovery.net/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
   </a>
+  <a href="https://expo.dev/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Expo.png" height="65"/>
+  </a>
 </p>
 
-**Note:** If your company or project is using Tart please consider [adding yourself to the list above](/Resources/Users/HowToAddYourself.md).
+**Note:** If your company or project is using Tart please consider [sharing with the community](https://github.com/cirruslabs/tart/discussions/857).
 
 <p align="center">
   <a href="https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws?utm_source=github&utm_medium=referral" target=_blank>
@@ -63,8 +66,8 @@ Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura)
 
 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
-tart run sonoma-base
+tart clone ghcr.io/cirruslabs/macos-tahoe-base:latest tahoe-base
+tart run tahoe-base
 ```
 
 Please check the [official documentation](https://tart.run) for more information and/or feel free to use [discussions](https://github.com/cirruslabs/tart/discussions)

Resources/Info.plist

@@ -2,22 +2,26 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-    <key>CFBundleName</key>
-    <string>tart</string>
-    <key>CFBundleIdentifier</key>
-    <string>org.cirruslabs.tart</string>
-    <key>CFBundleExecutable</key>
-    <string>tart</string>
-    <key>LSBackgroundOnly</key>
-    <string>1</string>
-    <key>CFBundleIconFiles</key>
-    <array>
-        <string>AppIcon.png</string>
-    </array>
-    <key>NSAppTransportSecurity</key>
-    <dict>
-        <key>NSAllowsArbitraryLoads</key>
-        <true/>
-    </dict>
+	<key>CFBundleName</key>
+	<string>Tart</string>
+	<key>CFBundleDisplayName</key>
+	<string>Tart</string>
+	<key>CFBundleIdentifier</key>
+	<string>org.cirruslabs.tart</string>
+	<key>CFBundleExecutable</key>
+	<string>tart</string>
+  <key>LSApplicationCategoryType</key>
+  <string>public.app-category.developer-tools</string>
+	<key>CFBundleIconFiles</key>
+	<array>
+		<string>AppIcon.png</string>
+	</array>
+	<key>NSAppTransportSecurity</key>
+	<dict>
+		<key>NSAllowsArbitraryLoads</key>
+		<true/>
+	</dict>
+  <key>NSLocalNetworkUsageDescription</key>
+  <string>Access to OCI registries on the local network</string>
 </dict>
 </plist>

Resources/Users/HowToAddYourself.md

@@ -1,4 +0,0 @@
-If you'd like to highlight your use of Tart, please create a `456px` by `130px` logo and create a PR
-that adds it to `README.md` in alphabetical order. Don't forget to include a small description of your usage pattern.
-
-You can refer to `Background.png` as a base for your logo.

Sources/tart/Commands/Clone.swift

@@ -9,12 +9,10 @@ struct Clone: AsyncParsableCommand {
     Creates a local virtual machine by cloning either a remote or another local virtual machine.
 
     Due to copy-on-write magic in Apple File System, a cloned VM won't actually claim all the space right away.
-    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
-    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
-    will be modified.
+    Only changes to a cloned disk will be written and claim new space. This also speeds up clones enormously.
 
-    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable. This might be helpful
-    for use cases when the original image is very big and a workload is known to only modify a fraction of the cloned disk.
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the cloned image
+    to fit. This behaviour is called "automatic pruning" and can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
     """
   )
 
@@ -30,6 +28,12 @@ struct Clone: AsyncParsableCommand {
   @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
   var concurrency: UInt = 4
 
+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
+  @Option(help: ArgumentHelp("limit automatic pruning to n gigabytes", valueName: "n"))
+  var pruneLimit: UInt = 100
+
   func validate() throws {
     if newName.contains("/") {
       throw ValidationError("<new-name> should be a local name")
@@ -41,13 +45,13 @@ struct Clone: AsyncParsableCommand {
   }
 
   func run() async throws {
-    let ociStorage = VMStorageOCI()
-    let localStorage = VMStorageLocal()
+    let ociStorage = try VMStorageOCI()
+    let localStorage = try VMStorageLocal()
 
     if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
       // Pull the VM in case it's OCI-based and doesn't exist locally yet
       let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
     }
 
     let sourceVM = try VMStorageHelper.open(sourceName)
@@ -70,10 +74,16 @@ struct Clone: AsyncParsableCommand {
 
       try lock.unlock()
 
-      // APFS is doing copy-on-write so the above cloning operation (just copying files on disk)
+      // APFS is doing copy-on-write, so the above cloning operation (just copying files on disk)
       // is not actually claiming new space until the VM is started and it writes something to disk.
-      // So once we clone the VM let's try to claim a little bit of space for the VM to run.
-      try Prune.reclaimIfNeeded(UInt64(sourceVM.allocatedSizeBytes()), sourceVM)
+      //
+      // So, once we clone the VM let's try to claim the rest of space for the VM to run without errors.
+      let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes()
+      // Avoid reclaiming an excessive amount of disk space.
+      let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024)
+      if reclaimBytes > 0 {
+        try Prune.reclaimIfNeeded(UInt64(reclaimBytes), sourceVM)
+      }
     }, onCancel: {
       try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
     })

Sources/tart/Commands/Create.swift

@@ -10,7 +10,7 @@ struct Create: AsyncParsableCommand {
   @Argument(help: "VM name")
   var name: String
 
-  @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file or URL (or \"latest\", to fetch the latest supported IPSW automatically)", valueName: "path"))
+  @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file or URL (or \"latest\", to fetch the latest supported IPSW automatically)", valueName: "path"), completion: .file())
   var fromIPSW: String?
 
   @Flag(help: "create a Linux VM")
@@ -19,6 +19,9 @@ struct Create: AsyncParsableCommand {
   @Option(help: ArgumentHelp("Disk size in GB"))
   var diskSize: UInt16 = 50
 
+  @Option(help: ArgumentHelp("Disk image format", discussion: "ASIF format provides better performance but requires macOS 26 Tahoe or later"))
+  var diskFormat: DiskImageFormat = .raw
+
   func validate() throws {
     if fromIPSW == nil && !linux {
       throw ValidationError("Please specify either a --from-ipsw or --linux option!")
@@ -28,6 +31,11 @@ struct Create: AsyncParsableCommand {
         throw ValidationError("Only Linux VMs are supported on Intel!")
       }
     #endif
+
+    // Validate disk format support
+    if !diskFormat.isSupported {
+      throw ValidationError("Disk format '\(diskFormat.rawValue)' is not supported on this system.")
+    }
   }
 
   func run() async throws {
@@ -58,12 +66,12 @@ struct Create: AsyncParsableCommand {
             ipswURL = URL(fileURLWithPath: NSString(string: fromIPSW).expandingTildeInPath)
           }
 
-          _ = try await VM(vmDir: tmpVMDir, ipswURL: ipswURL, diskSizeGB: diskSize)
+          _ = try await VM(vmDir: tmpVMDir, ipswURL: ipswURL, diskSizeGB: diskSize, diskFormat: diskFormat)
         }
       #endif
 
       if linux {
-        _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
+        _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize, diskFormat: diskFormat)
       }
 
       try VMStorageLocal().move(name, from: tmpVMDir)

Sources/tart/Commands/Exec.swift

@@ -0,0 +1,225 @@
+import ArgumentParser
+import Foundation
+import NIOPosix
+import GRPC
+import Cirruslabs_TartGuestAgent_Grpc_Swift
+
+struct ExecCustomExitCodeError: Error {
+  let exitCode: Int32
+}
+
+struct Exec: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(abstract: "Execute a command in a running VM", discussion: """
+  Requires Tart Guest Agent running in a guest VM.
+
+  Note that all non-vanilla Cirrus Labs VM images already have the Tart Guest Agent installed.
+  """)
+
+  @Flag(name: [.customShort("i")], help: "Attach host's standard input to a remote command")
+  var interactive: Bool = false
+
+  @Flag(name: [.customShort("t")], help: "Allocate a remote pseudo-terminal (PTY)")
+  var tty: Bool = false
+
+  @Argument(help: "VM name", completion: .custom(completeLocalMachines))
+  var name: String
+
+  @Argument(parsing: .captureForPassthrough, help: "Command to execute")
+  var command: [String]
+
+  func run() async throws {
+    // We only have withThrowingDiscardingTaskGroup available starting from macOS 14
+    if #unavailable(macOS 14) {
+      throw RuntimeError.Generic("\"tart exec\" is only available on macOS 14 (Sonoma) or newer")
+    }
+
+    // Open VM's directory
+    let vmDir = try VMStorageLocal().open(name)
+
+    // Ensure that the VM is running
+    if try !vmDir.running() {
+      throw RuntimeError.VMNotRunning(name)
+    }
+
+    // Create a gRPC channel connected to the VM's control socket
+    let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
+    defer {
+      try! group.syncShutdownGracefully()
+    }
+
+    let channel = try GRPCChannelPool.with(
+      target: .unixDomainSocket(vmDir.controlSocketURL.path()),
+      transportSecurity: .plaintext,
+      eventLoopGroup: group,
+    )
+    defer {
+      try! channel.close().wait()
+    }
+
+    // Switch controlling terminal into raw mode when remote pseudo-terminal is requested
+    var state: State? = nil
+
+    if tty && Term.IsTerminal() {
+      state = try Term.MakeRaw()
+    }
+    defer {
+      // Restore terminal to its initial state
+      if let state {
+        try! Term.Restore(state)
+      }
+    }
+
+    // Execute a command in a running VM
+    do {
+      try await execute(channel)
+    } catch let error as GRPCConnectionPoolError {
+      throw RuntimeError.Generic("Failed to connect to the VM using its control socket: \(error.localizedDescription), is the Tart Guest Agent running?")
+    }
+  }
+
+  private func execute(_ channel: GRPCChannel) async throws {
+    let agentAsyncClient = AgentAsyncClient(channel: channel)
+    let execCall = agentAsyncClient.makeExecCall()
+
+    try await execCall.requestStream.send(.with {
+      $0.type = .command(.with {
+        $0.name = command[0]
+        $0.args = Array(command.dropFirst(1))
+        $0.interactive = interactive
+        $0.tty = tty
+        if tty {
+          $0.terminalSize = .with {
+            let (width, height) = try! Term.GetSize()
+
+            $0.cols = UInt32(width)
+            $0.rows = UInt32(height)
+          }
+        }
+      })
+    })
+
+    // Process command events and optionally send our standard input and/or terminal dimensions
+    try await withThrowingTaskGroup { group in
+      // Stream host's standard input if interactive mode is enabled
+      if interactive {
+        let stdinStream = AsyncThrowingStream<Data, Error> { continuation in
+          let handle = FileHandle.standardInput
+
+          if isRegularFile(handle.fileDescriptor) {
+            // Standard input can be a regular file when input redirection (<) is used,
+            // in which case the handle won't receive any new readability events, so we
+            // just read the file normally here in chunks and consider done with it
+            //
+            // Ideally this is best handled by using non-blocking I/O, but Swift's
+            // standard library only offers inefficient bytes[1] property and SwiftNIO's
+            // NIOFileSystem doesn't seem to support opening raw file descriptors.
+            //
+            // [1]: https://developer.apple.com/documentation/foundation/filehandle/bytes
+            while true {
+              do {
+                let data = try handle.read(upToCount: 64 * 1024)
+                if let data = data {
+                  continuation.yield(data)
+                } else {
+                  continuation.finish()
+                  break
+                }
+              } catch (let error) {
+                continuation.finish(throwing: error)
+                break
+              }
+            }
+          } else {
+            handle.readabilityHandler = { handle in
+              let data = handle.availableData
+
+              if data.isEmpty {
+                continuation.finish()
+              } else {
+                continuation.yield(data)
+              }
+            }
+          }
+        }
+
+        group.addTask {
+          for try await stdinData in stdinStream {
+            try await execCall.requestStream.send(.with {
+              $0.type = .standardInput(.with {
+                $0.data = stdinData
+              })
+            })
+          }
+
+          // Signal EOF as we're done reading standard input
+          try await execCall.requestStream.send(.with {
+            $0.type = .standardInput(.with {
+              $0.data = Data()
+            })
+          })
+        }
+      }
+
+      // Stream host's terminal dimensions if pseudo-terminal is requested
+      signal(SIGWINCH, SIG_IGN)
+      let sigwinchSrc = DispatchSource.makeSignalSource(signal: SIGWINCH)
+      sigwinchSrc.activate()
+
+      if tty {
+        let terminalDimensionsStream = AsyncStream { continuation in
+          sigwinchSrc.setEventHandler {
+            continuation.yield(try! Term.GetSize())
+          }
+        }
+
+        group.addTask {
+          for await (width, height) in terminalDimensionsStream {
+            try await execCall.requestStream.send(.with {
+              $0.type = .terminalResize(.with {
+                $0.cols = UInt32(width)
+                $0.rows = UInt32(height)
+              })
+            })
+          }
+        }
+      }
+
+      // Process command events
+      group.addTask {
+        for try await response in execCall.responseStream {
+          switch response.type {
+          case .standardOutput(let ioChunk):
+            try FileHandle.standardOutput.write(contentsOf: ioChunk.data)
+          case .standardError(let ioChunk):
+            try FileHandle.standardError.write(contentsOf: ioChunk.data)
+          case .exit(let exit):
+            throw ExecCustomExitCodeError(exitCode: exit.code)
+          default:
+            // Unknown event, do nothing
+            continue
+          }
+        }
+      }
+
+      while !group.isEmpty {
+        do {
+          try await group.next()
+        } catch {
+          group.cancelAll()
+
+          throw error
+        }
+      }
+    }
+  }
+}
+
+private func isRegularFile(_ fileDescriptor: Int32) -> Bool {
+  var stat = stat()
+
+  if fstat(fileDescriptor, &stat) != 0 {
+    return false
+  }
+
+  return (stat.st_mode & S_IFMT) == S_IFREG
+}

Sources/tart/Commands/Export.swift

@@ -7,7 +7,7 @@ struct Export: AsyncParsableCommand {
   @Argument(help: "Source VM name.", completion: .custom(completeMachines))
   var name: String
 
-  @Argument(help: "Path to the destination file.")
+  @Argument(help: "Path to the destination file.", completion: .file())
   var path: String?
 
   func run() async throws {

Sources/tart/Commands/Get.swift

@@ -6,6 +6,7 @@ fileprivate struct VMInfo: Encodable {
   let CPU: Int
   let Memory: UInt64
   let Disk: Int
+  let DiskFormat: String
   let Size: String
   let Display: String
   let Running: Bool
@@ -26,7 +27,7 @@ struct Get: AsyncParsableCommand {
     let vmConfig = try VMConfig(fromURL: vmDir.configURL)
     let memorySizeInMb = vmConfig.memorySize / 1024 / 1024
 
-    let info = VMInfo(OS: vmConfig.os, CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000), Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state().rawValue)
+    let info = VMInfo(OS: vmConfig.os, CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), DiskFormat: vmConfig.diskFormat.rawValue, Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000), Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state().rawValue)
     print(format.renderSingle(info))
   }
 }

Sources/tart/Commands/IP.swift

@@ -5,9 +5,9 @@ import SystemConfiguration
 import Sentry
 
 enum IPResolutionStrategy: String, ExpressibleByArgument, CaseIterable {
-  case dhcp, arp
+  case dhcp, arp, agent
 
-  private(set) static var allValueStrings: [String] = Format.allCases.map { "\($0)"}
+  private(set) static var allValueStrings: [String] = Self.allCases.map { "\($0)"}
 }
 
 struct IP: AsyncParsableCommand {
@@ -19,13 +19,11 @@ struct IP: AsyncParsableCommand {
   @Option(help: "Number of seconds to wait for a potential VM booting")
   var wait: UInt16 = 0
 
-  @Option(help: ArgumentHelp("Strategy for resolving IP address: dhcp or arp",
+  @Option(help: ArgumentHelp("Strategy for resolving IP address",
                              discussion: """
-                             By default, Tart is looking up and parsing DHCP lease file to determine the IP of the VM.\n
-                             This method is fast and the most reliable but only returns local IP adresses.\n
-                             Alternatively, Tart can call external `arp` executable and parse it's output.\n
-                             In case of enabled Bridged Networking this method will return VM's IP address on the network interface used for Bridged Networking.\n
-                             Note that `arp` strategy won't work for VMs using `--net-softnet`.
+                             By default, Tart is using a "dhcp" resolver which parses the DHCP lease file on host and tries to find an entry containing the VM's MAC address. This method is fast and the most reliable, but only works for VMs are not using the bridged networking.\n
+                             Alternatively, Tart has an "arp" resolver which calls an external "arp" executable and parses it's output. This works for VMs using bridged networking and returns their IP, but when they generate enough network activity to populate the host's ARP table. Note that "arp" strategy won't work for VMs using the Softnet networking.\n
+                             A third strategy, "agent" works in all cases reliably, but requires Guest agent for Tart VMs (https://github.com/cirruslabs/tart-guest-agent) to be installed inside of a VM.
                              """))
   var resolver: IPResolutionStrategy = .dhcp
 
@@ -34,14 +32,16 @@ struct IP: AsyncParsableCommand {
     let vmConfig = try VMConfig.init(fromURL: vmDir.configURL)
     let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!
 
-    guard let ip = try await IP.resolveIP(vmMACAddress, resolutionStrategy: resolver, secondsToWait: wait) else {
+    guard let ip = try await IP.resolveIP(vmMACAddress, resolutionStrategy: resolver, secondsToWait: wait, controlSocketURL: vmDir.controlSocketURL) else {
       var message = "no IP address found"
 
       if try !vmDir.running() {
         message += ", is your VM running?"
       }
 
-      if (vmConfig.os == .linux && resolver == .arp) {
+      if (resolver == .agent) {
+        message += " (also make sure that Guest agent for Tart is running inside of a VM)"
+      } else if (vmConfig.os == .linux && resolver == .arp) {
         message += " (not all Linux distributions are compatible with the ARP resolver)"
       }
 
@@ -51,7 +51,7 @@ struct IP: AsyncParsableCommand {
     print(ip)
   }
 
-  static public func resolveIP(_ vmMACAddress: MACAddress, resolutionStrategy: IPResolutionStrategy = .dhcp, secondsToWait: UInt16 = 0) async throws -> IPv4Address? {
+  static public func resolveIP(_ vmMACAddress: MACAddress, resolutionStrategy: IPResolutionStrategy = .dhcp, secondsToWait: UInt16 = 0, controlSocketURL: URL? = nil) async throws -> IPv4Address? {
     let waitUntil = Calendar.current.date(byAdding: .second, value: Int(secondsToWait), to: Date.now)!
 
     repeat {
@@ -64,6 +64,14 @@ struct IP: AsyncParsableCommand {
         if let leases = try Leases(), let ip = leases.ResolveMACAddress(macAddress: vmMACAddress) {
           return ip
         }
+      case .agent:
+        guard let controlSocketURL = controlSocketURL else {
+          throw RuntimeError.Generic("Cannot perform IP resolution via Tart Guest Agent when control socket URL is not set")
+        }
+
+        if let ip = try await AgentResolver.ResolveIP(controlSocketURL) {
+          return ip
+        }
       }
 
       // wait a second

Sources/tart/Commands/Import.swift

@@ -4,10 +4,10 @@ import Foundation
 struct Import: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Import VM from a compressed .tvm file")
 
-  @Argument(help: "Path to a file created with \"tart export\".")
+  @Argument(help: "Path to a file created with \"tart export\".", completion: .file())
   var path: String
 
-  @Argument(help: "Destination VM name.")
+  @Argument(help: "Destination VM name.", completion: .custom(completeLocalMachines))
   var name: String
 
   func validate() throws {
@@ -17,7 +17,7 @@ struct Import: AsyncParsableCommand {
   }
 
   func run() async throws {
-    let localStorage = VMStorageLocal()
+    let localStorage = try VMStorageLocal()
 
     // Create a temporary VM directory to which we will load the export file
     let tmpVMDir = try VMDirectory.temporary()

Sources/tart/Commands/List.swift

@@ -7,6 +7,7 @@ fileprivate struct VMInfo: Encodable {
   let Name: String
   let Disk: Int
   let Size: Int
+  let SizeOnDisk: Int
   let Running: Bool
   let State: String
 }
@@ -17,7 +18,7 @@ struct List: AsyncParsableCommand {
   @Option(help: ArgumentHelp("Only display VMs from the specified source (e.g. --source local, --source oci)."))
   var source: String?
 
-  @Option(help: "Output format: text or json")
+  @Option(help: "Output format: text or json", completion: .list(["text", "json"]))
   var format: Format = .text
 
   @Flag(name: [.short, .long], help: ArgumentHelp("Only display VM names."))
@@ -38,13 +39,13 @@ struct List: AsyncParsableCommand {
 
     if source == nil || source == "local" {
       infos += sortedInfos(try VMStorageLocal().list().map { (name, vmDir) in
-        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
+        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), SizeOnDisk: vmDir.allocatedSizeGB() - vmDir.deduplicatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
       })
     }
 
     if source == nil || source == "oci" {
       infos += sortedInfos(try VMStorageOCI().list().map { (name, vmDir, _) in
-        try VMInfo(Source: "OCI", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
+        try VMInfo(Source: "OCI", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), SizeOnDisk: vmDir.allocatedSizeGB() - vmDir.deduplicatedSizeGB(), Running: vmDir.running(), State: vmDir.state().rawValue)
       })
     }
 

Sources/tart/Commands/Login.swift

@@ -49,9 +49,10 @@ struct Login: AsyncParsableCommand {
     ])
 
     if !noValidate {
+      let registry = try Registry(host: host, namespace: "", insecure: insecure,
+                                  credentialsProviders: [credentialsProvider])
+
       do {
-        let registry = try Registry(host: host, namespace: "", insecure: insecure,
-                                    credentialsProviders: [credentialsProvider])
         try await registry.ping()
       } catch {
         throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
@@ -63,6 +64,8 @@ struct Login: AsyncParsableCommand {
 }
 
 fileprivate class DictionaryCredentialsProvider: CredentialsProvider {
+  let userFriendlyName = "static dictionary credentials provider"
+
   var credentials: Dictionary<String, (String, String)>
 
   init(_ credentials: Dictionary<String, (String, String)>) {

Sources/tart/Commands/Prune.swift

@@ -7,7 +7,7 @@ import SwiftDate
 struct Prune: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches or local VMs")
 
-  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."))
+  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."), completion: .list(["caches", "vms"]))
   var entries: String = "caches"
 
   @Option(help: ArgumentHelp("Remove entries that were last accessed more than n days ago",
@@ -53,9 +53,9 @@ struct Prune: AsyncParsableCommand {
 
     switch entries {
     case "caches":
-      prunableStorages = [VMStorageOCI(), try IPSWCache()]
+      prunableStorages = [try VMStorageOCI(), try IPSWCache()]
     case "vms":
-      prunableStorages = [VMStorageLocal()]
+      prunableStorages = [try VMStorageLocal()]
     default:
       throw ValidationError("unsupported --entries value, please specify either \"caches\" or \"vms\"")
     }
@@ -152,7 +152,7 @@ struct Prune: AsyncParsableCommand {
     let transaction = SentrySDK.startTransaction(name: "Pruning cache", operation: "prune", bindToScope: true)
     defer { transaction.finish() }
 
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
+    let prunableStorages: [PrunableStorage] = [try VMStorageOCI(), try IPSWCache()]
     let prunables: [Prunable] = try prunableStorages
       .flatMap { try $0.prunables() }
       .sorted { try $0.accessDate() < $1.accessDate() }

Sources/tart/Commands/Pull.swift

@@ -9,8 +9,8 @@ struct Pull: AsyncParsableCommand {
     Pulls a virtual machine from a remote OCI-compatible registry. Supports authorization via Keychain (see "tart login --help"),
     Docker credential helpers defined in ~/.docker/config.json or via TART_REGISTRY_USERNAME/TART_REGISTRY_PASSWORD environment variables.
 
-    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image to fit via "tart prune".
-    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image
+    to fit. This behaviour is called "automatic pruning" and can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
     """
   )
 
@@ -23,6 +23,9 @@ struct Pull: AsyncParsableCommand {
   @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
   var concurrency: UInt = 4
 
+  @Flag(help: .hidden)
+  var deduplicate: Bool = false
+
   func validate() throws {
     if concurrency < 1 {
       throw ValidationError("network concurrency cannot be less than 1")
@@ -32,7 +35,7 @@ struct Pull: AsyncParsableCommand {
   func run() async throws {
     // Be more liberal when accepting local image as argument,
     // see https://github.com/cirruslabs/tart/issues/36
-    if VMStorageLocal().exists(remoteName) {
+    if try VMStorageLocal().exists(remoteName) {
       print("\"\(remoteName)\" is a local image, nothing to pull here!")
 
       return
@@ -43,6 +46,6 @@ struct Pull: AsyncParsableCommand {
 
     defaultLogger.appendNewLine("pulling \(remoteName)...")
 
-    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
   }
 }

Sources/tart/Commands/Push.swift

@@ -15,6 +15,9 @@ struct Push: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Option(help: "network concurrency to use when pushing a local VM to the OCI-compatible registry")
+  var concurrency: UInt = 4
+
   @Option(help: ArgumentHelp("chunk size in MB if registry supports chunked uploads",
                              discussion: """
                              By default monolithic method is used for uploading blobs to the registry but some registries support a more efficient chunked method.
@@ -23,6 +26,11 @@ struct Push: AsyncParsableCommand {
                              """))
   var chunkSize: Int = 0
 
+
+  @Option(name: [.customLong("label")], help: ArgumentHelp("additional metadata to attach to the OCI image configuration in key=value format",
+                                                           discussion: "Can be specified multiple times to attach multiple labels."))
+  var labels: [String] = []
+
   @Option(help: .hidden)
   var diskFormat: String = "v2"
 
@@ -31,8 +39,12 @@ struct Push: AsyncParsableCommand {
   var populateCache: Bool = false
 
   func run() async throws {
-    let ociStorage = VMStorageOCI()
+    let ociStorage = try VMStorageOCI()
     let localVMDir = try VMStorageHelper.open(localName)
+    let lock = try localVMDir.lock()
+    if try !lock.trylock() {
+      throw RuntimeError.VMIsRunning(localName)
+    }
 
     // Parse remote names supplied by the user
     let remoteNames = try remoteNames.map{
@@ -73,7 +85,9 @@ struct Push: AsyncParsableCommand {
           registry: registry,
           references: references,
           chunkSizeMb: chunkSize,
-          diskFormat: diskFormat
+          diskFormat: diskFormat,
+          concurrency: concurrency,
+          labels: parseLabels()
         )
         // Populate the local cache (if requested)
         if populateCache {
@@ -107,6 +121,28 @@ struct Push: AsyncParsableCommand {
     return RemoteName(host: registry.host!, namespace: registry.namespace,
                       reference: Reference(digest: digest))
   }
+
+  // Helper method to convert labels array to dictionary
+  func parseLabels() -> [String: String] {
+    var result = [String: String]()
+
+    for label in labels {
+      let parts = label.trimmingCharacters(in: .whitespaces).split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false)
+
+      let key = parts.count > 0 ? String(parts[0]) : ""
+      let value = parts.count > 1 ? String(parts[1]) : ""
+
+      // It sometimes makes sense to provide an empty value,
+      // but not an empty key
+      if key.isEmpty {
+        continue
+      }
+
+      result[key] = value
+    }
+
+    return result
+  }  
 }
 
 extension Collection where Element == RemoteName {

Sources/tart/Commands/Rename.swift

@@ -17,7 +17,7 @@ struct Rename: AsyncParsableCommand {
   }
 
   func run() async throws {
-    let localStorage = VMStorageLocal()
+    let localStorage = try VMStorageLocal()
 
     if !localStorage.exists(name) {
       throw ValidationError("failed to rename a non-existent local VM: \(name)")

Sources/tart/Commands/Run.swift

@@ -12,6 +12,56 @@ var vm: VM?
 struct IPNotFound: Error {
 }
 
+@available(macOS 14, *)
+extension VZDiskSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
+extension VZDiskImageSynchronizationMode {
+  public init(_ description: String) throws {
+    switch description {
+    case "none":
+      self = .none
+    case "fsync":
+      self = .fsync
+    case "full":
+      self = .full
+    case "":
+      self = .full
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image synchronization mode: \"\(description)\"")
+    }
+  }
+}
+
+extension VZDiskImageCachingMode {
+  public init?(_ description: String) throws {
+    switch description {
+    case "automatic":
+      self = .automatic
+    case "cached":
+      self = .cached
+    case "uncached":
+      self = .uncached
+    case "":
+      return nil
+    default:
+      throw RuntimeError.VMConfigurationError("unsupported disk image caching mode: \"\(description)\"")
+    }
+  }
+}
+
 struct Run: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Run a VM")
 
@@ -31,7 +81,7 @@ struct Run: AsyncParsableCommand {
   @Option(help: ArgumentHelp(
     "Attach an externally created serial console",
     discussion: "Alternative to `--serial` flag for programmatic integrations."
-  ))
+  ), completion: .file())
   var serialPath: String?
 
   @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .private))
@@ -42,7 +92,7 @@ struct Run: AsyncParsableCommand {
 
   @Flag(help: ArgumentHelp(
     "Disable clipboard sharing between host and guest.",
-    discussion: "Only works with Linux-based guest operating systems."))
+    discussion: "Clipboard sharing requires spice-vdagent package on Linux and https://github.com/cirruslabs/tart-guest-agent on macOS."))
   var noClipboard: Bool = false
 
   #if arch(arm64)
@@ -67,19 +117,29 @@ struct Run: AsyncParsableCommand {
   var vncExperimental: Bool = false
 
   @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\" --disk=\"/dev/disk0\" --disk=\"nbd://localhost:10809/myDisk\")
+  Additional disk attachments with an optional read-only and synchronization options in the form of path[:options] (e.g. --disk="disk.bin", --disk="ubuntu.iso:ro", --disk="/dev/disk0", --disk "ghcr.io/cirruslabs/xcode:16.0:ro" or --disk="nbd://localhost:10809/myDisk:sync=none")
   """, discussion: """
-  Can be either a disk image file, a block device like a local SSD on AWS EC2 Mac instances or a Network Block Device (NBD).
+  The disk attachment can be a:
 
-  Learn how to create a disk image using Disk Utility here:
-  https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+  * path to a disk image file
+  * path to a block device (for example, a local SSD on AWS EC2 Mac instances)
+  * remote VM name whose disk will be mounted
+  * Network Block Device (NBD) URL
+
+  Options are comma-separated and are as follows:
+
+  * ro — attach the specified disk in read-only mode instead of the default read-write (e.g. --disk="disk.img:ro")
+
+  * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --disk="disk.img:sync=none")
+
+  Learn how to create a disk image using Disk Utility here: https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
 
   To work with block devices, the easiest way is to modify their permissions (e.g. by using "sudo chown $USER /dev/diskX") or to run the Tart binary as root, which affects locating Tart VMs.
 
   To work around this pass TART_HOME explicitly:
 
-  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
-  """, valueName: "path[:ro]"))
+  sudo TART_HOME="$HOME/.tart" tart run sequoia --disk=/dev/disk0
+  """, valueName: "path[:options]"), completion: .file())
   var disk: [String] = []
 
   #if arch(arm64)
@@ -98,7 +158,7 @@ struct Run: AsyncParsableCommand {
   #endif
   var rosettaTag: String?
 
-  @Option(help: ArgumentHelp("Additional directory shares with an optional read-only and mount tag options (e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")", discussion: """
+  @Option(help: ArgumentHelp("Additional directory shares with an optional read-only and mount tag options in the form of [name:]path[:options] (e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")", discussion: """
   Requires host to be macOS 13.0 (Ventura) or newer. macOS guests must be running macOS 13.0 (Ventura) or newer too.
 
   Options are comma-separated and are as follows:
@@ -110,9 +170,12 @@ struct Run: AsyncParsableCommand {
   Mount tag can be overridden by appending tag property to the directory share (e.g. --dir=\"~/src/build:tag=build\" or --dir=\"~/src/build:ro,tag=build\"). Then it can be mounted via "mount_virtiofs build ~/build" inside guest macOS and "mount -t virtiofs build ~/build" inside guest Linux.
 
   In case of passing multiple directories per mount tag it is required to prefix them with names e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\". These names will be used as directory names under the mounting point inside guests. For the example above it will be "/Volumes/My Shared Files/build" and "/Volumes/My Shared Files/sources" respectively.
-  """, valueName: "[name:]path[:options]"))
+  """, valueName: "[name:]path[:options]"), completion: .directory)
   var dir: [String] = []
 
+  @Flag(help: ArgumentHelp("Enable nested virtualization if possible"))
+  var nested: Bool = false
+
   @Option(help: ArgumentHelp("""
   Use bridged networking instead of the default shared (NAT) networking \n(e.g. --net-bridged=en0 or --net-bridged=\"Wi-Fi\")
   """, discussion: """
@@ -120,16 +183,72 @@ struct Run: AsyncParsableCommand {
   """, valueName: "interface name"))
   var netBridged: [String] = []
 
-  @Flag(help: ArgumentHelp("Use software networking instead of the default shared (NAT) networking",
-                           discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
+  @Flag(help: ArgumentHelp("Use software networking provided by Softnet instead of the default shared (NAT) networking",
+                           discussion: """
+                           Softnet provides better network isolation and alleviates DHCP shortage on production systems. Tart invokes Softnet when this option is specified as a sub-process and communicates with it over socketpair(2).
+
+                           It is essentially a userspace packet filter which restricts the VM networking and prevents a class of security issues, such as ARP spoofing. By default, the VM will only be able to:
+
+                           * send traffic from its own MAC-address
+                           * send traffic from the IP-address assigned to it by the DHCP
+                           * send traffic to globally routable IPv4 addresses
+                           * send traffic to gateway IP of the vmnet bridge (this would normally be \"bridge100\" interface)
+                           * receive any incoming traffic
+
+                           In addition, Softnet tunes macOS built-in DHCP server to decrease its lease time from the default 86,400 seconds (one day) to 600 seconds (10 minutes). This is especially important when you use Tart to clone and run a lot of ephemeral VMs over a period of one day.
+
+                           More on Softnet here: https://github.com/cirruslabs/softnet
+                           """))
   var netSoftnet: Bool = false
 
-  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", valueName: "comma-separated CIDRs"))
+  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", discussion: """
+  This option allows you bypass the private IPv4 address space restrictions imposed by --net-softnet.
+
+  For example, you can allow the VM to communicate with the local network with e.g. --net-softnet-allow=10.0.0.0/16 or to completely disable the destination based restrictions with --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
+  """, valueName: "comma-separated CIDRs"))
   var netSoftnetAllow: String?
 
+  @Option(help: ArgumentHelp("Comma-separated list of TCP ports to expose (e.g. --net-softnet-expose 2222:22,8080:80)", discussion: """
+  Options are comma-separated and are as follows:
+
+  * EXTERNAL_PORT:INTERNAL_PORT — forward TCP traffic from the EXTERNAL_PORT on a host's egress interface (automatically detected and could be Wi-Fi, Ethernet and a VPN interface) to the INTERNAL_PORT on guest's IP (as reported by "tart ip")
+
+  Note that for the port forwarding to work correctly:
+
+  * the software in guest listening on INTERNAL_PORT should either listen on 0.0.0.0 or on an IP address assigned to that guest
+  * connection to the EXTERNAL_PORT should be performed from the local network that the host is attached to or from the internet, it's not possible to connect to that forwarded port from the host itself
+
+  Another thing to keep in mind is that regular Softnet restrictions will still apply even to port forwarding. So if you're planning to access your VM from local network, and your local network is 192.168.0.0/24, for example, then add --net-softnet-allow=192.168.0.0/24. If you only need port forwarding, to completely disable Softnet restrictions you can use --net-softnet-allow=0.0.0.0/0.
+
+  Implies --net-softnet.
+  """, valueName: "comma-separated port specifications"))
+  var netSoftnetExpose: String?
+
   @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
   var netHost: Bool = false
 
+  @Option(help: ArgumentHelp("Set the root disk options (e.g. --root-disk-opts=\"ro\" or --root-disk-opts=\"caching=cached,sync=none\")",
+                             discussion: """
+                             Options are comma-separated and are as follows:
+
+                             * ro — attach the root disk in read-only mode instead of the default read-write (e.g. --root-disk-opts="ro")
+
+                             * sync=none — disable data synchronization with the permanent storage to increase performance at the cost of a higher chance of data loss (e.g. --root-disk-opts="sync=none")
+
+                             * sync=fsync — enable data synchronization with the permanent storage, but don't ensure that it was actually written (e.g. --root-disk-opts="sync=fsync")
+
+                             * sync=full — enable data synchronization with the permanent storage and ensure that it was actually written (e.g. --root-disk-opts="sync=full")
+
+                             * caching=automatic — allows the virtualization framework to automatically determine whether to enable data caching
+
+                             * caching=cached — enabled data caching
+
+                             * caching=uncached — disables data caching
+                             """, valueName: "options"))
+  var rootDiskOpts: String = ""
+
   #if arch(arm64)
     @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
   #endif
@@ -141,12 +260,22 @@ struct Run: AsyncParsableCommand {
   #endif
   var captureSystemKeys: Bool = false
 
+  #if arch(arm64)
+    @Flag(help: ArgumentHelp("Don't add trackpad as a pointing device on macOS VMs"))
+  #endif
+  var noTrackpad: Bool = false
+
   mutating func validate() throws {
     if vnc && vncExperimental {
       throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
     }
 
-    // check that not more than one network option is specified
+    // Automatically enable --net-softnet when any of its related options are specified
+    if netSoftnetAllow != nil || netSoftnetExpose != nil {
+      netSoftnet = true
+    }
+
+    // Check that no more than one network option is specified
     var netFlags = 0
     if netBridged.count > 0 { netFlags += 1 }
     if netSoftnet { netFlags += 1 }
@@ -164,7 +293,15 @@ struct Run: AsyncParsableCommand {
       throw ValidationError("--captures-system-keys can only be used with the default VM view")
     }
 
-    let localStorage = VMStorageLocal()
+    if nested {
+      if #unavailable(macOS 15) {
+        throw ValidationError("Nested virtualization is supported on hosts starting with macOS 15 (Sequoia), and later.")
+      } else if !VZGenericPlatformConfiguration.isNestedVirtualizationSupported {
+        throw ValidationError("Nested virtualization is available for Mac with the M3 chip, and later.")
+      }
+    }
+
+    let localStorage = try VMStorageLocal()
     let vmDir = try localStorage.open(name)
     if try vmDir.state() == .Suspended {
       suspendable = true
@@ -175,8 +312,16 @@ struct Run: AsyncParsableCommand {
       if !(config.platform is PlatformSuspendable) {
         throw ValidationError("You can only suspend macOS VMs")
       }
-      if dir.count > 0 {
-        throw ValidationError("Suspending VMs with shared directories is not supported")
+
+      if noTrackpad {
+        throw ValidationError("--no-trackpad cannot be used with --suspendable")
+      }
+    }
+
+    if noTrackpad {
+      let config = try VMConfig.init(fromURL: vmDir.configURL)
+      if config.os != .darwin {
+        throw ValidationError("--no-trackpad can only be used with macOS VMs")
       }
     }
 
@@ -189,9 +334,15 @@ struct Run: AsyncParsableCommand {
 
   @MainActor
   func run() async throws {
-    let localStorage = VMStorageLocal()
+    let localStorage = try VMStorageLocal()
     let vmDir = try localStorage.open(name)
 
+    // Validate disk format support
+    let vmConfig = try VMConfig(fromURL: vmDir.configURL)
+    if !vmConfig.diskFormat.isSupported {
+      throw ValidationError("Disk format '\(vmConfig.diskFormat.rawValue)' is not supported on this system.")
+    }
+
     let storageLock = try FileLock(lockURL: Config().tartHomeDir)
     try storageLock.lock()
     // check if there is a running VM with the same MAC address
@@ -209,8 +360,6 @@ struct Run: AsyncParsableCommand {
       try Softnet.configureSUIDBitIfNeeded()
     }
 
-    let additionalDiskAttachments = try additionalDiskAttachments()
-
     var serialPorts: [VZSerialPortConfiguration] = []
     if serial {
       let tty_fd = createPTY()
@@ -229,15 +378,22 @@ struct Run: AsyncParsableCommand {
       serialPorts.append(createSerialPortConfiguration(tty_read!, tty_write!))
     }
 
+    // Parse root disk options
+    let diskOptions = DiskOptions(rootDiskOpts)
+
     vm = try VM(
       vmDir: vmDir,
       network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalStorageDevices: additionalDiskAttachments,
+      additionalStorageDevices: try additionalDiskAttachments(),
       directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
       serialPorts: serialPorts,
       suspendable: suspendable,
+      nested: nested,
       audio: !noAudio,
-      clipboard: !noClipboard
+      clipboard: !noClipboard,
+      sync: VZDiskImageSynchronizationMode(diskOptions.syncModeRaw),
+      caching: VZDiskImageCachingMode(diskOptions.cachingModeRaw),
+      noTrackpad: noTrackpad
     )
 
     let vncImpl: VNC? = try {
@@ -287,7 +443,35 @@ struct Run: AsyncParsableCommand {
           }
         #endif
 
-        try await vm!.start(recovery: recovery, resume: resume)
+        do {
+          try await vm!.start(recovery: recovery, resume: resume)
+        } catch let error as VZError {
+          if error.code == .virtualMachineLimitExceeded {
+            var hint = ""
+
+            do {
+              let runningVMs: [String] = try localStorage.list().compactMap { (name, vmDir) in
+                if try !vmDir.running() {
+                  return nil
+                }
+
+                return name
+              }
+
+              if !runningVMs.isEmpty {
+                let runningVMsJoined = runningVMs.joined(separator: ", ")
+
+                hint = " (other running VMs: \(runningVMsJoined))"
+              }
+            } catch {
+              // we can't provide any hint
+            }
+
+            throw RuntimeError.VirtualMachineLimitExceeded(hint)
+          }
+
+          throw error
+        }
 
         if let vncImpl = vncImpl {
           let vncURL = try await vncImpl.waitForURL(netBridged: !netBridged.isEmpty)
@@ -300,6 +484,12 @@ struct Run: AsyncParsableCommand {
           }
         }
 
+        if #available(macOS 14, *) {
+          Task {
+            try await ControlSocket(vmDir.controlSocketURL).run()
+          }
+        }
+
         try await vm!.run()
 
         if let vncImpl = vncImpl {
@@ -359,10 +549,24 @@ struct Run: AsyncParsableCommand {
     }
     sigusr1Src.activate()
 
+    // Gracefull shutdown support. For macOS this brings up a dialog,
+    // asking the user if they are sure they want to shut down.
+    signal(SIGUSR2, SIG_IGN)
+    let sigusr2Src = DispatchSource.makeSignalSource(signal: SIGUSR2)
+    sigusr2Src.setEventHandler {
+      Task {
+        print("Requesting guest OS to stop...")
+        try vm!.virtualMachine.requestStop()
+      }
+    }
+    sigusr2Src.activate()
+
     let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
     if noGraphics || useVNCWithoutGraphics {
-      // enter the main even loop, without bringing up any UI,
-      // and just wait for the VM to exit.
+      // Enter the main event loop without bringing up any UI,
+      // waiting for the VM to exit.
+      NSApplication.shared.setActivationPolicy(.prohibited)
+
       NSApplication.shared.run()
     } else {
       runUI(suspendable, captureSystemKeys)
@@ -384,21 +588,26 @@ struct Run: AsyncParsableCommand {
   }
 
   func userSpecifiedNetwork(vmDir: VMDirectory) throws -> Network? {
+    var softnetExtraArguments: [String] = []
+
+    if let netSoftnetAllow = netSoftnetAllow {
+      softnetExtraArguments += ["--allow", netSoftnetAllow]
+    }
+
+    if let netSoftnetExpose = netSoftnetExpose {
+      softnetExtraArguments += ["--expose", netSoftnetExpose]
+    }
+
     if netSoftnet {
       let config = try VMConfig.init(fromURL: vmDir.configURL)
 
-      var extraArguments: [String] = []
-
-      if let netSoftnetAllow = netSoftnetAllow {
-        extraArguments += ["--allow", netSoftnetAllow]
-      }
-
-      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: extraArguments)
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: softnetExtraArguments)
     }
 
     if netHost {
       let config = try VMConfig.init(fromURL: vmDir.configURL)
-      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"])
+
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"] + softnetExtraArguments)
     }
 
     if netBridged.count > 0 {
@@ -432,71 +641,9 @@ struct Run: AsyncParsableCommand {
   }
 
   func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
-    var result: [VZStorageDeviceConfiguration] = []
-    let readOnlySuffix = ":ro"
-    let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
-
-    for rawDisk in expandedDiskPaths {
-      let diskReadOnly = rawDisk.hasSuffix(readOnlySuffix)
-      let diskPath = diskReadOnly ? String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count)) : rawDisk
-
-      let diskURL = URL(string: diskPath)
-      if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
-        guard #available(macOS 14, *) else {
-          throw UnsupportedOSError("attaching Network Block Devices", "are")
-        }
-
-        let nbdAttachment = try VZNetworkBlockDeviceStorageDeviceAttachment(
-          url: diskURL!,
-          timeout: 30,
-          isForcedReadOnly: diskReadOnly,
-          synchronizationMode: VZDiskSynchronizationMode.none
-        )
-        result.append(VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment))
-        continue
-      }
-
-      let diskFileURL = URL(fileURLWithPath: diskPath)
-
-      if pathHasMode(diskPath, mode: S_IFBLK) {
-        guard #available(macOS 14, *) else {
-          throw UnsupportedOSError("attaching block devices", "are")
-        }
-
-        let fd = open(diskPath, diskReadOnly ? O_RDONLY : O_RDWR)
-        if fd == -1 {
-          let details = Errno(rawValue: CInt(errno))
-
-          switch details.rawValue {
-          case EBUSY:
-            throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "already in use, try umounting it via \"diskutil unmountDisk\" (when the whole disk) or \"diskutil umount\" (when mounting a single partition)")
-          case EACCES:
-            throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "permission denied, consider changing the disk's owner using \"sudo chown $USER \(diskFileURL.url.path)\" or run Tart as a superuser (see --disk help for more details on how to do that correctly)")
-          default:
-            throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "\(details)")
-          }
-        }
-
-        let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
-                                                                           readOnly: diskReadOnly, synchronizationMode: .full)
-        result.append(VZVirtioBlockDeviceConfiguration(attachment: blockAttachment))
-        continue
-      }
-
-      // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-      // see https://github.com/cirruslabs/tart/issues/323 for more details.
-      if try !diskReadOnly && !FileLock(lockURL: diskFileURL).trylock() {
-        throw RuntimeError.DiskAlreadyInUse("disk \(diskFileURL.url.path) seems to be already in use, unmount it first in Finder")
-      }
-
-      let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
-        url: diskFileURL,
-        readOnly: diskReadOnly
-      )
-      result.append(VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
+    try disk.map {
+      try AdditionalDisk(parseFrom: $0).configuration
     }
-
-    return result
   }
 
   func directoryShares() throws -> [VZDirectorySharingDeviceConfiguration] {
@@ -579,7 +726,7 @@ struct MainApp: App {
   static var suspendable: Bool = false
   static var capturesSystemKeys: Bool = false
 
-  @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
+  @NSApplicationDelegateAdaptor private var appDelegate: AppDelegate
 
   var body: some Scene {
     WindowGroup(vm!.name) {
@@ -635,18 +782,7 @@ struct MainApp: App {
   }
 }
 
-// The only way to fully remove Edit menu item.
-class MinimalMenuAppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
-  let indexOfEditMenu = 2
-
-  func applicationDidFinishLaunching(_ : Notification) {
-    NSApplication.shared.mainMenu?.removeItem(at: indexOfEditMenu)
-
-    let nsApp = NSApplication.shared
-    nsApp.setActivationPolicy(.regular)
-    nsApp.activate(ignoringOtherApps: true)
-  }
-
+class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
   func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply {
     if (kill(getpid(), MainApp.suspendable ? SIGUSR1 : SIGINT) == 0) {
       return .terminateLater
@@ -699,12 +835,12 @@ struct VMView: NSViewRepresentable {
 
     machineView.capturesSystemKeys = capturesSystemKeys
 
-    // Enable automatic display reconfiguration
-    // for guests that support it
+    // If not specified, enable automatic display
+    // reconfiguration for guests that support it
     //
     // This is disabled for Linux because of poor HiDPI
     // support, which manifests in fonts being too small
-    if #available(macOS 14.0, *), vm.config.os != .linux {
+    if #available(macOS 14.0, *), vm.config.displayRefit ?? (vm.config.os != .linux) {
       machineView.automaticallyReconfiguresDisplay = true
     }
 
@@ -716,6 +852,138 @@ struct VMView: NSViewRepresentable {
   }
 }
 
+struct AdditionalDisk {
+  let configuration: VZStorageDeviceConfiguration
+
+  init(parseFrom: String) throws {
+    let (diskPath, readOnly, syncModeRaw, cachingModeRaw) = Self.parseOptions(parseFrom)
+
+    self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw, cachingModeRaw: cachingModeRaw)
+  }
+
+  static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String, cachingModeRaw: String) throws -> VZStorageDeviceConfiguration {
+    let diskURL = URL(string: diskPath)
+
+    if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
+      guard #available(macOS 14, *) else {
+        throw UnsupportedOSError("attaching Network Block Devices", "are")
+      }
+
+      let nbdAttachment = try VZNetworkBlockDeviceStorageDeviceAttachment(
+        url: diskURL!,
+        timeout: 30,
+        isForcedReadOnly: diskReadOnly,
+        synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)
+      )
+
+      return VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment)
+    }
+
+    // Expand the tilde (~) since at this point we're dealing with a local path,
+    // and "expandingTildeInPath" seems to corrupt the remote URLs like nbd://
+    let diskPath = NSString(string: diskPath).expandingTildeInPath
+
+    let diskFileURL = URL(fileURLWithPath: diskPath)
+
+    if pathHasMode(diskPath, mode: S_IFBLK) {
+      guard #available(macOS 14, *) else {
+        throw UnsupportedOSError("attaching block devices", "are")
+      }
+
+      let fd = open(diskPath, diskReadOnly ? O_RDONLY : O_RDWR)
+      if fd == -1 {
+        let details = Errno(rawValue: CInt(errno))
+
+        switch details.rawValue {
+        case EBUSY:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "already in use, try umounting it via \"diskutil unmountDisk\" (when the whole disk) or \"diskutil umount\" (when mounting a single partition)")
+        case EACCES:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "permission denied, consider changing the disk's owner using \"sudo chown $USER \(diskFileURL.url.path)\" or run Tart as a superuser (see --disk help for more details on how to do that correctly)")
+        default:
+          throw RuntimeError.FailedToOpenBlockDevice(diskFileURL.url.path, "\(details)")
+        }
+      }
+
+      let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
+                                                                         readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw))
+
+      return VZVirtioBlockDeviceConfiguration(attachment: blockAttachment)
+    }
+
+    // Support remote VM names in --disk command-line argument
+    if let remoteName = try? RemoteName(diskPath) {
+      let vmDir = try VMStorageOCI().open(remoteName)
+
+      // Unfortunately, VZDiskImageStorageDeviceAttachment does not support
+      // FileHandle, so we can't easily clone the disk, open it and unlink(2)
+      // to simplify the garbage collection, so use an intermediate directory.
+      let clonedDiskURL = try Config().tartTmpDir.appendingPathComponent("run-disk-\(UUID().uuidString)")
+
+      try FileManager.default.copyItem(at: vmDir.diskURL, to: clonedDiskURL)
+
+      let lock = try FileLock(lockURL: clonedDiskURL)
+      try lock.lock()
+
+      let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: clonedDiskURL, readOnly: diskReadOnly)
+
+      return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+    }
+
+    // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
+    // see https://github.com/cirruslabs/tart/issues/323 for more details.
+    if try !diskReadOnly && !FileLock(lockURL: diskFileURL).trylock() {
+      throw RuntimeError.DiskAlreadyInUse("disk \(diskFileURL.url.path) seems to be already in use, unmount it first in Finder")
+    }
+
+    let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
+      url: diskFileURL,
+      readOnly: diskReadOnly,
+      cachingMode: try VZDiskImageCachingMode(cachingModeRaw) ?? .automatic,
+      synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
+    )
+
+    return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
+  }
+
+  static func parseOptions(_ parseFrom: String) -> (String, Bool, String, String) {
+    var arguments = parseFrom.split(separator: ":")
+
+    let options = DiskOptions(String(arguments.last!))
+    if options.foundAtLeastOneOption {
+      arguments.removeLast()
+    }
+
+    return (arguments.joined(separator: ":"), options.readOnly, options.syncModeRaw, options.cachingModeRaw)
+  }
+}
+
+struct DiskOptions {
+  var readOnly: Bool = false
+  var syncModeRaw: String = ""
+  var cachingModeRaw: String = ""
+  var foundAtLeastOneOption: Bool = false
+
+  init(_ parseFrom: String) {
+    let options = parseFrom.split(separator: ",")
+
+    for option in options {
+      switch true {
+      case option == "ro":
+        self.readOnly = true
+        self.foundAtLeastOneOption = true
+      case option.hasPrefix("sync="):
+        self.syncModeRaw = String(option.dropFirst("sync=".count))
+        self.foundAtLeastOneOption = true
+      case option.hasPrefix("caching="):
+        self.cachingModeRaw = String(option.dropFirst("caching=".count))
+        self.foundAtLeastOneOption = true
+      default:
+        continue
+      }
+    }
+  }
+}
+
 struct DirectoryShare {
   let name: String?
   let path: URL
@@ -832,7 +1100,7 @@ struct DirectoryShare {
     process.standardInput = inPipe
     process.launch()
 
-    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.write(contentsOf: response!.data)
     try inPipe.fileHandleForWriting.close()
     process.waitUntilExit()
 

Sources/tart/Commands/Set.swift

@@ -17,6 +17,9 @@ struct Set: AsyncParsableCommand {
   @Option(help: "VM display resolution in a format of <width>x<height>. For example, 1200x800")
   var display: VMDisplayConfig?
 
+  @Flag(inversion: .prefixedNo, help: ArgumentHelp("Whether to automatically reconfigure the VM's display to fit the window"))
+  var displayRefit: Bool? = nil
+
   @Flag(help: ArgumentHelp("Generate a new random MAC address for the VM."))
   var randomMAC: Bool = false
 
@@ -25,17 +28,12 @@ struct Set: AsyncParsableCommand {
   #endif
   var randomSerial: Bool = false
 
+  @Option(help: ArgumentHelp("Replace the VM's disk contents with the disk contents at path.", valueName: "path"))
+  var disk: String?
+
   @Option(help: ArgumentHelp("Resize the VMs disk to the specified size in GB (note that the disk size can only be increased to avoid losing data)",
                              discussion: """
-                             Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images
-                             have the \"cloud-initramfs-growroot\" package installed that runs on boot) and on the rest of the
-                             distributions by running the \"growpart\" or \"resize2fs\" commands.
-
-                             For macOS, however, things are a bit more complicated: you need to remove the recovery partition
-                             first and then run various \"diskutil\" commands, see Tart's packer plugin source code for more
-                             details[1].
-
-                             [1]: https://github.com/cirruslabs/packer-plugin-tart/blob/main/builder/tart/step_disk_resize.go
+                             See https://tart.run/faq/#disk-resizing for more details.
                              """))
   var diskSize: UInt16?
 
@@ -60,19 +58,28 @@ struct Set: AsyncParsableCommand {
       }
     }
 
+    vmConfig.displayRefit = displayRefit
+
     if randomMAC {
       vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
     }
 
     #if arch(arm64)
-      if randomSerial {
-        let oldPlatform = vmConfig.platform as! Darwin
+      if randomSerial, let oldPlatform = vmConfig.platform as? Darwin {
         vmConfig.platform = Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: oldPlatform.hardwareModel)
       }
     #endif
 
     try vmConfig.save(toURL: vmDir.configURL)
 
+    if let disk = disk {
+      let temporaryDiskURL = try Config().tartTmpDir.appendingPathComponent("set-disk-\(UUID().uuidString)")
+
+      try FileManager.default.copyItem(atPath: disk, toPath: temporaryDiskURL.path())
+
+      _ = try FileManager.default.replaceItemAt(vmDir.diskURL, withItemAt: temporaryDiskURL)
+    }
+
     if diskSize != nil {
       try vmDir.resizeDisk(diskSize!)
     }

Sources/tart/Config.swift

@@ -9,7 +9,8 @@ struct Config {
     var tartHomeDir: URL
 
     if let customTartHome = ProcessInfo.processInfo.environment["TART_HOME"] {
-      tartHomeDir = URL(fileURLWithPath: customTartHome)
+      tartHomeDir = URL(fileURLWithPath: customTartHome, isDirectory: true)
+      try Self.validateTartHome(url: tartHomeDir)
     } else {
       tartHomeDir = FileManager.default
         .homeDirectoryForCurrentUser
@@ -49,4 +50,24 @@ struct Config {
   static func jsonDecoder() -> JSONDecoder {
     JSONDecoder()
   }
+
+  private static func validateTartHome(url: URL) throws {
+    let urlComponents = url.pathComponents
+
+    let descendingURLs = urlComponents.indices.map { i in
+      URL(fileURLWithPath: urlComponents[0...i].joined(separator: "/"))
+    }
+
+    for descendingURL in descendingURLs {
+      if FileManager.default.fileExists(atPath: descendingURL.path) {
+        continue
+      }
+
+      do {
+        try FileManager.default.createDirectory(at: descendingURL, withIntermediateDirectories: false)
+      } catch {
+        throw RuntimeError.Generic("TART_HOME is invalid: \(descendingURL.path) does not exist, yet we can't create it: \(error.localizedDescription)")
+      }
+    }
+  }
 }

Sources/tart/ControlSocket.swift

@@ -0,0 +1,89 @@
+import Foundation
+import Network
+import os.log
+import NIO
+import NIOPosix
+
+@available(macOS 14, *)
+class ControlSocket {
+  let controlSocketURL: URL
+  let vmPort: UInt32
+  let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1)
+  let logger: os.Logger = os.Logger(subsystem: "org.cirruslabs.tart.control-socket", category: "network")
+
+  init(_ controlSocketURL: URL, vmPort: UInt32 = 8080) {
+    self.controlSocketURL = controlSocketURL
+    self.vmPort = vmPort
+  }
+
+  func run() async throws {
+    // Remove control socket file from previous "tart run" invocations,
+    // if any, otherwise we may get the "address already in use" error
+    try? FileManager.default.removeItem(atPath: controlSocketURL.path())
+
+    let serverChannel = try await ServerBootstrap(group: eventLoopGroup)
+      .bind(unixDomainSocketPath: controlSocketURL.path()) { childChannel in
+        childChannel.eventLoop.makeCompletedFuture {
+          return try NIOAsyncChannel<ByteBuffer, ByteBuffer>(
+            wrappingChannelSynchronously: childChannel
+          )
+        }
+      }
+
+    try await withThrowingDiscardingTaskGroup { group in
+      try await serverChannel.executeThenClose { serverInbound in
+        for try await clientChannel in serverInbound {
+          group.addTask {
+            try await self.handleClient(clientChannel)
+          }
+        }
+      }
+    }
+  }
+
+  func handleClient(_ clientChannel: NIOAsyncChannel<ByteBuffer, ByteBuffer>) async throws {
+    self.logger.info("received new control socket connection from a client")
+
+    try await clientChannel.executeThenClose { clientInbound, clientOutbound in
+      self.logger.info("dialing to VM on port \(self.vmPort)...")
+
+      do {
+        guard let vmConnection = try await vm?.connect(toPort: self.vmPort) else {
+          throw RuntimeError.VMSocketFailed(self.vmPort, "VM is not running")
+        }
+
+        self.logger.info("running control socket proxy")
+
+        let vmChannel = try await ClientBootstrap(group: eventLoopGroup).withConnectedSocket(vmConnection.fileDescriptor) { childChannel in
+          childChannel.eventLoop.makeCompletedFuture {
+            try NIOAsyncChannel<ByteBuffer, ByteBuffer>(
+              wrappingChannelSynchronously: childChannel
+            )
+          }
+        }
+
+        try await vmChannel.executeThenClose { (vmInbound, vmOutbound) in
+          try await withThrowingDiscardingTaskGroup { group in
+            // Proxy data from a client (e.g. "tart exec") to a VM
+            group.addTask {
+              for try await message in clientInbound {
+                try await vmOutbound.write(message)
+              }
+            }
+
+            // Proxy data from a VM to a client (e.g. "tart exec")
+            group.addTask {
+              for try await message in vmInbound {
+                try await clientOutbound.write(message)
+              }
+            }
+          }
+        }
+
+        self.logger.info("control socket client disconnected")
+      } catch (let error) {
+        self.logger.error("control socket connection failed: \(error)")
+      }
+    }
+  }
+}

Sources/tart/Credentials/CredentialsProvider.swift

@@ -5,6 +5,7 @@ enum CredentialsProviderError: Error {
 }
 
 protocol CredentialsProvider {
+  var userFriendlyName: String { get }
   func retrieve(host: String) throws -> (String, String)?
   func store(host: String, user: String, password: String) throws
 }

Sources/tart/Credentials/DockerConfigCredentialsProvider.swift

@@ -1,6 +1,8 @@
 import Foundation
 
 class DockerConfigCredentialsProvider: CredentialsProvider {
+  let userFriendlyName = "Docker configuration credentials provider"
+
   func retrieve(host: String) throws -> (String, String)? {
     let dockerConfigURL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".docker").appendingPathComponent("config.json")
     if !FileManager.default.fileExists(atPath: dockerConfigURL.path) {
@@ -36,12 +38,17 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 
     process.launch()
 
-    inPipe.fileHandleForWriting.write("\(host)\n".data(using: .utf8)!)
+    do {
+      try inPipe.fileHandleForWriting.write(contentsOf: "\(host)\n".data(using: .utf8)!)
+    } catch {
+      throw CredentialsProviderError.Failed(message: "Failed to write host to Docker helper!")
+    }
     inPipe.fileHandleForWriting.closeFile()
 
+    let outputData = try outPipe.fileHandleForReading.readToEnd()
+
     process.waitUntilExit()
 
-    let outputData = try outPipe.fileHandleForReading.readToEnd()
     if !(process.terminationReason == .exit && process.terminationStatus == 0) {
       if let outputData = outputData {
         print(String(decoding: outputData, as: UTF8.self))

Sources/tart/Credentials/EnvironmentCredentialsProvider.swift

@@ -1,6 +1,8 @@
 import Foundation
 
 class EnvironmentCredentialsProvider: CredentialsProvider {
+  let userFriendlyName = "environment variable credentials provider"
+
   func retrieve(host: String) throws -> (String, String)? {
     if let tartRegistryHostname = ProcessInfo.processInfo.environment["TART_REGISTRY_HOSTNAME"],
        tartRegistryHostname != host {

Sources/tart/Credentials/KeychainCredentialsProvider.swift

@@ -1,6 +1,8 @@
 import Foundation
 
 class KeychainCredentialsProvider: CredentialsProvider {
+  let userFriendlyName = "Keychain credentials provider"
+
   func retrieve(host: String) throws -> (String, String)? {
     let query: [String: Any] = [kSecClass as String: kSecClassInternetPassword,
                                 kSecAttrProtocol as String: kSecAttrProtocolHTTPS,

Sources/tart/Credentials/StdinCredentials.swift

@@ -6,6 +6,8 @@ enum StdinCredentialsError: Error {
 }
 
 class StdinCredentials {
+  let userFriendlyName = "standard input credentials provider"
+
   static func retrieve() throws -> (String, String) {
     let user = try readStdinCredential(name: "username", prompt: "User: ", isSensitive: false)
     let password = try readStdinCredential(name: "password", prompt: "Password: ", isSensitive: true)

Sources/tart/DiskImageFormat.swift

@@ -0,0 +1,43 @@
+import Foundation
+import ArgumentParser
+
+enum DiskImageFormat: String, CaseIterable, Codable {
+  case raw = "raw"
+  case asif = "asif"
+
+  var displayName: String {
+    switch self {
+    case .raw:
+      return "RAW"
+    case .asif:
+      return "ASIF (Apple Sparse Image Format)"
+    }
+  }
+
+
+  /// Check if the format is supported on the current system
+  var isSupported: Bool {
+    switch self {
+    case .raw:
+      return true
+    case .asif:
+      if #available(macOS 26, *) {
+        return true
+      } else {
+        return false
+      }
+    }
+  }
+
+
+}
+
+extension DiskImageFormat: ExpressibleByArgument {
+  init?(argument: String) {
+    self.init(rawValue: argument.lowercased())
+  }
+
+  static var allValueStrings: [String] {
+    return allCases.map { $0.rawValue }
+  }
+}

Sources/tart/Fetcher.swift

@@ -1,69 +1,6 @@
 import Foundation
-import AsyncAlgorithms
 
-fileprivate let urlSession = createURLSession()
-
-class DownloadDelegate: NSObject, URLSessionTaskDelegate {
-  let progress: Progress
-  init(_ progress: Progress) throws {
-    self.progress = progress
-  }
-
-  func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) {
-    self.progress.addChild(task.progress, withPendingUnitCount: self.progress.totalUnitCount)
-  }
-}
-
-class Fetcher {
-  static func fetch(_ request: URLRequest, viaFile: Bool = false, progress: Progress? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let delegate = progress != nil ? try DownloadDelegate(progress!) : nil
-
-    if viaFile {
-      return try await fetchViaFile(request, delegate: delegate)
-    }
-
-    return try await fetchViaMemory(request, delegate: delegate)
-  }
-
-  private static func fetchViaMemory(_ request: URLRequest, delegate: URLSessionTaskDelegate? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let dataCh = AsyncThrowingChannel<Data, Error>()
-
-    let (data, response) = try await urlSession.data(for: request, delegate: delegate)
-
-    Task {
-      await dataCh.send(data)
-
-      dataCh.finish()
-    }
-
-    return (dataCh, response as! HTTPURLResponse)
-  }
-
-  private static func fetchViaFile(_ request: URLRequest, delegate: URLSessionTaskDelegate? = nil) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
-    let dataCh = AsyncThrowingChannel<Data, Error>()
-
-    let (fileURL, response) = try await urlSession.download(for: request, delegate: delegate)
-
-    // Acquire a handle to the downloaded file and then remove it.
-    //
-    // This keeps a working reference to that file, yet we don't
-    // have to deal with the cleanup any more.
-    let mappedFile = try Data(contentsOf: fileURL, options: [.alwaysMapped])
-    try FileManager.default.removeItem(at: fileURL)
-
-    Task {
-      for chunk in (0 ..< mappedFile.count).chunks(ofCount: 64 * 1024 * 1024) {
-        await dataCh.send(mappedFile.subdata(in: chunk))
-      }
-
-      dataCh.finish()
-    }
-
-    return (dataCh, response as! HTTPURLResponse)
-  }
-}
-
-fileprivate func createURLSession() -> URLSession {
+fileprivate var urlSession: URLSession = {
   let config = URLSessionConfiguration.default
 
   // Harbor expects a CSRF token to be present if the HTTP client
@@ -77,4 +14,84 @@ fileprivate func createURLSession() -> URLSession {
   config.httpShouldSetCookies = false
 
   return URLSession(configuration: config)
+}()
+
+class Fetcher {
+  static func fetch(_ request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
+    let task = urlSession.dataTask(with: request)
+
+    let delegate = Delegate()
+    task.delegate = delegate
+
+    let stream = AsyncThrowingStream<Data, Error> { continuation in
+      delegate.streamContinuation = continuation
+    }
+
+    let response = try await withCheckedThrowingContinuation { continuation in
+      delegate.responseContinuation = continuation
+      task.resume()
+    }
+
+    return (stream, response as! HTTPURLResponse)
+  }
+}
+
+fileprivate class Delegate: NSObject, URLSessionDataDelegate {
+  var responseContinuation: CheckedContinuation<URLResponse, Error>?
+  var streamContinuation: AsyncThrowingStream<Data, Error>.Continuation?
+
+  private var buffer: Data = Data()
+  private let bufferFlushSize = 16 * 1024 * 1024
+
+  func urlSession(
+    _ session: URLSession,
+    dataTask: URLSessionDataTask,
+    didReceive response: URLResponse,
+    completionHandler: @escaping (URLSession.ResponseDisposition) -> Void
+  ) {
+    // Soft-limit for the maximum buffer capacity
+    let capacity = min(response.expectedContentLength, Int64(bufferFlushSize))
+
+    // Pre-initialize buffer as we now know the capacity
+    buffer = Data(capacity: Int(capacity))
+
+    responseContinuation?.resume(returning: response)
+    responseContinuation = nil
+    completionHandler(.allow)
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    dataTask: URLSessionDataTask,
+    didReceive data: Data
+  ) {
+    buffer.append(data)
+
+    if buffer.count >= bufferFlushSize {
+      streamContinuation?.yield(buffer)
+      buffer.removeAll(keepingCapacity: true)
+    }
+  }
+
+  func urlSession(
+    _ session: URLSession,
+    task: URLSessionTask,
+    didCompleteWithError error: Error?
+  ) {
+    if let error = error {
+      responseContinuation?.resume(throwing: error)
+      responseContinuation = nil
+
+      streamContinuation?.finish(throwing: error)
+      streamContinuation = nil
+    } else {
+      if !buffer.isEmpty {
+        streamContinuation?.yield(buffer)
+        buffer.removeAll(keepingCapacity: true)
+      }
+
+      streamContinuation?.finish()
+      streamContinuation = nil
+    }
+  }
 }

Sources/tart/LocalLayerCache.swift

@@ -1,10 +1,25 @@
 import Foundation
 
 struct LocalLayerCache {
+  struct DigestInfo {
+    let range: Range<Data.Index>
+    let compressedDigest: String
+    let uncompressedContentDigest: String?
+  }
+
+  let name: String
+  let deduplicatedBytes: UInt64
+  let diskURL: URL
+
   private let mappedDisk: Data
-  private var digestToRange: [String : Range<Data.Index>] = [:]
+  private var digestToRange: [String: DigestInfo] = [:]
+  private var offsetToRange: [UInt64: DigestInfo] = [:]
+
+  init?(_ name: String, _ deduplicatedBytes: UInt64, _ diskURL: URL, _ manifest: OCIManifest) throws {
+    self.name = name
+    self.deduplicatedBytes = deduplicatedBytes
+    self.diskURL = diskURL
 
-  init?(_ diskURL: URL, _ manifest: OCIManifest) throws {
     // mmap(2) the disk that contains the layers from the manifest
     self.mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
 
@@ -16,17 +31,27 @@ struct LocalLayerCache {
         return nil
       }
 
-      self.digestToRange[layer.digest] = Int(offset)..<Int(offset+uncompressedSize)
+      let info = DigestInfo(
+        range: Int(offset)..<Int(offset + uncompressedSize),
+        compressedDigest: layer.digest,
+        uncompressedContentDigest: layer.uncompressedContentDigest()!
+      )
+      self.digestToRange[layer.digest] = info
+      self.offsetToRange[offset] = info
 
       offset += uncompressedSize
     }
   }
 
-  func find(_ digest: String) -> Data? {
-    guard let foundRange = self.digestToRange[digest] else {
-      return nil
+  func findInfo(digest: String, offsetHint: UInt64) -> DigestInfo? {
+    // Layers can have the same digests, for example, empty ones. Let's use the offset hint to make a better guess.
+    if let info = self.offsetToRange[offsetHint], info.compressedDigest == digest {
+      return info
     }
+    return self.digestToRange[digest]
+  }
 
-    return self.mappedDisk.subdata(in: foundRange)
+  func subdata(_ range: Range<Data.Index>) -> Data {
+    return self.mappedDisk.subdata(in: range)
   }
 }

Sources/tart/Logging/ProgressObserver.swift

@@ -4,18 +4,28 @@ public class ProgressObserver: NSObject {
   @objc var progressToObserve: Progress
   var observation: NSKeyValueObservation?
   var lastTimeUpdated = Date.now
+  private var lastRenderedLine: String?
 
   public init(_ progress: Progress) {
     progressToObserve = progress
   }
 
   func log(_ renderer: Logger) {
-    renderer.appendNewLine(ProgressObserver.lineToRender(progressToObserve))
+    let initialLine = ProgressObserver.lineToRender(progressToObserve)
+    renderer.appendNewLine(initialLine)
+    lastRenderedLine = initialLine
     observation = observe(\.progressToObserve.fractionCompleted) { progress, _ in
       let currentTime = Date.now
       if self.progressToObserve.isFinished || currentTime.timeIntervalSince(self.lastTimeUpdated) >= 1.0 {
         self.lastTimeUpdated = currentTime
-        renderer.updateLastLine(ProgressObserver.lineToRender(self.progressToObserve))
+        let line = ProgressObserver.lineToRender(self.progressToObserve)
+        // Skip identical renders so non-interactive logs only see new percent values.
+        if line == self.lastRenderedLine {
+          return
+        }
+
+        self.lastRenderedLine = line
+        renderer.updateLastLine(line)
       }
     }
   }

Sources/tart/MACAddressResolver/ARPCache.swift

@@ -52,6 +52,11 @@ struct ARPCache {
     process.standardInput = FileHandle.nullDevice
 
     try process.run()
+
+    guard let arpCommandOutput = try pipe.fileHandleForReading.readToEnd() else {
+      throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
+    }
+
     process.waitUntilExit()
 
     if !(process.terminationReason == .exit && process.terminationStatus == 0) {
@@ -60,10 +65,6 @@ struct ARPCache {
         terminationStatus: process.terminationStatus)
     }
 
-    guard let arpCommandOutput = try pipe.fileHandleForReading.readToEnd() else {
-      throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
-    }
-
     self.arpCommandOutput = arpCommandOutput
   }
 

Sources/tart/MACAddressResolver/AgentResolver.swift

@@ -0,0 +1,42 @@
+import Foundation
+import Network
+import NIOPosix
+import GRPC
+import Cirruslabs_TartGuestAgent_Apple_Swift
+import Cirruslabs_TartGuestAgent_Grpc_Swift
+
+class AgentResolver {
+  static func ResolveIP(_ controlSocketURL: URL) async throws -> IPv4Address? {
+    do {
+      return try await resolveIP(controlSocketURL)
+    } catch let error as GRPCConnectionPoolError {
+      return nil
+    }
+  }
+
+  private static func resolveIP(_ controlSocketURL: URL) async throws -> IPv4Address? {
+    // Create a gRPC channel connected to the VM's control socket
+    let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
+    defer {
+      try! group.syncShutdownGracefully()
+    }
+
+    let channel = try GRPCChannelPool.with(
+      target: .unixDomainSocket(controlSocketURL.path()),
+      transportSecurity: .plaintext,
+      eventLoopGroup: group,
+    )
+    defer {
+      try! channel.close().wait()
+    }
+
+    // Invoke ResolveIP() gRPC method
+    let callOptions = CallOptions(timeLimit: .timeout(.seconds(1)))
+    let agentAsyncClient = AgentAsyncClient(channel: channel)
+    let resolveIPCall = agentAsyncClient.makeResolveIpCall(ResolveIPRequest(), callOptions: callOptions)
+
+    let response = try await resolveIPCall.response
+
+    return IPv4Address(response.ip)
+  }
+}

Sources/tart/OCI/Layerizer/Disk.swift

@@ -1,6 +1,6 @@
 import Foundation
 
 protocol Disk {
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer]
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?) async throws
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
 }

Sources/tart/OCI/Layerizer/DiskV1.swift

@@ -5,7 +5,7 @@ class DiskV1: Disk {
   private static let bufferSizeBytes = 4 * 1024 * 1024
   private static let layerLimitBytes = 500 * 1000 * 1000
 
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
     var pushedLayers: [OCIManifestLayer] = []
 
     // Open the disk file
@@ -45,7 +45,7 @@ class DiskV1: Disk {
     return pushedLayers
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
     if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
       throw OCIError.FailedToCreateVmFile
     }
@@ -57,7 +57,7 @@ class DiskV1: Disk {
     // Decompress the layers onto the disk in a single stream
     let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
       if let data = data {
-        disk.write(data)
+        try disk.write(contentsOf: data)
       }
     }
 

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -1,44 +1,104 @@
 import Foundation
 import Compression
+import System
+import Retry
 
 class DiskV2: Disk {
   private static let bufferSizeBytes = 4 * 1024 * 1024
-  private static let layerLimitBytes = 500 * 1000 * 1000
+  private static let layerLimitBytes = 512 * 1024 * 1024
 
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
-    var pushedLayers: [OCIManifestLayer] = []
+  // A zero chunk for faster than byte-by-byte comparisons
+  //
+  // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
+  // on the last block since it costs only 4 MiB of excess data per 512 MiB layer.
+  //
+  // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
+  // +--------------------------------------+---------------------------------------------------+
+  // | Operation                            | time(1) result                                    |
+  // +--------------------------------------+---------------------------------------------------+
+  // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
+  // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
+  // +--------------------------------------+---------------------------------------------------+
+  private static let holeGranularityBytes = 4 * 1024 * 1024
+  private static let zeroChunk = Data(count: holeGranularityBytes)
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []
 
     // Open the disk file
     let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
 
     // Compress the disk file as multiple individually decompressible streams,
     // each equal ``Self.layerLimitBytes`` bytes or less due to LZ4 compression
-    for data in mappedDisk.chunks(ofCount: layerLimitBytes) {
-      let compressedData = try (data as NSData).compressed(using: .lz4) as Data
+    try await withThrowingTaskGroup(of: (Int, OCIManifestLayer).self) { group in
+      for (index, data) in mappedDisk.chunks(ofCount: layerLimitBytes).enumerated() {
+        // Respect the concurrency limit
+        if index >= concurrency {
+          if let (index, pushedLayer) = try await group.next() {
+            pushedLayers.append((index, pushedLayer))
+          }
+        }
 
-      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+        // Launch a disk layer pushing task
+        group.addTask {
+          let compressedData = try (data as NSData).compressed(using: .lz4) as Data
+          let compressedDataDigest = Digest.hash(compressedData)
 
-      pushedLayers.append(OCIManifestLayer(
-        mediaType: diskV2MediaType,
-        size: compressedData.count,
-        digest: layerDigest,
-        uncompressedSize: UInt64(data.count),
-        uncompressedContentDigest: Digest.hash(data)
-      ))
+          try await retry(maxAttempts: 5) {
+            if try await !registry.blobExists(compressedDataDigest) {
+              _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error: \(error.localizedDescription)")
+              print("Attempting to re-try...")
 
-      // Update progress using a relative value
-      progress.completedUnitCount += Int64(data.count)
+              return .retry
+            }
+
+            return .throw
+          }
+
+          // Update progress using a relative value
+          progress.completedUnitCount += Int64(data.count)
+
+          return (index, OCIManifestLayer(
+            mediaType: diskV2MediaType,
+            size: compressedData.count,
+            digest: compressedDataDigest,
+            uncompressedSize: UInt64(data.count),
+            uncompressedContentDigest: Digest.hash(data)
+          ))
+        }
+      }
+
+      for try await pushedLayer in group {
+        pushedLayers.append(pushedLayer)
+      }
     }
 
-    return pushedLayers
+    return pushedLayers.sorted {
+      $0.index < $1.index
+    }.map {
+      $0.pushedLayer
+    }
   }
 
-  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil) async throws {
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache? = nil, deduplicate: Bool = false) async throws {
     // Support resumable pulls
     let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
 
-    if !pullResumed && !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
-      throw OCIError.FailedToCreateVmFile
+    if !pullResumed {
+      if deduplicate, let localLayerCache = localLayerCache {
+        // Clone the local layer cache's disk and use it as a base, potentially
+        // reducing the space usage since some blocks won't be written at all
+        try FileManager.default.copyItem(at: localLayerCache.diskURL, to: diskURL)
+      } else {
+        // Otherwise create an empty disk
+        if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+          throw OCIError.FailedToCreateVmFile
+        }
+      }
     }
 
     // Calculate the uncompressed disk size
@@ -58,6 +118,15 @@ class DiskV2: Disk {
     try disk.truncate(atOffset: uncompressedDiskSize)
     try disk.close()
 
+    // Determine the file system block size
+    var st = stat()
+    if stat(diskURL.path, &st) == -1 {
+      let details = Errno(rawValue: errno)
+
+      throw RuntimeError.PullFailed("failed to stat(2) disk \(diskURL.path): \(details)")
+    }
+    let fsBlockSize = UInt64(st.st_blksize)
+
     // Concurrently fetch and decompress layers
     try await withThrowingTaskGroup(of: Void.self) { group in
       var globalDiskWritingOffset: UInt64 = 0
@@ -82,22 +151,46 @@ class DiskV2: Disk {
         // Launch a fetching and decompression task
         group.addTask {
           // No need to fetch and decompress anything if we've already done so
-          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
-            // Update the progress
-            progress.completedUnitCount += Int64(diskLayer.size)
+          if pullResumed {
+            // do not check hash in the condition above to make it lazy e.g. only do expensive calculations if needed
+            if try Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+              // Update the progress
+              progress.completedUnitCount += Int64(diskLayer.size)
 
-            return
+              return
+            }
           }
 
-          // Open the disk file
+          // Open the disk file for writing
           let disk = try FileHandle(forWritingTo: diskURL)
 
-          // Check if we already have this layer contents in the local layer cache
-          if let localLayerCache = localLayerCache, let data = localLayerCache.find(diskLayer.digest), Digest.hash(data) == uncompressedLayerContentDigest {
-            // Fulfil the layer contents from the local blob cache
-            _ = try zeroSkippingWrite(disk, diskWritingOffset, data)
+          // Also open the disk file for reading and verifying
+          // its contents in case the local layer cache is used
+          let rdisk: FileHandle? = if deduplicate && localLayerCache != nil {
+            try FileHandle(forReadingFrom: diskURL)
+          } else {
+            nil
+          }
+
+          // Check if we already have this layer contents in the local layer cache,
+          // or perhaps even on the cloned disk (when the deduplication is enabled)
+          if let localLayerCache = localLayerCache,
+             let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset),
+             localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest {
+            if deduplicate && localLayerInfo.range.lowerBound == diskWritingOffset {
+              // Do nothing, because the data is already on the disk that we've inherited from
+            } else {
+              // Fulfil the layer contents from the local blob cache
+              let data = localLayerCache.subdata(localLayerInfo.range)
+              _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
+            }
+
             try disk.close()
 
+            if let rdisk = rdisk {
+              try rdisk.close()
+            }
+
             // Update the progress
             progress.completedUnitCount += Int64(diskLayer.size)
 
@@ -112,19 +205,38 @@ class DiskV2: Disk {
               return
             }
 
-            diskWritingOffset = try zeroSkippingWrite(disk, diskWritingOffset, data)
+            diskWritingOffset = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data)
           }
 
-          try await registry.pullBlob(diskLayer.digest) { data in
-            try filter.write(data)
+          var rangeStart: Int64 = 0
 
-            // Update the progress
-            progress.completedUnitCount += Int64(data.count)
+          try await retry(maxAttempts: 5) {
+            try await registry.pullBlob(diskLayer.digest, rangeStart: rangeStart) { data in
+              try filter.write(data)
+
+              // Update the progress
+              progress.completedUnitCount += Int64(data.count)
+
+              // Update the current range start
+              rangeStart += Int64(data.count)
+            }
+          } recoverFromFailure: { error in
+            if error is URLError {
+              print("Error pulling disk layer \(index + 1): \"\(error.localizedDescription)\", attempting to re-try...")
+
+              return .retry
+            }
+
+            return .throw
           }
 
           try filter.finalize()
 
           try disk.close()
+
+          if let rdisk = rdisk {
+            try rdisk.close()
+          }
         }
 
         globalDiskWritingOffset += uncompressedLayerSize
@@ -132,30 +244,45 @@ class DiskV2: Disk {
     }
   }
 
-  private static func zeroSkippingWrite(_ disk: FileHandle, _ offset: UInt64, _ data: Data) throws -> UInt64 {
-    let holeGranularityBytes = 64 * 1024
-
-    // A zero chunk for faster than byte-by-byte comparisons
-    //
-    // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
-    // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
-    //
-    // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
-    // +--------------------------------------+---------------------------------------------------+
-    // | Operation                            | time(1) result                                    |
-    // +--------------------------------------+---------------------------------------------------+
-    // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
-    // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
-    // +--------------------------------------+---------------------------------------------------+
-    let zeroChunk = Data(count: holeGranularityBytes)
-
+  private static func zeroSkippingWrite(_ disk: FileHandle, _ rdisk: FileHandle?, _ fsBlockSize: UInt64, _ offset: UInt64, _ data: Data) throws -> UInt64 {
     var offset = offset
 
     for chunk in data.chunks(ofCount: holeGranularityBytes) {
-      // Only write chunks that are not zero
+      // If the local layer cache is used, only write chunks that differ
+      // since the base disk can contain anything at any position
+      if let rdisk = rdisk {
+        // F_PUNCHHOLE requires the holes to be aligned to file system block boundaries
+        let isHoleAligned = (offset % fsBlockSize) == 0 && (UInt64(chunk.count) % fsBlockSize) == 0
+
+        if isHoleAligned && chunk == zeroChunk {
+          var arg = fpunchhole_t(fp_flags: 0, reserved: 0, fp_offset: off_t(offset), fp_length: off_t(chunk.count))
+
+          if fcntl(disk.fileDescriptor, F_PUNCHHOLE, &arg) == -1 {
+            let details = Errno(rawValue: errno)
+
+            throw RuntimeError.PullFailed("failed to punch hole: \(details)")
+          }
+        } else {
+          try rdisk.seek(toOffset: offset)
+          let actualContentsOnDisk = try rdisk.read(upToCount: chunk.count)
+
+          if chunk != actualContentsOnDisk {
+            try disk.seek(toOffset: offset)
+            try disk.write(contentsOf: chunk)
+          }
+        }
+
+        offset += UInt64(chunk.count)
+
+        continue
+      }
+
+      // Otherwise, only write chunks that are not zero
+      // since the base disk is created from scratch and
+      // is zeroed via truncate(2)
       if chunk != zeroChunk {
         try disk.seek(toOffset: offset)
-        disk.write(chunk)
+        try disk.write(contentsOf: chunk)
       }
 
       offset += UInt64(chunk.count)

Sources/tart/OCI/Manifest.swift

@@ -14,6 +14,9 @@ let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
 let uncompressedDiskSizeAnnotation = "org.cirruslabs.tart.uncompressed-disk-size"
 let uploadTimeAnnotation = "org.cirruslabs.tart.upload-time"
 
+// Manifest labels
+let diskFormatLabel = "org.cirruslabs.tart.disk.format"
+
 // Layer annotations
 let uncompressedSizeAnnotation = "org.cirruslabs.tart.uncompressed-size"
 let uncompressedContentDigestAnnotation = "org.cirruslabs.tart.uncompressed-content-digest"
@@ -66,6 +69,11 @@ struct OCIManifest: Codable, Equatable {
 struct OCIConfig: Codable {
   var architecture: Architecture = .arm64
   var os: OS = .darwin
+  var config: ConfigContainer?
+
+  struct ConfigContainer: Codable {
+    var Labels: [String: String]?
+  }
 
   func toJSON() throws -> Data {
     try Config.jsonEncoder().encode(self)
@@ -78,7 +86,7 @@ struct OCIManifestConfig: Codable, Equatable {
   var digest: String
 }
 
-struct OCIManifestLayer: Codable, Equatable {
+struct OCIManifestLayer: Codable, Equatable, Hashable {
   var mediaType: String
   var size: Int
   var digest: String
@@ -113,6 +121,14 @@ struct OCIManifestLayer: Codable, Equatable {
   func uncompressedContentDigest() -> String? {
     annotations?[uncompressedContentDigestAnnotation]
   }
+
+  static func == (lhs: Self, rhs: Self) -> Bool {
+    return lhs.digest == rhs.digest
+  }
+
+  func hash(into hasher: inout Hasher) {
+    hasher.combine(digest)
+  }
 }
 
 struct Descriptor: Equatable {

Sources/tart/OCI/Reference/Generated/ReferenceBaseListener.swift

@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 
 import Antlr4
 

Sources/tart/OCI/Reference/Generated/ReferenceLexer.swift

@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4
 
 open class ReferenceLexer: Lexer {
@@ -49,7 +49,7 @@ open class ReferenceLexer: Lexer {
 
 	public
 	required init(_ input: CharStream) {
-	    RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION)
+	    RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION)
 		super.init(input)
 		_interp = LexerATNSimulator(self, ReferenceLexer._ATN, ReferenceLexer._decisionToDFA, ReferenceLexer._sharedContextCache)
 	}

Sources/tart/OCI/Reference/Generated/ReferenceListener.swift

@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4
 
 /**

Sources/tart/OCI/Reference/Generated/ReferenceParser.swift

@@ -1,4 +1,4 @@
-// Generated from java-escape by ANTLR 4.11.1
+// Generated from Reference.g4 by ANTLR 4.13.2
 import Antlr4
 
 open class ReferenceParser: Parser {
@@ -41,7 +41,7 @@ open class ReferenceParser: Parser {
 	static let VOCABULARY = Vocabulary(_LITERAL_NAMES, _SYMBOLIC_NAMES)
 
 	override open
-	func getGrammarFileName() -> String { return "java-escape" }
+	func getGrammarFileName() -> String { return "Reference.g4" }
 
 	override open
 	func getRuleNames() -> [String] { return ReferenceParser.ruleNames }
@@ -60,7 +60,7 @@ open class ReferenceParser: Parser {
 
 	override public
 	init(_ input:TokenStream) throws {
-	    RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION)
+	    RuntimeMetaData.checkVersion("4.13.2", RuntimeMetaData.VERSION)
 		try super.init(input)
 		_interp = ParserATNSimulator(self,ReferenceParser._ATN,ReferenceParser._decisionToDFA, ReferenceParser._sharedContextCache)
 	}
@@ -460,7 +460,7 @@ open class ReferenceParser: Parser {
 		 		setState(63)
 		 		try _errHandler.sync(self)
 		 		_la = try _input.LA(1)
-		 		if ((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0) {
+		 		if (((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
 		 			setState(62)
 		 			try separator()
 
@@ -611,7 +611,7 @@ open class ReferenceParser: Parser {
 		 	setState(84)
 		 	try _errHandler.sync(self)
 		 	_la = try _input.LA(1)
-		 	while ((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0) {
+		 	while (((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
 		 		setState(79)
 		 		try separator()
 		 		setState(80)
@@ -664,7 +664,7 @@ open class ReferenceParser: Parser {
 		 	try enterOuterAlt(_localctx, 1)
 		 	setState(87)
 		 	_la = try _input.LA(1)
-		 	if (!((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0)) {
+		 	if (!(((Int64(_la) & ~0x3f) == 0 && ((Int64(1) << _la) & 88) != 0))) {
 		 	try _errHandler.recoverInline(self)
 		 	}
 		 	else {

Sources/tart/OCI/Registry.swift

@@ -1,6 +1,5 @@
 import Foundation
 import Algorithms
-import AsyncAlgorithms
 
 enum RegistryError: Error {
   case UnexpectedHTTPStatusCode(when: String, code: Int, details: String = "")
@@ -10,6 +9,7 @@ enum RegistryError: Error {
 }
 
 enum HTTPMethod: String {
+  case HEAD = "HEAD"
   case GET = "GET"
   case POST = "POST"
   case PUT = "PUT"
@@ -20,21 +20,35 @@ enum HTTPCode: Int {
   case Ok = 200
   case Created = 201
   case Accepted = 202
+  case PartialContent = 206
   case Unauthorized = 401
+  case NotFound = 404
 }
 
 extension Data {
   func asText() -> String {
     String(decoding: self, as: UTF8.self)
   }
+
+  func asTextPreview(limit: Int = 1000) -> String {
+    guard count > limit else {
+      return asText()
+    }
+
+    return "\(asText().prefix(limit))..."
+  }
 }
 
-extension AsyncThrowingChannel<Data, Error> {
-  func asData() async throws -> Data {
+extension AsyncThrowingStream<Data, Error> {
+  func asData(limitBytes: Int64? = nil) async throws -> Data {
     var result = Data()
 
     for try await chunk in self {
       result += chunk
+
+      if let limitBytes, result.count > limitBytes {
+        return result
+      }
     }
 
     return result
@@ -112,11 +126,11 @@ class Registry {
     return host
   }
 
-  init(urlComponents: URLComponents,
+  init(baseURL: URL,
        namespace: String,
        credentialsProviders: [CredentialsProvider] = [EnvironmentCredentialsProvider(), DockerConfigCredentialsProvider(), KeychainCredentialsProvider()]
   ) throws {
-    baseURL = urlComponents.url!
+    self.baseURL = baseURL
     self.namespace = namespace
     self.credentialsProviders = credentialsProviders
   }
@@ -130,7 +144,17 @@ class Registry {
     let proto = insecure ? "http" : "https"
     let baseURLComponents = URLComponents(string: proto + "://" + host + "/v2/")!
 
-    try self.init(urlComponents: baseURLComponents, namespace: namespace, credentialsProviders: credentialsProviders)
+    guard let baseURL = baseURLComponents.url else {
+      var hint = ""
+
+      if host.hasPrefix("http://") || host.hasPrefix("https://") {
+        hint += ", make sure that it doesn't start with http:// or https://"
+      }
+
+      throw RuntimeError.ImproperlyFormattedHost(host, hint)
+    }
+
+    try self.init(baseURL: baseURL, namespace: namespace, credentialsProviders: credentialsProviders)
   }
 
   func ping() async throws {
@@ -148,7 +172,7 @@ class Registry {
                                                  body: manifestJSON)
     if response.statusCode != HTTPCode.Created.rawValue {
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing manifest", code: response.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
     }
 
     return Digest.hash(manifestJSON)
@@ -159,7 +183,7 @@ class Registry {
                                                  headers: ["Accept": ociManifestMediaType])
     if response.statusCode != HTTPCode.Ok.rawValue {
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling manifest", code: response.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
     }
 
     let manifest = try OCIManifest(fromJSON: data)
@@ -179,19 +203,19 @@ class Registry {
     return URLComponents(url: uploadLocation.absolutize(baseURL), resolvingAgainstBaseURL: true)!
   }
 
-  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0) async throws -> String {
+  public func pushBlob(fromData: Data, chunkSizeMb: Int = 0, digest: String? = nil) async throws -> String {
     // Initiate a blob upload
     let (data, postResponse) = try await dataRequest(.POST, endpointURL("\(namespace)/blobs/uploads/"),
                                                      headers: ["Content-Length": "0"])
     if postResponse.statusCode != HTTPCode.Accepted.rawValue {
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (POST)", code: postResponse.statusCode,
-                                                   details: data.asText())
+                                                   details: data.asTextPreview())
     }
 
     // Figure out where to upload the blob
     var uploadLocation = try uploadLocationFromResponse(postResponse)
 
-    let digest = Digest.hash(fromData)
+    let digest = digest ?? Digest.hash(fromData)
 
     if chunkSizeMb == 0 {
       // monolithic upload
@@ -206,7 +230,7 @@ class Registry {
       )
       if response.statusCode != HTTPCode.Created.rawValue {
         throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT) to \(uploadLocation)",
-                                                     code: response.statusCode, details: data.asText())
+                                                     code: response.statusCode, details: data.asTextPreview())
       }
       return digest
     }
@@ -229,7 +253,7 @@ class Registry {
       // always accept both statuses since AWS ECR is not following specification
       if response.statusCode != HTTPCode.Created.rawValue && response.statusCode != HTTPCode.Accepted.rawValue {
         throw RegistryError.UnexpectedHTTPStatusCode(when: "streaming blob to \(uploadLocation)",
-                                                     code: response.statusCode, details: data.asText())
+                                                     code: response.statusCode, details: data.asTextPreview())
       }
       uploadedBytes += chunk.count
       // Update location for the next chunk
@@ -239,10 +263,35 @@ class Registry {
     return digest
   }
 
-  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
-    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
-    if response.statusCode != HTTPCode.Ok.rawValue {
-      let body = try await channel.asData().asText()
+  public func blobExists(_ digest: String) async throws -> Bool {
+    let (data, response) = try await dataRequest(.HEAD, endpointURL("\(namespace)/blobs/\(digest)"))
+
+    switch response.statusCode {
+    case HTTPCode.Ok.rawValue:
+      return true
+    case HTTPCode.NotFound.rawValue:
+      return false
+    default:
+      throw RegistryError.UnexpectedHTTPStatusCode(when: "checking blob", code: response.statusCode, details: data.asTextPreview())
+    }
+  }
+
+  public func pullBlob(_ digest: String, rangeStart: Int64 = 0, handler: (Data) async throws -> Void) async throws {
+    var expectedStatusCode = HTTPCode.Ok
+    var headers: [String: String] = [:]
+
+    // Send Range header and expect HTTP 206 in return
+    //
+    // However, do not send Range header at all when rangeStart is 0,
+    // because it makes no sense and we might get HTTP 200 in return
+    if rangeStart != 0 {
+      expectedStatusCode = HTTPCode.PartialContent
+      headers["Range"] = "bytes=\(rangeStart)-"
+    }
+
+    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), headers: headers, viaFile: true)
+    if response.statusCode != expectedStatusCode.rawValue {
+      let body = try await channel.asData(limitBytes: 4096).asTextPreview()
       throw RegistryError.UnexpectedHTTPStatusCode(when: "pulling blob", code: response.statusCode,
                                                    details: body)
     }
@@ -282,7 +331,7 @@ class Registry {
     body: Data? = nil,
     doAuth: Bool = true,
     viaFile: Bool = false
-  ) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  ) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
     var urlComponents = urlComponents
 
     if urlComponents.queryItems == nil && !parameters.isEmpty {
@@ -302,12 +351,11 @@ class Registry {
       request.httpBody = body
     }
 
-    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
 
     if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
-      _ = try await channel.asData()
       try await auth(response: response)
-      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
+      (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile, doAuth: doAuth)
     }
 
     return (channel, response)
@@ -367,7 +415,7 @@ class Registry {
     let (data, response) = try await dataRequest(.GET, authenticateURL, headers: headers, doAuth: false)
     if response.statusCode != HTTPCode.Ok.rawValue {
       throw RegistryError.AuthFailed(why: "received unexpected HTTP status code \(response.statusCode) "
-        + "while retrieving an authentication token", details: data.asText())
+        + "while retrieving an authentication token", details: data.asTextPreview())
     }
 
     await authenticationKeeper.set(try TokenResponse.parse(fromData: data))
@@ -381,18 +429,24 @@ class Registry {
     }
 
     for provider in credentialsProviders {
-      if let (user, password) = try provider.retrieve(host: host) {
-        return (user, password)
+      do {
+        if let (user, password) = try provider.retrieve(host: host) {
+          return (user, password)
+        }
+      } catch (let e) {
+        print("Failed to retrieve credentials using \(provider.userFriendlyName), authentication may fail: \(e)")
       }
     }
     return nil
   }
 
-  private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
+  private func authAwareRequest(request: URLRequest, viaFile: Bool = false, doAuth: Bool) async throws -> (AsyncThrowingStream<Data, Error>, HTTPURLResponse) {
     var request = request
 
-    if let (name, value) = await authenticationKeeper.header() {
-      request.addValue(value, forHTTPHeaderField: name)
+    if doAuth {
+      if let (name, value) = await authenticationKeeper.header() {
+        request.addValue(value, forHTTPHeaderField: name)
+      }
     }
 
     request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",

Sources/tart/PIDLock.swift

@@ -11,7 +11,7 @@ class PIDLock {
     if fd == -1 {
       let details = Errno(rawValue: CInt(errno))
 
-      throw RuntimeError.PIDLockFailed("failed to open lock file \(url): \(details)")
+      throw RuntimeError.PIDLockMissing("failed to open lock file \(url): \(details)")
     }
   }
 

Sources/tart/Platform/Darwin.swift

@@ -38,7 +38,7 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
         throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
       }
       guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
-        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+        throw UnsupportedHostOSError()
       }
       self.hardwareModel = hardwareModel
     }
@@ -58,7 +58,11 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
       VZMacOSBootLoader()
     }
 
-    func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
+    func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+      if needsNestedVirtualization {
+        throw RuntimeError.VMConfigurationError("macOS virtual machines do not support nested virtualization")
+      }
+
       let result = VZMacPlatformConfiguration()
 
       result.machineIdentifier = ecid
@@ -123,6 +127,11 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
       [VZUSBScreenCoordinatePointingDeviceConfiguration(), VZMacTrackpadConfiguration()]
     }
 
+    func pointingDevicesSimplified() -> [VZPointingDeviceConfiguration] {
+      // Only include the USB pointing device, not the trackpad
+      return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+    }
+
     func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration] {
       if #available(macOS 14, *) {
         return [VZMacTrackpadConfiguration()]

Sources/tart/Platform/Linux.swift

@@ -14,8 +14,12 @@ struct Linux: Platform {
     return result
   }
 
-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
-    VZGenericPlatformConfiguration()
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration {
+    let config = VZGenericPlatformConfiguration()
+    if #available(macOS 15, *) {
+      config.isNestedVirtualizationEnabled = needsNestedVirtualization
+    }
+    return config
   }
 
   func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
@@ -38,4 +42,9 @@ struct Linux: Platform {
   func pointingDevices() -> [VZPointingDeviceConfiguration] {
     [VZUSBScreenCoordinatePointingDeviceConfiguration()]
   }
+
+  func pointingDevicesSimplified() -> [VZPointingDeviceConfiguration] {
+    // Linux doesn't support trackpad, so just return the regular pointing devices
+    return pointingDevices()
+  }
 }

Sources/tart/Platform/Platform.swift

@@ -3,10 +3,11 @@ import Virtualization
 protocol Platform: Codable {
   func os() -> OS
   func bootLoader(nvramURL: URL) throws -> VZBootLoader
-  func platform(nvramURL: URL) throws -> VZPlatformConfiguration
+  func platform(nvramURL: URL, needsNestedVirtualization: Bool) throws -> VZPlatformConfiguration
   func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
   func keyboards() -> [VZKeyboardConfiguration]
   func pointingDevices() -> [VZPointingDeviceConfiguration]
+  func pointingDevicesSimplified() -> [VZPointingDeviceConfiguration]
 }
 
 protocol PlatformSuspendable: Platform {

Sources/tart/Root.swift

@@ -18,6 +18,7 @@ struct Root: AsyncParsableCommand {
       Login.self,
       Logout.self,
       IP.self,
+      Exec.self,
       Pull.self,
       Push.self,
       Import.self,
@@ -30,39 +31,6 @@ struct Root: AsyncParsableCommand {
     ])
 
   public static func main() async throws {
-    // Initialize Sentry
-    if let dsn = ProcessInfo.processInfo.environment["SENTRY_DSN"] {
-      SentrySDK.start { options in
-        options.dsn = dsn
-        options.releaseName = CI.release
-        options.tracesSampleRate = Float(
-          ProcessInfo.processInfo.environment["SENTRY_TRACES_SAMPLE_RATE"] ?? "1.0"
-        ) as NSNumber?
-
-        // By default only 5XX are captured
-        // Let's capture everything but 401 (unauthorized)
-        options.enableCaptureFailedRequests = true
-        options.failedRequestStatusCodes = [
-          HttpStatusCodeRange(min: 400, max: 400),
-          HttpStatusCodeRange(min: 402, max: 599)
-        ]
-      }
-    }
-    defer { SentrySDK.flush(timeout: 2.seconds.timeInterval) }
-
-    SentrySDK.configureScope { scope in
-      scope.setExtra(value: ProcessInfo.processInfo.arguments, key: "Command-line arguments")
-    }
-
-    // Enrich future events with Cirrus CI-specific tags
-    if let tags = ProcessInfo.processInfo.environment["CIRRUS_SENTRY_TAGS"] {
-      SentrySDK.configureScope { scope in
-        for (key, value) in tags.split(separator: ",").compactMap({ parseCirrusSentryTag($0) }) {
-          scope.setTag(value: value, key: key)
-        }
-      }
-    }
-
     // Add commands that are only available on specific macOS versions
     if #available(macOS 14, *) {
       configuration.subcommands.append(Suspend.self)
@@ -82,25 +50,64 @@ struct Root: AsyncParsableCommand {
     // Set line-buffered output for stdout
     setlinebuf(stdout)
 
-    // Parse and run command
     do {
+      // Parse command
       var command = try parseAsRoot()
 
+      // Initialize Sentry
+      if let dsn = ProcessInfo.processInfo.environment["SENTRY_DSN"] {
+        SentrySDK.start { options in
+          options.dsn = dsn
+          options.releaseName = CI.release
+          options.tracesSampleRate = Float(
+            ProcessInfo.processInfo.environment["SENTRY_TRACES_SAMPLE_RATE"] ?? "1.0"
+          ) as NSNumber?
+
+          // By default only 5XX are captured
+          // Let's capture everything but 401 (unauthorized)
+          options.enableCaptureFailedRequests = true
+          options.failedRequestStatusCodes = [
+            HttpStatusCodeRange(min: 400, max: 400),
+            HttpStatusCodeRange(min: 402, max: 599)
+          ]
+        }
+      }
+      defer { SentrySDK.flush(timeout: 2.seconds.timeInterval) }
+
+      SentrySDK.configureScope { scope in
+        scope.setExtra(value: ProcessInfo.processInfo.arguments, key: "Command-line arguments")
+      }
+
+      // Enrich future events with Cirrus CI-specific tags
+      if let tags = ProcessInfo.processInfo.environment["CIRRUS_SENTRY_TAGS"] {
+        SentrySDK.configureScope { scope in
+          for (key, value) in tags.split(separator: ",").compactMap({ parseCirrusSentryTag($0) }) {
+            scope.setTag(value: value, key: key)
+          }
+        }
+      }
+
       // Run garbage-collection before each command (shouldn't take too long)
       if type(of: command) != type(of: Pull()) && type(of: command) != type(of: Clone()){
         do {
           try Config().gc()
         } catch {
-          fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+          fputs("Failed to perform garbage collection: \(error)\n", stderr)
         }
       }
 
+      // Run command
       if var asyncCommand = command as? AsyncParsableCommand {
         try await asyncCommand.run()
       } else {
         try command.run()
       }
     } catch {
+      // Not an error, just a custom exit code from "tart exec"
+      if let execCustomExitCodeError = error as? ExecCustomExitCodeError {
+        Foundation.exit(execCustomExitCodeError.exitCode)
+      }
+
       // Capture the error into Sentry
       SentrySDK.capture(error: error)
       SentrySDK.flush(timeout: 2.seconds.timeInterval)

Sources/tart/ShellCompletions/ShellCompletions.swift

@@ -5,7 +5,7 @@ fileprivate func normalizeName(_ name: String) -> String {
   return name.replacingOccurrences(of: ":", with: "\\:")
 }
 
-func completeMachines(_ arguments: [String]) -> [String] {
+func completeMachines(_ arguments: [String], _ argumentIdx: Int, _ argumentPrefix: String) -> [String] {
   let localVMs = (try? VMStorageLocal().list().map { name, _ in
     normalizeName(name)
   }) ?? []
@@ -15,12 +15,12 @@ func completeMachines(_ arguments: [String]) -> [String] {
   return (localVMs + ociVMs)
 }
 
-func completeLocalMachines(_ arguments: [String]) -> [String] {
+func completeLocalMachines(_ arguments: [String], _ argumentIdx: Int, _ argumentPrefix: String) -> [String] {
   let localVMs = (try? VMStorageLocal().list()) ?? []
   return localVMs.map { name, _ in normalizeName(name) }
 }
 
-func completeRunningMachines(_ arguments: [String]) -> [String] {
+func completeRunningMachines(_ arguments: [String], _ argumentIdx: Int, _ argumentPrefix: String) -> [String] {
   let localVMs = (try? VMStorageLocal().list()) ?? []
   return localVMs
     .filter { _, vmDir in (try? vmDir.state() == .Running) ?? false}

Sources/tart/Term.swift

@@ -0,0 +1,60 @@
+import Foundation
+import System
+
+struct State {
+  fileprivate let termios: termios
+}
+
+class Term {
+  static func IsTerminal() -> Bool {
+    var termios = termios()
+
+    return tcgetattr(FileHandle.standardInput.fileDescriptor, &termios) != -1
+  }
+
+  static func MakeRaw() throws -> State {
+    var termiosOrig = termios()
+
+    var ret = tcgetattr(FileHandle.standardInput.fileDescriptor, &termiosOrig)
+    if ret == -1 {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.TerminalOperationFailed("failed to retrieve terminal parameters: \(details)")
+    }
+
+    var termiosRaw = termiosOrig
+    cfmakeraw(&termiosRaw)
+
+    ret = tcsetattr(FileHandle.standardInput.fileDescriptor, TCSANOW, &termiosRaw)
+    if ret == -1 {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.TerminalOperationFailed("failed to set terminal parameters: \(details)")
+    }
+
+    return State(termios: termiosOrig)
+  }
+
+  static func Restore(_ state: State) throws {
+    var termios = state.termios
+
+    let ret = tcsetattr(FileHandle.standardInput.fileDescriptor, TCSANOW, &termios)
+    if ret == -1 {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.TerminalOperationFailed("failed to set terminal parameters: \(details)")
+    }
+  }
+
+  static func GetSize() throws -> (width: UInt16, height: UInt16) {
+    var winsize = winsize()
+
+    guard ioctl(STDOUT_FILENO, TIOCGWINSZ, &winsize) != -1 else {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.TerminalOperationFailed("failed to get terminal size: \(details)")
+    }
+
+    return (width: winsize.ws_col, height: winsize.ws_row)
+  }
+}

Sources/tart/URL+AccessDate.swift

@@ -1,4 +1,5 @@
 import Foundation
+import System
 
 extension URL {
   func accessDate() throws -> Date {
@@ -13,7 +14,9 @@ extension URL {
     let times = [accessDate.asTimeval(), modificationDate.asTimeval()]
     let ret = utimes(path, times)
     if ret != 0 {
-      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(ret.explanation())")
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.FailedToUpdateAccessDate("utimes(2) failed: \(details)")
     }
   }
 }

Sources/tart/URL+Prunable.swift

@@ -1,4 +1,5 @@
 import Foundation
+import XAttr
 
 extension URL: Prunable {
   var url: URL {
@@ -13,7 +14,31 @@ extension URL: Prunable {
     try resourceValues(forKeys: [.totalFileAllocatedSizeKey]).totalFileAllocatedSize!
   }
 
+  func deduplicatedSizeBytes() throws -> Int {
+    let values = try resourceValues(forKeys: [.totalFileAllocatedSizeKey, .mayShareFileContentKey])
+    // make sure the file's origin file is there and duplication works
+    if values.mayShareFileContent == true {
+      return Int(deduplicatedBytes())
+    }
+    return 0
+  }
+
   func sizeBytes() throws -> Int {
     try resourceValues(forKeys: [.totalFileSizeKey]).totalFileSize!
   }
+
+  func setDeduplicatedBytes(_ size: UInt64) {
+    let data = "\(size)".data(using: .utf8)!
+    try! self.setExtendedAttribute(name: "run.tart.deduplicated-bytes", value: data)
+  }
+
+  func deduplicatedBytes() -> UInt64 {
+    guard let data = try? self.extendedAttributeValue(forName: "run.tart.deduplicated-bytes") else {
+      return 0
+    }
+    if let strValue = String(data: data, encoding: .utf8) {
+      return UInt64(strValue) ?? 0
+    }
+    return 0
+  }
 }

Sources/tart/VM.swift

@@ -1,6 +1,5 @@
 import Foundation
 import Virtualization
-import AsyncAlgorithms
 import Semaphore
 
 struct UnsupportedRestoreImageError: Error {
@@ -47,8 +46,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
        directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
        serialPorts: [VZSerialPortConfiguration] = [],
        suspendable: Bool = false,
+       nested: Bool = false,
        audio: Bool = true,
-       clipboard: Bool = true
+       clipboard: Bool = true,
+       sync: VZDiskImageSynchronizationMode = .full,
+       caching: VZDiskImageCachingMode? = nil,
+       noTrackpad: Bool = false
   ) throws {
     name = vmDir.name
     config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -65,8 +68,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
                                                 directorySharingDevices: directorySharingDevices,
                                                 serialPorts: serialPorts,
                                                 suspendable: suspendable,
+                                                nested: nested,
                                                 audio: audio,
-                                                clipboard: clipboard
+                                                clipboard: clipboard,
+                                                sync: sync,
+                                                caching: caching,
+                                                noTrackpad: noTrackpad
     )
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
@@ -94,16 +101,13 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Download the IPSW
     defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")
 
-    let downloadProgress = Progress(totalUnitCount: 100)
-    ProgressObserver(downloadProgress).log(defaultLogger)
-
     let request = URLRequest(url: remoteURL)
-    let (channel, response) = try await Fetcher.fetch(request, viaFile: true, progress: downloadProgress)
+    let (channel, response) = try await Fetcher.fetch(request, viaFile: true)
 
     let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".ipsw")
-    defaultLogger.appendNewLine("Computing digest for \(temporaryLocation.path)...")
-    let digestProgress = Progress(totalUnitCount: response.expectedContentLength)
-    ProgressObserver(digestProgress).log(defaultLogger)
+
+    let progress = Progress(totalUnitCount: response.expectedContentLength)
+    ProgressObserver(progress).log(defaultLogger)
 
     FileManager.default.createFile(atPath: temporaryLocation.path, contents: nil)
     let lock = try FileLock(lockURL: temporaryLocation)
@@ -113,10 +117,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     let digest = Digest()
 
     for try await chunk in channel {
-      let chunkAsData = Data(chunk)
-      fileHandle.write(chunkAsData)
-      digest.update(chunkAsData)
-      digestProgress.completedUnitCount += Int64(chunk.count)
+      try fileHandle.write(contentsOf: chunk)
+      digest.update(chunk)
+      progress.completedUnitCount += Int64(chunk.count)
     }
 
     try fileHandle.close()
@@ -140,6 +143,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       vmDir: VMDirectory,
       ipswURL: URL,
       diskSizeGB: UInt16,
+      diskFormat: DiskImageFormat = .raw,
       network: Network = NetworkShared(),
       additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
       directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
@@ -172,14 +176,15 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
       _ = try VZMacAuxiliaryStorage(creatingStorageAt: vmDir.nvramURL, hardwareModel: requirements.hardwareModel)
 
       // Create disk
-      try vmDir.resizeDisk(diskSizeGB)
+      try vmDir.resizeDisk(diskSizeGB, format: diskFormat)
 
       name = vmDir.name
       // Create config
       config = VMConfig(
         platform: Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: requirements.hardwareModel),
         cpuCountMin: requirements.minimumSupportedCPUCount,
-        memorySizeMin: requirements.minimumSupportedMemorySize
+        memorySizeMin: requirements.minimumSupportedMemorySize,
+        diskFormat: diskFormat
       )
       // allocate at least 4 CPUs because otherwise VMs are frequently freezing
       try config.setCPU(cpuCount: max(4, requirements.minimumSupportedCPUCount))
@@ -221,15 +226,15 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   #endif
 
   @available(macOS 13, *)
-  static func linux(vmDir: VMDirectory, diskSizeGB: UInt16) async throws -> VM {
+  static func linux(vmDir: VMDirectory, diskSizeGB: UInt16, diskFormat: DiskImageFormat = .raw) async throws -> VM {
     // Create NVRAM
     _ = try VZEFIVariableStore(creatingVariableStoreAt: vmDir.nvramURL)
 
     // Create disk
-    try vmDir.resizeDisk(diskSizeGB)
+    try vmDir.resizeDisk(diskSizeGB, format: diskFormat)
 
     // Create config
-    let config = VMConfig(platform: Linux(), cpuCountMin: 4, memorySizeMin: 4096 * 1024 * 1024)
+    let config = VMConfig(platform: Linux(), cpuCountMin: 4, memorySizeMin: 4096 * 1024 * 1024, diskFormat: diskFormat)
     try config.save(toURL: vmDir.configURL)
 
     return try VM(vmDir: vmDir)
@@ -245,6 +250,19 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     }
   }
 
+  @MainActor
+  func connect(toPort: UInt32) async throws -> VZVirtioSocketConnection {
+    guard let socketDevice = virtualMachine.socketDevices.first else {
+      throw RuntimeError.VMSocketFailed(toPort, ", VM has no socket devices configured")
+    }
+
+    guard let virtioSocketDevice = socketDevice as? VZVirtioSocketDevice else {
+      throw RuntimeError.VMSocketFailed(toPort, ", expected VM's first socket device to have a type of VZVirtioSocketDevice, got \(type(of: socketDevice)) instead")
+    }
+
+    return try await virtioSocketDevice.connect(toPort: toPort)
+  }
+
   func run() async throws {
     do {
       try await sema.waitUnlessCancelled()
@@ -293,8 +311,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
     serialPorts: [VZSerialPortConfiguration],
     suspendable: Bool = false,
+    nested: Bool = false,
     audio: Bool = true,
-    clipboard: Bool = true
+    clipboard: Bool = true,
+    sync: VZDiskImageSynchronizationMode = .full,
+    caching: VZDiskImageCachingMode? = nil,
+    noTrackpad: Bool = false
   ) throws -> VZVirtualMachineConfiguration {
     let configuration = VZVirtualMachineConfiguration()
 
@@ -306,29 +328,40 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     configuration.memorySize = vmConfig.memorySize
 
     // Platform
-    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL)
+    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL, needsNestedVirtualization: nested)
 
     // Display
     configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]
 
     // Audio
+    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+
     if audio && !suspendable {
-      let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
       let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
       let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+
+      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
       outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+
       soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
-      configuration.audioDevices = [soundDeviceConfiguration]
+    } else {
+      // just a null speaker
+      soundDeviceConfiguration.streams = [VZVirtioSoundDeviceOutputStreamConfiguration()]
     }
 
+    configuration.audioDevices = [soundDeviceConfiguration]
+
     // Keyboard and mouse
     if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
       configuration.keyboards = platformSuspendable.keyboardsSuspendable()
       configuration.pointingDevices = platformSuspendable.pointingDevicesSuspendable()
     } else {
       configuration.keyboards = vmConfig.platform.keyboards()
-      configuration.pointingDevices = vmConfig.platform.pointingDevices()
+      if noTrackpad {
+        configuration.pointingDevices = vmConfig.platform.pointingDevicesSimplified()
+      } else {
+        configuration.pointingDevices = vmConfig.platform.pointingDevices()
+      }
     }
 
     // Networking
@@ -340,28 +373,29 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     }
 
     // Clipboard sharing via Spice agent
-    if clipboard && vmConfig.os == .linux {
+    if clipboard {
       let spiceAgentConsoleDevice = VZVirtioConsoleDeviceConfiguration()
       let spiceAgentPort = VZVirtioConsolePortConfiguration()
       spiceAgentPort.name = VZSpiceAgentPortAttachment.spiceAgentPortName
-      spiceAgentPort.attachment = VZSpiceAgentPortAttachment()
+      let spiceAgentPortAttachment = VZSpiceAgentPortAttachment()
+      spiceAgentPortAttachment.sharesClipboard = true
+      spiceAgentPort.attachment = spiceAgentPortAttachment
       spiceAgentConsoleDevice.ports[0] = spiceAgentPort
       configuration.consoleDevices.append(spiceAgentConsoleDevice)
     }
 
     // Storage
-    let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
-      // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: .full) :
-      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
+    var attachment = try VZDiskImageStorageDeviceAttachment(
+      url: diskURL,
+      readOnly: false,
+      // When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1]
+      //
+      // [1]: https://github.com/cirruslabs/tart/pull/675
+      cachingMode: caching ?? (vmConfig.os == .linux ? .cached : .automatic),
+      synchronizationMode: sync
+    )
 
-    var device: VZStorageDeviceConfiguration
-    if #available(macOS 14, *), vmConfig.os == .linux {
-      device = VZNVMExpressControllerDeviceConfiguration(attachment: attachment)
-    } else {
-      device = VZVirtioBlockDeviceConfiguration(attachment: attachment)
-    }
-    var devices: [VZStorageDeviceConfiguration] = [device]
+    var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)]
     devices.append(contentsOf: additionalStorageDevices)
     configuration.storageDevices = devices
 
@@ -380,15 +414,16 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     //
     // A dummy console device useful for implementing
     // host feature checks in the guest agent software.
-    if !suspendable {
-      let consolePort = VZVirtioConsolePortConfiguration()
-      consolePort.name = "tart-version-\(CI.version)"
+    let consolePort = VZVirtioConsolePortConfiguration()
+    consolePort.name = "tart-version-\(CI.version)"
 
-      let consoleDevice = VZVirtioConsoleDeviceConfiguration()
-      consoleDevice.ports[0] = consolePort
+    let consoleDevice = VZVirtioConsoleDeviceConfiguration()
+    consoleDevice.ports[0] = consolePort
 
-      configuration.consoleDevices.append(consoleDevice)
-    }
+    configuration.consoleDevices.append(consoleDevice)
+
+    // Socket device
+    configuration.socketDevices = [VZVirtioSocketDeviceConfiguration()]
 
     try configuration.validate()
 

Sources/tart/VMConfig.swift

@@ -24,6 +24,8 @@ enum CodingKeys: String, CodingKey {
   case memorySize
   case macAddress
   case display
+  case displayRefit
+  case diskFormat
 
   // macOS-specific keys
   case ecid
@@ -52,12 +54,15 @@ struct VMConfig: Codable {
   private(set) var memorySize: UInt64
   var macAddress: VZMACAddress
   var display: VMDisplayConfig = VMDisplayConfig()
+  var displayRefit: Bool?
+  var diskFormat: DiskImageFormat = .raw
 
   init(
     platform: Platform,
     cpuCountMin: Int,
     memorySizeMin: UInt64,
-    macAddress: VZMACAddress = VZMACAddress.randomLocallyAdministered()
+    macAddress: VZMACAddress = VZMACAddress.randomLocallyAdministered(),
+    diskFormat: DiskImageFormat = .raw
   ) {
     self.os = platform.os()
     self.arch = CurrentArchitecture()
@@ -65,6 +70,7 @@ struct VMConfig: Codable {
     self.macAddress = macAddress
     self.cpuCountMin = cpuCountMin
     self.memorySizeMin = memorySizeMin
+    self.diskFormat = diskFormat
     cpuCount = cpuCountMin
     memorySize = memorySizeMin
   }
@@ -121,6 +127,9 @@ struct VMConfig: Codable {
     self.macAddress = macAddress
 
     display = try container.decodeIfPresent(VMDisplayConfig.self, forKey: .display) ?? VMDisplayConfig()
+    displayRefit = try container.decodeIfPresent(Bool.self, forKey: .displayRefit)
+    let diskFormatString = try container.decodeIfPresent(String.self, forKey: .diskFormat) ?? "raw"
+    diskFormat = DiskImageFormat(rawValue: diskFormatString) ?? .raw
   }
 
   func encode(to encoder: Encoder) throws {
@@ -136,6 +145,10 @@ struct VMConfig: Codable {
     try container.encode(memorySize, forKey: .memorySize)
     try container.encode(macAddress.string, forKey: .macAddress)
     try container.encode(display, forKey: .display)
+    if let displayRefit = displayRefit {
+      try container.encode(displayRefit, forKey: .displayRefit)
+    }
+    try container.encode(diskFormat.rawValue, forKey: .diskFormat)
   }
 
   mutating func setCPU(cpuCount: Int) throws {

Sources/tart/VMDirectory+OCI.swift

@@ -1,3 +1,4 @@
+import Compression
 import Foundation
 import Sentry
 
@@ -10,10 +11,7 @@ enum OCIError: Error {
 }
 
 extension VMDirectory {
-  private static let bufferSizeBytes = 64 * 1024 * 1024
-  private static let layerLimitBytes = 500 * 1000 * 1000
-
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
     // Pull VM's config file layer and re-serialize it into a config file
     let configLayers = manifest.layers.filter {
       $0.mediaType == configMediaType
@@ -26,7 +24,7 @@ extension VMDirectory {
     }
     let configFile = try FileHandle(forWritingTo: configURL)
     try await registry.pullBlob(configLayers.first!.digest) { data in
-      configFile.write(data)
+      try configFile.write(contentsOf: data)
     }
     try configFile.close()
 
@@ -53,8 +51,19 @@ extension VMDirectory {
     let progress = Progress(totalUnitCount: diskCompressedSize)
     ProgressObserver(progress).log(defaultLogger)
 
-    try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL, concurrency: concurrency, progress: progress,
-                                localLayerCache: localLayerCache)
+    do {
+      try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
+                                  concurrency: concurrency, progress: progress,
+                                  localLayerCache: localLayerCache,
+                                  deduplicate: deduplicate)
+    } catch let error where error is FilterError {
+      throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
+    }
+
+    if deduplicate, let llc = localLayerCache {
+      // set custom attribute to remember deduplicated bytes
+      diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
+    }
 
     // Pull VM's NVRAM file layer and store it in an NVRAM file
     defaultLogger.appendNewLine("pulling NVRAM...")
@@ -70,16 +79,23 @@ extension VMDirectory {
     }
     let nvram = try FileHandle(forWritingTo: nvramURL)
     try await registry.pullBlob(nvramLayers.first!.digest) { data in
-      nvram.write(data)
+      try nvram.write(contentsOf: data)
     }
     try nvram.close()
+
+    // Serialize VM's manifest to enable better deduplication on subsequent "tart pull"'s
+    try manifest.toJSON().write(to: manifestURL)
   }
 
-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String, concurrency: UInt, labels: [String: String] = [:]) async throws -> RemoteName {
     var layers = Array<OCIManifestLayer>()
 
     // Read VM's config and push it as blob
     let config = try VMConfig(fromURL: configURL)
+
+    // Add disk format label automatically
+    var labels = labels
+    labels[diskFormatLabel] = config.diskFormat.rawValue
     let configJSON = try JSONEncoder().encode(config)
     defaultLogger.appendNewLine("pushing config...")
     let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb)
@@ -94,9 +110,9 @@ extension VMDirectory {
 
     switch diskFormat {
     case "v1":
-      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
     case "v2":
-      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
     default:
       throw RuntimeError.OCIUnsupportedDiskFormat(diskFormat)
     }
@@ -109,7 +125,8 @@ extension VMDirectory {
     layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest))
 
     // Craft a stub OCI config for Docker Hub compatibility
-    let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os).toJSON()
+    let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels)
+    let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON()
     let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb)
     let manifest = OCIManifest(
       config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),

Sources/tart/VMDirectory.swift

@@ -2,6 +2,25 @@ import Foundation
 import Virtualization
 import CryptoKit
 
+// MARK: - Disk Image Info Structures
+struct DiskImageInfo: Codable {
+  let sizeInfo: SizeInfo?
+  let size: UInt64?
+
+  enum CodingKeys: String, CodingKey {
+    case sizeInfo = "Size Info"
+    case size = "Size"
+  }
+}
+
+struct SizeInfo: Codable {
+  let totalBytes: UInt64?
+
+  enum CodingKeys: String, CodingKey {
+    case totalBytes = "Total Bytes"
+  }
+}
+
 struct VMDirectory: Prunable {
   enum State: String {
     case Running = "running"
@@ -23,6 +42,12 @@ struct VMDirectory: Prunable {
   var stateURL: URL {
     baseURL.appendingPathComponent("state.vzvmsave")
   }
+  var manifestURL: URL {
+    baseURL.appendingPathComponent("manifest.json")
+  }
+  var controlSocketURL: URL {
+    baseURL.appendingPathComponent("control.sock")
+  }
 
   var explicitlyPulledMark: URL {
     baseURL.appendingPathComponent(".explicitly-pulled")
@@ -136,14 +161,34 @@ struct VMDirectory: Prunable {
     try vmConfig.save(toURL: configURL)
   }
 
-  func resizeDisk(_ sizeGB: UInt16) throws {
-    if !FileManager.default.fileExists(atPath: diskURL.path) {
-      FileManager.default.createFile(atPath: diskURL.path, contents: nil, attributes: nil)
-    }
+  func resizeDisk(_ sizeGB: UInt16, format: DiskImageFormat = .raw) throws {
+    let diskExists = FileManager.default.fileExists(atPath: diskURL.path)
 
+    if diskExists {
+      // Existing disk - resize it
+      try resizeExistingDisk(sizeGB)
+    } else {
+      // New disk - create it with the specified format
+      try createDisk(sizeGB: sizeGB, format: format)
+    }
+  }
+
+  private func resizeExistingDisk(_ sizeGB: UInt16) throws {
+    // Check if this is an ASIF disk by reading the VM config
+    let vmConfig = try VMConfig(fromURL: configURL)
+
+    if vmConfig.diskFormat == .asif {
+      try resizeASIFDisk(sizeGB)
+    } else {
+      try resizeRawDisk(sizeGB)
+    }
+  }
+
+  private func resizeRawDisk(_ sizeGB: UInt16) throws {
     let diskFileHandle = try FileHandle.init(forWritingTo: diskURL)
     let currentDiskFileLength = try diskFileHandle.seekToEnd()
     let desiredDiskFileLength = UInt64(sizeGB) * 1000 * 1000 * 1000
+
     if desiredDiskFileLength < currentDiskFileLength {
       let currentLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(currentDiskFileLength))
       let desiredLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(desiredDiskFileLength))
@@ -155,6 +200,157 @@ struct VMDirectory: Prunable {
     try diskFileHandle.close()
   }
 
+  private func resizeASIFDisk(_ sizeGB: UInt16) throws {
+    guard let diskutilURL = resolveBinaryPath("diskutil") else {
+      throw RuntimeError.FailedToResizeDisk("diskutil not found in PATH")
+    }
+
+    // First, get current disk image info to check current size
+    let infoProcess = Process()
+    infoProcess.executableURL = diskutilURL
+    infoProcess.arguments = ["image", "info", "--plist", diskURL.path]
+
+    let infoPipe = Pipe()
+    infoProcess.standardOutput = infoPipe
+    infoProcess.standardError = infoPipe
+
+    do {
+      try infoProcess.run()
+      infoProcess.waitUntilExit()
+
+      let infoData = infoPipe.fileHandleForReading.readDataToEndOfFile()
+
+      if infoProcess.terminationStatus != 0 {
+        let output = String(data: infoData, encoding: .utf8) ?? "Unknown error"
+        throw RuntimeError.FailedToResizeDisk("Failed to get ASIF disk info: \(output)")
+      }
+
+      // Parse the plist using PropertyListDecoder
+      do {
+        let diskImageInfo = try PropertyListDecoder().decode(DiskImageInfo.self, from: infoData)
+
+        // Extract current size from the decoded structure
+        var currentSizeBytes: UInt64?
+
+        // Try to get size from Size Info -> Total Bytes first
+        if let totalBytes = diskImageInfo.sizeInfo?.totalBytes {
+          currentSizeBytes = totalBytes
+        } else if let size = diskImageInfo.size {
+          // Fallback to top-level Size field
+          currentSizeBytes = size
+        }
+
+        guard let currentSizeBytes = currentSizeBytes else {
+          throw RuntimeError.FailedToResizeDisk("Could not find size information in disk image info")
+        }
+
+        let desiredSizeBytes = UInt64(sizeGB) * 1000 * 1000 * 1000
+
+        if desiredSizeBytes < currentSizeBytes {
+          let currentLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(currentSizeBytes))
+          let desiredLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(desiredSizeBytes))
+          throw RuntimeError.InvalidDiskSize("new disk size of \(desiredLengthHuman) should be larger " +
+            "than the current disk size of \(currentLengthHuman)")
+        } else if desiredSizeBytes > currentSizeBytes {
+          // Resize the ASIF disk image using diskutil
+          try performASIFResize(sizeGB)
+        }
+        // If sizes are equal, no action needed
+      } catch let error as RuntimeError {
+        throw error
+      } catch {
+        let outputString = String(data: infoData, encoding: .utf8) ?? "Unable to decode output"
+        throw RuntimeError.FailedToResizeDisk("Failed to parse disk image info: \(error). Output: \(outputString)")
+      }
+    } catch {
+      throw RuntimeError.FailedToResizeDisk("Failed to get disk image info: \(error)")
+    }
+  }
+
+  private func performASIFResize(_ sizeGB: UInt16) throws {
+    guard let diskutilURL = resolveBinaryPath("diskutil") else {
+      throw RuntimeError.FailedToResizeDisk("diskutil not found in PATH")
+    }
+
+    let process = Process()
+    process.executableURL = diskutilURL
+    process.arguments = [
+      "image", "resize",
+      "--size", "\(sizeGB)G",
+      diskURL.path
+    ]
+
+    let pipe = Pipe()
+    process.standardOutput = pipe
+    process.standardError = pipe
+
+    do {
+      try process.run()
+      process.waitUntilExit()
+
+      let data = pipe.fileHandleForReading.readDataToEndOfFile()
+
+      if process.terminationStatus != 0 {
+        let output = String(data: data, encoding: .utf8) ?? "Unknown error"
+        throw RuntimeError.FailedToResizeDisk("Failed to resize ASIF disk image: \(output)")
+      }
+    } catch {
+      throw RuntimeError.FailedToResizeDisk("Failed to execute diskutil resize: \(error)")
+    }
+  }
+
+  private func createDisk(sizeGB: UInt16, format: DiskImageFormat) throws {
+    switch format {
+    case .raw:
+      try createRawDisk(sizeGB: sizeGB)
+    case .asif:
+      try createASIFDisk(sizeGB: sizeGB)
+    }
+  }
+
+  private func createRawDisk(sizeGB: UInt16) throws {
+    // Create traditional raw disk image
+    FileManager.default.createFile(atPath: diskURL.path, contents: nil, attributes: nil)
+
+    let diskFileHandle = try FileHandle.init(forWritingTo: diskURL)
+    let desiredDiskFileLength = UInt64(sizeGB) * 1000 * 1000 * 1000
+    try diskFileHandle.truncate(atOffset: desiredDiskFileLength)
+    try diskFileHandle.close()
+  }
+
+  private func createASIFDisk(sizeGB: UInt16) throws {
+    guard let diskutilURL = resolveBinaryPath("diskutil") else {
+      throw RuntimeError.FailedToCreateDisk("diskutil not found in PATH")
+    }
+
+    let process = Process()
+    process.executableURL = diskutilURL
+    process.arguments = [
+      "image", "create", "blank",
+      "--format", "ASIF",
+      "--size", "\(sizeGB)G",
+      "--volumeName", "Tart",
+      diskURL.path
+    ]
+
+    let pipe = Pipe()
+    process.standardOutput = pipe
+    process.standardError = pipe
+
+    do {
+      try process.run()
+      process.waitUntilExit()
+
+      if process.terminationStatus != 0 {
+        let data = pipe.fileHandleForReading.readDataToEndOfFile()
+        let output = String(data: data, encoding: .utf8) ?? "Unknown error"
+        throw RuntimeError.FailedToCreateDisk("Failed to create ASIF disk image: \(output)")
+      }
+    } catch {
+      throw RuntimeError.FailedToCreateDisk("Failed to execute diskutil: \(error)")
+    }
+  }
+
   func delete() throws {
     let lock = try lock()
 
@@ -179,6 +375,14 @@ struct VMDirectory: Prunable {
     try allocatedSizeBytes() / 1000 / 1000 / 1000
   }
 
+  func deduplicatedSizeBytes() throws -> Int {
+    try configURL.deduplicatedSizeBytes() + diskURL.deduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes()
+  }
+
+  func deduplicatedSizeGB() throws -> Int {
+    try deduplicatedSizeBytes() / 1000 / 1000 / 1000
+  }
+
   func sizeBytes() throws -> Int {
     try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes()
   }

Sources/tart/VMStorageHelper.swift

@@ -24,6 +24,8 @@ class VMStorageHelper {
   private static func missingVMWrap<R: Any>(_ name: String, closure: () throws -> R) throws -> R {
     do {
       return try closure()
+    } catch RuntimeError.PIDLockMissing {
+      throw RuntimeError.VMDoesNotExist(name: name)
     } catch {
       if error.isFileNotFound() {
         throw RuntimeError.VMDoesNotExist(name: name)
@@ -47,6 +49,7 @@ extension Error {
 }
 
 enum RuntimeError : Error {
+  case Generic(_ message: String)
   case VMConfigurationError(_ message: String)
   case VMDoesNotExist(name: String)
   case VMMissingFiles(_ message: String)
@@ -57,10 +60,14 @@ enum RuntimeError : Error {
   case DiskAlreadyInUse(_ message: String)
   case FailedToOpenBlockDevice(_ path: String, _ explanation: String)
   case InvalidDiskSize(_ message: String)
+  case FailedToCreateDisk(_ message: String)
+  case FailedToResizeDisk(_ message: String)
   case FailedToUpdateAccessDate(_ message: String)
   case PIDLockFailed(_ message: String)
+  case PIDLockMissing(_ message: String)
   case FailedToParseRemoteName(_ message: String)
   case VMTerminationFailed(_ message: String)
+  case ImproperlyFormattedHost(_ host: String, _ hint: String)
   case InvalidCredentials(_ message: String)
   case VMDirectoryAlreadyInitialized(_ message: String)
   case ExportFailed(_ message: String)
@@ -69,6 +76,10 @@ enum RuntimeError : Error {
   case OCIStorageError(_ message: String)
   case OCIUnsupportedDiskFormat(_ format: String)
   case SuspendFailed(_ message: String)
+  case PullFailed(_ message: String)
+  case VirtualMachineLimitExceeded(_ hint: String)
+  case VMSocketFailed(_ port: UInt32, _ explanation: String)
+  case TerminalOperationFailed(_ message: String)
 }
 
 protocol HasExitCode {
@@ -78,6 +89,8 @@ protocol HasExitCode {
 extension RuntimeError : CustomStringConvertible {
   public var description: String {
     switch self {
+    case .Generic(let message):
+      return message
     case .VMConfigurationError(let message):
       return message
     case .VMDoesNotExist(let name):
@@ -98,14 +111,22 @@ extension RuntimeError : CustomStringConvertible {
       return "failed to open block device \(path): \(explanation)"
     case .InvalidDiskSize(let message):
       return message
+    case .FailedToCreateDisk(let message):
+      return message
+    case .FailedToResizeDisk(let message):
+      return message
     case .FailedToUpdateAccessDate(let message):
       return message
     case .PIDLockFailed(let message):
       return message
+    case .PIDLockMissing(let message):
+      return message
     case .FailedToParseRemoteName(let cause):
       return "failed to parse remote name: \(cause)"
     case .VMTerminationFailed(let message):
       return message
+    case .ImproperlyFormattedHost(let host, let hint):
+      return "improperly formatted host \"\(host)\" was provided\(hint)"
     case .InvalidCredentials(let message):
       return message
     case .VMDirectoryAlreadyInitialized(let message):
@@ -122,6 +143,14 @@ extension RuntimeError : CustomStringConvertible {
       return "OCI disk format \(format) is not supported by this version of Tart"
     case .SuspendFailed(let message):
       return "Failed to suspend the VM: \(message)"
+    case .PullFailed(let message):
+      return message
+    case .VirtualMachineLimitExceeded(let hint):
+      return "The number of VMs exceeds the system limit\(hint)"
+    case .VMSocketFailed(let port, let explanation):
+      return "Failed to establish a VM socket connection to port \(port): \(explanation)"
+    case .TerminalOperationFailed(let message):
+      return message
     }
   }
 }

Sources/tart/VMStorageLocal.swift

@@ -1,7 +1,11 @@
 import Foundation
 
 class VMStorageLocal: PrunableStorage {
-  let baseURL: URL = try! Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)
+  let baseURL: URL
+
+  init() throws {
+    baseURL = try Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)
+  }
 
   private func vmURL(_ name: String) -> URL {
     baseURL.appendingPathComponent(name, isDirectory: true)
@@ -66,7 +70,7 @@ class VMStorageLocal: PrunableStorage {
   }
 
   func prunables() throws -> [Prunable] {
-    try list().map { (_, vmDir) in vmDir }
+    try list().map { (_, vmDir) in vmDir }.filter { try !$0.running() }
   }
 
   func hasVMsWithMACAddress(macAddress: String) throws -> Bool {

Sources/tart/VMStorageOCI.swift

@@ -3,7 +3,11 @@ import Sentry
 import Retry
 
 class VMStorageOCI: PrunableStorage {
-  let baseURL = try! Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
+  let baseURL: URL
+
+  init() throws {
+    baseURL = try Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
+  }
 
   private func vmURL(_ name: RemoteName) -> URL {
     baseURL.appendingRemoteName(name)
@@ -27,12 +31,12 @@ class VMStorageOCI: PrunableStorage {
     return digest
   }
 
-  func open(_ name: RemoteName) throws -> VMDirectory {
+  func open(_ name: RemoteName, _ accessDate: Date = Date()) throws -> VMDirectory {
     let vmDir = VMDirectory(baseURL: vmURL(name))
 
     try vmDir.validate(userFriendlyName: name.description)
 
-    try vmDir.baseURL.updateAccessDate()
+    try vmDir.baseURL.updateAccessDate(accessDate)
 
     return vmDir
   }
@@ -140,7 +144,7 @@ class VMStorageOCI: PrunableStorage {
     try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
   }
 
-  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
     SentrySDK.configureScope { scope in
       scope.setContext(value: ["imageName": name.description], key: "OCI")
     }
@@ -180,6 +184,10 @@ class VMStorageOCI: PrunableStorage {
       let transaction = SentrySDK.startTransaction(name: name.description, operation: "pull", bindToScope: true)
       let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description)
 
+      // Open an existing VM directory corresponding to this name, if any,
+      // marking it as outdated to speed up the garbage collection process
+      _ = try? open(name, Date(timeIntervalSince1970: 0))
+
       // Lock the temporary VM directory to prevent it's garbage collection
       let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
       try tmpVMDirLock.lock()
@@ -196,22 +204,29 @@ class VMStorageOCI: PrunableStorage {
       }
 
       try await withTaskCancellationHandler(operation: {
-        try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) {
-          var localLayerCache: LocalLayerCache? = nil
+        try await retry(maxAttempts: 5) {
+          // Choose the best base image which has the most deduplication ratio
+          let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
 
-          if name.reference.type == .Tag,
-             let vmDir = try? open(name),
-             let digest = try? digest(name),
-             let (manifest, _) = try? await registry.pullManifest(reference: digest) {
-            localLayerCache = try LocalLayerCache(vmDir.diskURL, manifest)
+          if let llc = localLayerCache {
+            let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)
+
+            if deduplicate {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to deduplicate \(deduplicatedHuman), using it as a base...")
+            } else {
+              defaultLogger.appendNewLine("found an image \(llc.name) that will allow us to avoid fetching \(deduplicatedHuman), will try use it...")
+            }
           }
 
-          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache)
+          try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache, deduplicate: deduplicate)
         } recoverFromFailure: { error in
-          print("Error: \(error.localizedDescription)")
-          print("Attempting to re-try...")
+          if error is URLError {
+            print("Error pulling image: \"\(error.localizedDescription)\", attempting to re-try...")
 
-          return .retry
+            return .retry
+          }
+
+          return .throw
         }
         try move(digestName, from: tmpVMDir)
         transaction.finish()
@@ -231,6 +246,9 @@ class VMStorageOCI: PrunableStorage {
       // are excluded from garbage collection
       VMDirectory(baseURL: vmURL(name)).markExplicitlyPulled()
     }
+
+    // to explicitly set the image as being accessed so it won't get pruned immediately
+    _ = try VMStorageOCI().open(name)
   }
 
   func linked(from: RemoteName, to: RemoteName) -> Bool {
@@ -249,6 +267,59 @@ class VMStorageOCI: PrunableStorage {
 
     try gc()
   }
+
+  func chooseLocalLayerCache(_ name: RemoteName, _ manifest: OCIManifest, _ registry: Registry) async throws -> LocalLayerCache? {
+    // Establish a closure that will calculate how much bytes
+    // we'll deduplicate if we re-use the given manifest
+    let target = Swift.Set(manifest.layers)
+
+    let calculateDeduplicatedBytes = { (manifest: OCIManifest) -> UInt64 in
+      target.intersection(manifest.layers).map({ UInt64($0.size) }).reduce(0, +)
+    }
+
+    // Load OCI VM images and their manifests (if present)
+    var candidates: [(name: String, vmDir: VMDirectory, manifest: OCIManifest, deduplicatedBytes: UInt64)] = []
+
+    for (name, vmDir, isSymlink) in try list() {
+      if isSymlink {
+        continue
+      }
+
+      guard let manifestJSON = try? Data(contentsOf: vmDir.manifestURL) else {
+        continue
+      }
+
+      guard let manifest = try? OCIManifest(fromJSON: manifestJSON) else {
+        continue
+      }
+
+      candidates.append((name, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
+    }
+
+    // Previously we haven't stored the OCI VM image manifests, but still fetched the VM image manifest if
+    // what the user was trying to pull was a tagged image, and we already had that image in the OCI VM cache
+    //
+    // Keep supporting this behavior for backwards comaptibility, but only communicate
+    // with the registry if we haven't already retrieved the manifest for that OCI VM image.
+    if name.reference.type == .Tag,
+       let vmDir = try? open(name),
+       let digest = try? digest(name),
+       try !candidates.contains(where: {try $0.manifest.digest() == digest}),
+       let (manifest, _) = try? await registry.pullManifest(reference: digest) {
+      candidates.append((name.description, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
+    }
+
+    // Now, find the best match based on how many bytes we'll deduplicate
+    let choosen = candidates.filter {
+      $0.deduplicatedBytes > 1024 * 1024 * 1024 // save at least 1GB
+    }.max { left, right in
+      return left.deduplicatedBytes < right.deduplicatedBytes
+    }
+
+    return try choosen.flatMap({ choosen in
+      try LocalLayerCache(choosen.name, choosen.deduplicatedBytes, choosen.vmDir.diskURL, choosen.manifest)
+    })
+  }
 }
 
 extension URL {

Tests/TartTests/DiskImageFormatTests.swift

@@ -0,0 +1,63 @@
+import XCTest
+@testable import tart
+
+final class DiskImageFormatTests: XCTestCase {
+  func testASIFFormatSupport() throws {
+    // ASIF should be supported on macOS 26+
+    if #available(macOS 26, *) {
+      XCTAssertTrue(DiskImageFormat.asif.isSupported)
+    } else {
+      XCTAssertFalse(DiskImageFormat.asif.isSupported)
+    }
+  }
+
+  func testFormatFromString() throws {
+    XCTAssertEqual(DiskImageFormat(rawValue: "raw"), .raw)
+    XCTAssertEqual(DiskImageFormat(rawValue: "asif"), .asif)
+    XCTAssertNil(DiskImageFormat(rawValue: "invalid"))
+  }
+
+  func testCaseInsensitivity() throws {
+    XCTAssertEqual(DiskImageFormat(argument: "ASIF"), .asif) // case insensitive
+    XCTAssertEqual(DiskImageFormat(argument: "Raw"), .raw)  // case insensitive
+  }
+
+  func testAllValueStrings() throws {
+    let allValues = DiskImageFormat.allValueStrings
+    XCTAssertTrue(allValues.contains("raw"))
+    XCTAssertTrue(allValues.contains("asif"))
+    XCTAssertEqual(allValues.count, 2)
+  }
+
+  func testVMConfigDiskFormatSerialization() throws {
+    // Test that VMConfig properly serializes and deserializes disk format
+    let config = VMConfig(
+      platform: Linux(),
+      cpuCountMin: 2,
+      memorySizeMin: 1024 * 1024 * 1024,
+      diskFormat: .asif
+    )
+
+    XCTAssertEqual(config.diskFormat, .asif)
+
+    // Test JSON encoding/decoding
+    let encoder = JSONEncoder()
+    let data = try encoder.encode(config)
+
+    let decoder = JSONDecoder()
+    let decodedConfig = try decoder.decode(VMConfig.self, from: data)
+
+    XCTAssertEqual(decodedConfig.diskFormat, .asif)
+  }
+
+  func testVMConfigDefaultDiskFormat() throws {
+    // Test that VMConfig defaults to raw format
+    let config = VMConfig(
+      platform: Linux(),
+      cpuCountMin: 2,
+      memorySizeMin: 1024 * 1024 * 1024
+    )
+
+    XCTAssertEqual(config.diskFormat, .raw)
+  }
+}

Tests/TartTests/LayerizerTests.swift

@@ -36,7 +36,7 @@ final class LayerizerTests: XCTestCase {
     let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
 
     print("pushing disk...")
-    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())
 
     print("pulling disk...")
     try await DiskV1.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
@@ -57,7 +57,7 @@ final class LayerizerTests: XCTestCase {
     let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
 
     print("pushing disk...")
-    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())
 
     print("pulling disk...")
     try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())

Tests/TartTests/Util/RegistryRunner.swift

@@ -31,7 +31,7 @@ class RegistryRunner {
 
   init() async throws {
     // Start container
-    let container = try Self.dockerCmd("run", "-d", "--rm", "-p", "5000", "registry:2")
+    let container = try Self.dockerCmd("run", "-d", "--rm", "-p", "127.0.0.1:0:5000", "registry:2")
       .trimmingCharacters(in: CharacterSet.newlines)
     containerID = container
 
@@ -39,7 +39,7 @@ class RegistryRunner {
     let port = try Self.dockerCmd("inspect", containerID, "--format", "{{(index (index .NetworkSettings.Ports \"5000/tcp\") 0).HostPort}}")
       .trimmingCharacters(in: CharacterSet.newlines)
 
-    registry = try Registry(urlComponents: URLComponents(string: "http://127.0.0.1:\(port)/v2/")!,
+    registry = try Registry(baseURL: URL(string: "http://127.0.0.1:\(port)/v2/")!,
                             namespace: "vm-image")
 
     // Wait for the Docker Registry to start

benchmark/README.md

@@ -2,10 +2,18 @@
 
 Tart comes with a Golang-based benchmarking utility that allows one to easily compare host and guest performance.
 
-Currently, only Flexible I/O tester workloads are supported. To run them, [install Golang](https://go.dev/) and run the following command from this (`benchmark/`) directory:
+Currently, only Flexible I/O tester workloads are supported. To run them, first make sure that [passwordless sudo](https://serverfault.com/questions/160581/how-to-setup-passwordless-sudo-on-linux) is configured.
+
+Then, [install Golang](https://go.dev/). The easiest way is through [Homebrew](https://brew.sh/):
 
 ```shell
-go run cmd/main.go fio
+brew install go
+```
+
+Finally, run the following command from this (`benchmark/`) directory:
+
+```shell
+go run cmd/main.go fio --image ghcr.io/cirruslabs/macos-sequoia-base:latest --prepare 'sudo purge && sync'
 ```
 
 You can also enable the debugging output to diagnose issues:
@@ -16,6 +24,8 @@ go run cmd/main.go fio --debug
 
 ## Results
 
+### Mar 27, 2024
+
 Host:
 
 * Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
@@ -37,3 +47,164 @@ Random writing of 100MB 	Tart    	2.0 GB/s 	493.31 kIOPS
 Random writing of 1000MB	local   	1.7 GB/s 	414.89 kIOPS  
 Random writing of 1000MB	Tart    	1.1 GB/s 	287.4 kIOPS  
 ```
+
+### Dec 2, 2024
+
+Host:
+
+* Hardware: MacBook Pro (Apple M1 Pro, 8 performance and 2 efficiency cores, 32 GB RAM, `MacBookPro18,3`)
+* OS: macOS Sequoia 15.1.1
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)          	Latency (write)         	Latency (sync)      
+Single 4KiB random write process        	local                                             	0 B/s     	19 MB/s    	0 IOPS     	4.81 kIOPS  	0s ± 0s                 	203.418µs ± 155.865µs   	0s ± 0s             
+Single 4KiB random write process        	Tart                                              	0 B/s     	18 MB/s    	0 IOPS     	4.54 kIOPS  	0s ± 0s                 	213.655µs ± 188.822µs   	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	19 MB/s    	0 IOPS     	4.68 kIOPS  	0s ± 0s                 	208.413µs ± 183.45µs    	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	24 MB/s    	0 IOPS     	6.11 kIOPS  	0s ± 0s                 	158.07µs ± 2.294654ms   	0s ± 0s             
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	22 MB/s    	0 IOPS     	5.49 kIOPS  	0s ± 0s                 	173.414µs ± 310.213µs   	0s ± 0s             
+16 parallel 64KiB random write processes	local                                             	0 B/s     	18 GB/s    	0 IOPS     	273.76 kIOPS	0s ± 0s                 	323.423µs ± 604.999µs   	0s ± 0s             
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	16 GB/s    	0 IOPS     	273.48 kIOPS	0s ± 0s                 	335.086µs ± 7.591748ms  	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	18 GB/s    	0 IOPS     	281.49 kIOPS	0s ± 0s                 	326.655µs ± 7.485473ms  	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	17 GB/s    	0 IOPS     	266.79 kIOPS	0s ± 0s                 	340µs ± 7.868384ms      	0s ± 0s             
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	16 GB/s    	0 IOPS     	251.02 kIOPS	0s ± 0s                 	355.077µs ± 8.354218ms  	0s ± 0s             
+Single 1MiB random write process        	local                                             	0 B/s     	1.3 GB/s   	0 IOPS     	1.31 kIOPS  	0s ± 0s                 	751.716µs ± 370.731µs   	0s ± 0s             
+Single 1MiB random write process        	Tart                                              	0 B/s     	1.1 GB/s   	0 IOPS     	1.1 kIOPS   	0s ± 0s                 	885.833µs ± 3.572539ms  	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.1 GB/s   	0 IOPS     	1.08 kIOPS  	0s ± 0s                 	898.427µs ± 3.464261ms  	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	1000 MB/s  	0 IOPS     	976.47 IOPS 	0s ± 0s                 	972.491µs ± 6.87654ms   	0s ± 0s             
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	1.1 GB/s   	0 IOPS     	1.03 kIOPS  	0s ± 0s                 	925.545µs ± 4.261693ms  	0s ± 0s             
+Random reads/writes (4k)                	local                                             	62 MB/s   	62 MB/s    	15.37 kIOPS	15.37 kIOPS 	2.059453ms ± 1.431822ms 	2.098761ms ± 1.445082ms 	0s ± 0s             
+Random reads/writes (4k)                	Tart                                              	38 MB/s   	38 MB/s    	9.6 kIOPS  	9.61 kIOPS  	3.30369ms ± 1.500464ms  	3.350589ms ± 1.512986ms 	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	39 MB/s   	39 MB/s    	9.82 kIOPS 	9.83 kIOPS  	3.228106ms ± 1.367512ms 	3.27626ms ± 1.385964ms  	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	35 MB/s   	35 MB/s    	8.74 kIOPS 	8.76 kIOPS  	3.640772ms ± 15.472355ms	3.661779ms ± 15.264288ms	0s ± 0s             
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	24 MB/s   	24 MB/s    	5.98 kIOPS 	5.99 kIOPS  	5.31188ms ± 4.55205ms   	5.375047ms ± 5.113847ms 	0s ± 0s             
+Random reads/writes (64k)               	local                                             	435 MB/s  	436 MB/s   	6.79 kIOPS 	6.8 kIOPS   	4.955892ms ± 2.066685ms 	4.440414ms ± 1.860036ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart                                              	352 MB/s  	353 MB/s   	5.5 kIOPS  	5.51 kIOPS  	5.946067ms ± 2.041124ms 	5.658948ms ± 1.928372ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	331 MB/s  	332 MB/s   	5.16 kIOPS 	5.17 kIOPS  	6.330765ms ± 1.726782ms 	6.033862ms ± 1.671028ms 	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	428 MB/s  	428 MB/s   	6.68 kIOPS 	6.69 kIOPS  	4.661666ms ± 18.342779ms	4.904961ms ± 18.396772ms	0s ± 0s             
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	297 MB/s  	298 MB/s   	4.64 kIOPS 	4.65 kIOPS  	6.591009ms ± 2.827053ms 	7.166883ms ± 3.001036ms 	0s ± 0s             
+sync test                               	local                                             	0 B/s     	48 MB/s    	0 IOPS     	21.15 kIOPS 	0s ± 0s                 	23.471µs ± 81.868µs     	23.374µs ± 6.255µs  
+sync test                               	Tart                                              	0 B/s     	24 MB/s    	0 IOPS     	10.72 kIOPS 	0s ± 0s                 	24.983µs ± 61.761µs     	67.575µs ± 76.196µs 
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	21 MB/s    	0 IOPS     	9.5 kIOPS   	0s ± 0s                 	26.973µs ± 63.935µs     	77.388µs ± 47.103µs 
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	30 MB/s    	0 IOPS     	13.19 kIOPS 	0s ± 0s                 	11.923µs ± 25.225µs     	62.894µs ± 208.933µs
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	38 MB/s    	0 IOPS     	17.02 kIOPS 	0s ± 0s                 	10.124µs ± 21.868µs     	47.803µs ± 33.706µs
+```
+
+### Dec 4, 2024
+
+Host:
+
+* AWS instance: `mac2.metal` + `gp3` EBS volume
+* Hardware: Mac mini (Apple M1, 4 performance and 4 efficiency cores, 16 GB RAM, `Macmini9,1`)
+* OS: macOS Sequoia 15.0
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 14.6
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)            	Latency (write)           	Latency (sync)         
+Single 4KiB random write process        	local                                             	0 B/s     	4.4 MB/s   	0 IOPS     	1.1 kIOPS   	0s ± 0s                   	702.357µs ± 359.925µs     	0s ± 0s                
+Single 4KiB random write process        	Tart                                              	0 B/s     	2.6 MB/s   	0 IOPS     	656.37 IOPS 	0s ± 0s                   	1.140086ms ± 1.450472ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	2.7 MB/s   	0 IOPS     	677.07 IOPS 	0s ± 0s                   	1.179872ms ± 1.219626ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	3.3 MB/s   	0 IOPS     	832.66 IOPS 	0s ± 0s                   	948.648µs ± 94.141338ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	15 MB/s    	0 IOPS     	3.65 kIOPS  	0s ± 0s                   	260.717µs ± 19.977757ms   	0s ± 0s                
+16 parallel 64KiB random write processes	local                                             	0 B/s     	9.5 GB/s   	0 IOPS     	147.89 kIOPS	0s ± 0s                   	753.289µs ± 8.028974ms    	0s ± 0s                
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	10 GB/s    	0 IOPS     	176.96 kIOPS	0s ± 0s                   	429.83µs ± 33.792264ms    	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	12 GB/s    	0 IOPS     	180.89 kIOPS	0s ± 0s                   	383.524µs ± 17.524971ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	336 MB/s   	0 IOPS     	5.24 kIOPS  	0s ± 0s                   	9.970844ms ± 365.808663ms 	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	9.4 GB/s   	0 IOPS     	147.04 kIOPS	0s ± 0s                   	524.139µs ± 34.100009ms   	0s ± 0s                
+Single 1MiB random write process        	local                                             	0 B/s     	178 MB/s   	0 IOPS     	173.36 IOPS 	0s ± 0s                   	3.835103ms ± 2.917977ms   	0s ± 0s                
+Single 1MiB random write process        	Tart                                              	0 B/s     	140 MB/s   	0 IOPS     	136.48 IOPS 	0s ± 0s                   	4.721178ms ± 7.744965ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	144 MB/s   	0 IOPS     	140.63 IOPS 	0s ± 0s                   	4.443507ms ± 11.572454ms  	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	47 MB/s    	0 IOPS     	45.55 IOPS  	0s ± 0s                   	13.267881ms ± 358.283094ms	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	196 MB/s   	0 IOPS     	191.4 IOPS  	0s ± 0s                   	4.102516ms ± 73.117503ms  	0s ± 0s                
+Random reads/writes (4k)                	local                                             	8.7 MB/s  	8.7 MB/s   	2.16 kIOPS 	2.16 kIOPS  	193.370794ms ± 42.593607ms	222.272016ms ± 56.586971ms	0s ± 0s                
+Random reads/writes (4k)                	Tart                                              	4.1 MB/s  	4.1 MB/s   	1.02 kIOPS 	1.03 kIOPS  	31.038867ms ± 13.508668ms 	31.184305ms ± 14.032766ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	4.2 MB/s  	4.2 MB/s   	1.04 kIOPS 	1.05 kIOPS  	30.368422ms ± 13.505627ms 	30.595412ms ± 13.840944ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	2.2 MB/s  	2.2 MB/s   	545.33 IOPS	548.86 IOPS 	59.31316ms ± 716.351086ms 	57.647852ms ± 711.503882ms	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	6.0 MB/s  	6.0 MB/s   	1.5 kIOPS  	1.5 kIOPS   	21.244222ms ± 47.808399ms 	21.39459ms ± 44.716307ms  	0s ± 0s                
+Random reads/writes (64k)               	local                                             	121 MB/s  	121 MB/s   	1.89 kIOPS 	1.89 kIOPS  	61.894699ms ± 21.353345ms 	73.176462ms ± 13.02948ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart                                              	72 MB/s   	72 MB/s    	1.12 kIOPS 	1.12 kIOPS  	27.842263ms ± 15.320781ms 	29.161858ms ± 15.765314ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	71 MB/s   	72 MB/s    	1.11 kIOPS 	1.11 kIOPS  	28.009493ms ± 16.333136ms 	29.285868ms ± 16.540589ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	28 MB/s   	28 MB/s    	441.85 IOPS	444.81 IOPS 	71.726725ms ± 633.215756ms	72.597238ms ± 630.969305ms	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	81 MB/s   	81 MB/s    	1.26 kIOPS 	1.26 kIOPS  	24.872043ms ± 36.980111ms 	25.568559ms ± 37.027145ms 	0s ± 0s                
+sync test                               	local                                             	0 B/s     	1.9 MB/s   	0 IOPS     	868.08 IOPS 	0s ± 0s                   	92.08µs ± 233.598µs       	1.059033ms ± 98.751µs  
+sync test                               	Tart                                              	0 B/s     	1.5 MB/s   	0 IOPS     	649.42 IOPS 	0s ± 0s                   	146.737µs ± 434.261µs     	1.391898ms ± 699.148µs 
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.3 MB/s   	0 IOPS     	568.82 IOPS 	0s ± 0s                   	158.736µs ± 504.002µs     	1.59798ms ± 14.161331ms
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	13 MB/s    	0 IOPS     	5.77 kIOPS  	0s ± 0s                   	26.596µs ± 832.169µs      	145.785µs ± 2.864048ms 
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	19 MB/s    	0 IOPS     	8.37 kIOPS  	0s ± 0s                   	20.135µs ± 108.817µs      	98.274µs ± 239.631µs   
+```
+
+Host:
+
+* AWS instance: `mac2.metal` + `gp3` EBS volume
+* Hardware: Mac mini (Apple M1, 4 performance and 4 efficiency cores, 16 GB RAM, `Macmini9,1`)
+* OS: macOS Sequoia 15.0
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sequoia 15.1
+
+```
+Name                                    	Executor                                          	B/W (read)	B/W (write)	I/O (read) 	I/O (write) 	Latency (read)            	Latency (write)           	Latency (sync)         
+Single 4KiB random write process        	local                                             	0 B/s     	4.8 MB/s   	0 IOPS     	1.19 kIOPS  	0s ± 0s                   	690.818µs ± 326.595µs     	0s ± 0s                
+Single 4KiB random write process        	Tart                                              	0 B/s     	2.8 MB/s   	0 IOPS     	700.94 IOPS 	0s ± 0s                   	1.090362ms ± 918.444µs    	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	3.0 MB/s   	0 IOPS     	746.23 IOPS 	0s ± 0s                   	1.028192ms ± 974.533µs    	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	4.2 MB/s   	0 IOPS     	1.04 kIOPS  	0s ± 0s                   	916.36µs ± 105.318323ms   	0s ± 0s                
+Single 4KiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	14 MB/s    	0 IOPS     	3.57 kIOPS  	0s ± 0s                   	269.796µs ± 22.419599ms   	0s ± 0s                
+16 parallel 64KiB random write processes	local                                             	0 B/s     	9.5 GB/s   	0 IOPS     	148.74 kIOPS	0s ± 0s                   	753.46µs ± 8.06509ms      	0s ± 0s                
+16 parallel 64KiB random write processes	Tart                                              	0 B/s     	5.2 GB/s   	0 IOPS     	81.46 kIOPS 	0s ± 0s                   	778.624µs ± 11.705178ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none")               	0 B/s     	5.3 GB/s   	0 IOPS     	83.47 kIOPS 	0s ± 0s                   	865.448µs ± 38.369176ms   	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="caching=cached")          	0 B/s     	116 MB/s   	0 IOPS     	1.8 kIOPS   	0s ± 0s                   	37.601112ms ± 727.319309ms	0s ± 0s                
+16 parallel 64KiB random write processes	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	5.3 GB/s   	0 IOPS     	83.19 kIOPS 	0s ± 0s                   	900.751µs ± 51.223205ms   	0s ± 0s                
+Single 1MiB random write process        	local                                             	0 B/s     	177 MB/s   	0 IOPS     	173.27 IOPS 	0s ± 0s                   	3.833194ms ± 2.873871ms   	0s ± 0s                
+Single 1MiB random write process        	Tart                                              	0 B/s     	151 MB/s   	0 IOPS     	147.44 IOPS 	0s ± 0s                   	4.925853ms ± 7.793808ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none")               	0 B/s     	151 MB/s   	0 IOPS     	147.87 IOPS 	0s ± 0s                   	4.884797ms ± 7.563512ms   	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="caching=cached")          	0 B/s     	72 MB/s    	0 IOPS     	69.9 IOPS   	0s ± 0s                   	8.909771ms ± 214.311644ms 	0s ± 0s                
+Single 1MiB random write process        	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	159 MB/s   	0 IOPS     	155.69 IOPS 	0s ± 0s                   	4.863448ms ± 88.965211ms  	0s ± 0s                
+Random reads/writes (4k)                	local                                             	8.7 MB/s  	8.7 MB/s   	2.16 kIOPS 	2.16 kIOPS  	193.353233ms ± 42.728494ms	222.325905ms ± 56.901372ms	0s ± 0s                
+Random reads/writes (4k)                	Tart                                              	3.5 MB/s  	3.5 MB/s   	862.89 IOPS	865.54 IOPS 	36.893229ms ± 12.644216ms 	37.143334ms ± 12.772017ms 	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none")               	3.6 MB/s  	3.6 MB/s   	907.4 IOPS 	911.55 IOPS 	35.048969ms ± 10.559354ms 	35.3046ms ± 10.67824ms    	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="caching=cached")          	2.7 MB/s  	2.8 MB/s   	684.11 IOPS	688.05 IOPS 	48.815322ms ± 687.806727ms	44.395556ms ± 635.532064ms	0s ± 0s                
+Random reads/writes (4k)                	Tart (--root-disk-opts="sync=none,caching=cached")	7.0 MB/s  	7.0 MB/s   	1.74 kIOPS 	1.74 kIOPS  	18.00448ms ± 93.784447ms  	18.617037ms ± 107.423001ms	0s ± 0s                
+Random reads/writes (64k)               	local                                             	121 MB/s  	121 MB/s   	1.89 kIOPS 	1.89 kIOPS  	61.983727ms ± 21.324782ms 	73.228597ms ± 12.730945ms 	0s ± 0s                
+Random reads/writes (64k)               	Tart                                              	75 MB/s   	75 MB/s    	1.17 kIOPS 	1.17 kIOPS  	26.830538ms ± 7.643051ms  	27.709602ms ± 7.830965ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none")               	76 MB/s   	77 MB/s    	1.19 kIOPS 	1.19 kIOPS  	26.255337ms ± 7.302592ms  	27.256805ms ± 7.388266ms  	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="caching=cached")          	32 MB/s   	33 MB/s    	505.26 IOPS	508.66 IOPS 	65.170269ms ± 747.794957ms	61.062186ms ± 695.614904ms	0s ± 0s                
+Random reads/writes (64k)               	Tart (--root-disk-opts="sync=none,caching=cached")	79 MB/s   	79 MB/s    	1.23 kIOPS 	1.23 kIOPS  	25.861503ms ± 171.669777ms	25.992302ms ± 164.647788ms	0s ± 0s                
+sync test                               	local                                             	0 B/s     	1.9 MB/s   	0 IOPS     	865.16 IOPS 	0s ± 0s                   	100.95µs ± 268.722µs      	1.054051ms ± 365.377µs 
+sync test                               	Tart                                              	0 B/s     	1.6 MB/s   	0 IOPS     	704.13 IOPS 	0s ± 0s                   	133.886µs ± 390.263µs     	1.285085ms ± 575.27µs  
+sync test                               	Tart (--root-disk-opts="sync=none")               	0 B/s     	1.6 MB/s   	0 IOPS     	728.26 IOPS 	0s ± 0s                   	129.246µs ± 472.724µs     	1.242713ms ± 1.281286ms
+sync test                               	Tart (--root-disk-opts="caching=cached")          	0 B/s     	35 MB/s    	0 IOPS     	15.67 kIOPS 	0s ± 0s                   	11.319µs ± 24.771µs       	51.731µs ± 42.208µs    
+sync test                               	Tart (--root-disk-opts="sync=none,caching=cached")	0 B/s     	17 MB/s    	0 IOPS     	7.39 kIOPS  	0s ± 0s                   	21.23µs ± 81.749µs        	113.239µs ± 191.266µs  
+```
+
+### March 23, 2025
+
+Host:
+
+* Hardware: Mac mini (Apple M2 Pro, 8 performance and 4 efficiency cores, 32 GB RAM, `Mac14,12`)
+* OS: macOS Sequoia 15.3.2
+* Xcode: 16.2
+
+Guest:
+
+* Hardware: [Virtualization.Framework](https://developer.apple.com/documentation/virtualization)
+* OS: macOS Sonoma 15.3.2 
+* Xcode: 16.2
+
+```
+Name                    	Executor                                          	Time 
+XcodeBenchmark (d869315)	local                                             	2m19s
+XcodeBenchmark (d869315)	Tart                                              	3m59s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none")               	3m48s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="caching=cached")          	3m35s
+XcodeBenchmark (d869315)	Tart (--root-disk-opts="sync=none,caching=cached")	3m14s
+```

benchmark/go.mod

@@ -1,21 +1,24 @@
 module github.com/cirruslabs/tart/benchmark
 
 go 1.22.1
+toolchain go1.24.1
 
 require (
 	github.com/avast/retry-go/v4 v4.5.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/google/uuid v1.6.0
 	github.com/gosuri/uitable v0.0.4
+	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.21.0
+	golang.org/x/crypto v0.35.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -23,7 +26,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

benchmark/go.sum

@@ -7,6 +7,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
@@ -26,26 +28,31 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
+golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

benchmark/internal/command/fio/benchmark.go

@@ -7,23 +7,63 @@ type Benchmark struct {
 
 var benchmarks = []Benchmark{
 	{
-		Name: "Random writing of 1MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 1MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Ars Technica's "Single 4KiB random write process" test[1]
+		// with JSON output and created file cleanup
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "Single 4KiB random write process",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=4k --size=4g --numjobs=1 --iodepth=1 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1",
 	},
 	{
-		Name: "Random writing of 10MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 10MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Ars Technica's "16 parallel 64KiB random write processes" test[1]
+		// with JSON outpu, created file cleanup and group reporting (for
+		// easier analysis)
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "16 parallel 64KiB random write processes",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=64k --size=256m --numjobs=16 --iodepth=16 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1 --group_reporting",
 	},
 	{
-		Name: "Random writing of 100MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 100MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Ars Technica's "16 parallel 64KiB random write processes" test[1]
+		// with JSON output, created file cleanup and reduced file I/O size
+		// from 16 to 10 GB to avoid "No space left on device".
+		//
+		// [1]: https://arstechnica.com/gadgets/2020/02/how-fast-are-your-disks-find-out-the-open-source-way-with-fio/
+		Name: "Single 1MiB random write process",
+		Command: "fio --name=benchmark --ioengine=posixaio --rw=randwrite --bs=1m --size=10g --numjobs=1 --iodepth=1 --runtime=60 --time_based --end_fsync=1" +
+			" --output-format json --unlink 1",
 	},
 	{
-		Name: "Random writing of 1000MB",
-		Command: "fio --rw randwrite --runtime 30 --time_based --unlink 1 --output-format json " +
-			"--size 1000MB --name unnamed --numjobs 1 --iodepth 1 --end_fsync 1",
+		// Oracle's "Test random read/writes" (in IOPS Performance Tests[1]) category
+		// with JSON output, created file cleanup, without ETA newline, without custom
+		// file path, with file I/O size reduced from 500GB to 2GB to prevent
+		// "No space left on device" and with posixaio instead of libaio.
+		//
+		// [1]: https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm#FIO_Commands
+		Name: "Random reads/writes (4k)",
+		Command: "fio --name=benchmark --size=2GB --direct=1 --rw=randrw --bs=4k --ioengine=posixaio --iodepth=256 --runtime=120 --numjobs=4 --time_based --group_reporting" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// Oracle's "Test random read/writes" (in Throughput Performance Tests[1]) category
+		// with JSON output, created file cleanup, without ETA newline, without custom
+		// file path, with file I/O size reduced from 500GB to 2GB to prevent
+		// "No space left on device" and with posixaio instead of libaio.
+		//
+		// [1]: https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm#Throughput_Performance_Tests
+		Name: "Random reads/writes (64k)",
+		Command: "fio --name=benchmark --size=2GB --direct=1 --rw=randrw --bs=64k --ioengine=posixaio --iodepth=64 --runtime=120 --numjobs=4 --time_based --group_reporting" +
+			" --output-format json --unlink 1",
+	},
+	{
+		// RedHat's "How can I test to see if my environment is fast enough for etcd"[1]
+		// with custom name
+		//
+		// [1]: https://access.redhat.com/solutions/5726511
+		Name: "sync test",
+		Command: "mkdir -p test-data && fio --name=benchmark --rw=write --ioengine=sync --fdatasync=1 --directory=test-data --size=22m --bs=2300" +
+			" --output-format json --unlink 1",
 	},
 }

benchmark/internal/command/fio/fio.go

@@ -1,20 +1,21 @@
 package fio
 
 import (
-	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"github.com/cirruslabs/tart/benchmark/internal/executor"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
-	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
 	"github.com/dustin/go-humanize"
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
 )
 
 var debug bool
+var image string
+var prepare string
 
 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
@@ -24,6 +25,8 @@ func NewCommand() *cobra.Command {
 	}
 
 	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-base:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
 
 	return cmd
 }
@@ -41,28 +44,50 @@ func run(cmd *cobra.Command, args []string) error {
 		_ = logger.Sync()
 	}()
 
-	executors, err := initializeExecutors(cmd.Context(), logger)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		errs := []error{err}
-
-		for _, executor := range executors {
-			if err := executor.Close(); err != nil {
-				errs = append(errs, fmt.Errorf("failed to close executor %s: %w", executor.Name(), err))
-			}
-		}
-
-		err = errors.Join(errs...)
-	}()
-
 	table := uitable.New()
-	table.AddRow("Name", "Executor", "Bandwidth", "I/O operations")
+	table.AddRow("Name", "Executor", "B/W (read)", "B/W (write)", "I/O (read)", "I/O (write)",
+		"Latency (read)", "Latency (write)", "Latency (sync)")
 
 	for _, benchmark := range benchmarks {
-		for _, executor := range executors {
-			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, executor.Name())
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("installing Flexible I/O tester (fio) on executor %s",
+				executorInitializer.Name)
+
+			if _, err := executor.Run(cmd.Context(), "brew install fio"); err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
 
 			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
 			if err != nil {
@@ -82,12 +107,28 @@ func run(cmd *cobra.Command, args []string) error {
 
 			job := fioResult.Jobs[0]
 
+			readBandwidth := humanize.Bytes(uint64(job.Read.BW)*humanize.KByte) + "/s"
+			readIOPS := humanize.SIWithDigits(job.Read.IOPS, 2, "IOPS")
+
+			logger.Sugar().Infof("read bandwidth: %s, read IOPS: %s, read latency: %s",
+				readBandwidth, readIOPS, job.Read.LatencyNS.String())
+
 			writeBandwidth := humanize.Bytes(uint64(job.Write.BW)*humanize.KByte) + "/s"
 			writeIOPS := humanize.SIWithDigits(job.Write.IOPS, 2, "IOPS")
 
-			logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s\n", writeBandwidth, writeIOPS)
+			logger.Sugar().Infof("write bandwidth: %s, write IOPS: %s, write latency: %s",
+				writeBandwidth, writeIOPS, job.Write.LatencyNS.String())
 
-			table.AddRow(benchmark.Name, executor.Name(), writeBandwidth, writeIOPS)
+			logger.Sugar().Infof("sync latency: %s", job.Sync.LatencyNS.String())
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, readBandwidth, writeBandwidth,
+				readIOPS, writeIOPS, job.Read.LatencyNS.String(), job.Write.LatencyNS.String(),
+				job.Sync.LatencyNS.String())
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
 		}
 	}
 
@@ -95,37 +136,3 @@ func run(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
-
-func initializeExecutors(ctx context.Context, logger *zap.Logger) ([]executor.Executor, error) {
-	var result []executor.Executor
-
-	logger.Info("initializing local executor")
-
-	local, err := local.New(logger)
-	if err != nil {
-		return nil, err
-	}
-	result = append(result, local)
-
-	logger.Info("local executor initialized")
-
-	logger.Info("initializing Tart executor")
-
-	tart, err := tart.New(ctx, logger)
-	if err != nil {
-		return nil, err
-	}
-	result = append(result, tart)
-
-	logger.Info("Tart executor initialized")
-
-	for _, executor := range result {
-		logger.Sugar().Infof("installing Flexible I/O tester (fio) on %s executor", executor.Name())
-
-		if _, err := executor.Run(ctx, "brew install fio"); err != nil {
-			return nil, err
-		}
-	}
-
-	return result, nil
-}

benchmark/internal/command/fio/json.go

@@ -1,15 +1,35 @@
 package fio
 
+import (
+	"fmt"
+	"time"
+)
+
 type Result struct {
 	Jobs []Job `json:"jobs"`
 }
 
 type Job struct {
 	Name  string `json:"jobname"`
-	Write Write  `json:"write"`
+	Read  Stats  `json:"read"`
+	Write Stats  `json:"write"`
+	Sync  Stats  `json:"sync"`
 }
 
-type Write struct {
-	BW   float64 `json:"bw"`
-	IOPS float64 `json:"iops"`
+type Stats struct {
+	BW        float64 `json:"bw"`
+	IOPS      float64 `json:"iops"`
+	LatencyNS Latency `json:"lat_ns"`
+}
+
+type Latency struct {
+	Mean   float64 `json:"mean"`
+	Stddev float64 `json:"stddev"`
+}
+
+func (latency Latency) String() string {
+	meanDuration := time.Duration(latency.Mean) * time.Nanosecond
+	stddevDuration := time.Duration(latency.Stddev) * time.Nanosecond
+
+	return fmt.Sprintf("%v ± %v", meanDuration, stddevDuration)
 }

benchmark/internal/command/root.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"github.com/cirruslabs/tart/benchmark/internal/command/fio"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
 	"github.com/spf13/cobra"
 )
 
@@ -14,6 +15,7 @@ func NewCommand() *cobra.Command {
 
 	cmd.AddCommand(
 		fio.NewCommand(),
+		xcode.NewCommand(),
 	)
 
 	return cmd

benchmark/internal/command/xcode/benchmarks.go

@@ -0,0 +1,13 @@
+package xcode
+
+type Benchmark struct {
+	Name    string
+	Command string
+}
+
+var benchmarks = []Benchmark{
+	{
+		Name:    "XcodeBenchmark (d869315)",
+		Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && xcrun simctl list && sh benchmark.sh",
+	},
+}

benchmark/internal/command/xcode/output.go

@@ -0,0 +1,49 @@
+package xcode
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+type Output struct {
+	Started time.Time
+	Ended   time.Time
+}
+
+func ParseOutput(s string) (*Output, error) {
+	// Ensure that the build has succeeded
+	matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err)
+	}
+	if !matched {
+		return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " +
+			"not found on a separate line, make sure you have Xcode installed")
+	}
+
+	re := regexp.MustCompile("Started\\s+(?P<started>.*)\\n.*Ended\\s+(?P<ended>.*)\\n")
+
+	matches := re.FindStringSubmatch(s)
+
+	if len(matches) != re.NumSubexp()+1 {
+		return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times")
+	}
+
+	startedRaw := matches[re.SubexpIndex("started")]
+	started, err := time.Parse(time.TimeOnly, startedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw)
+	}
+
+	endedRaw := matches[re.SubexpIndex("ended")]
+	ended, err := time.Parse(time.TimeOnly, endedRaw)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw)
+	}
+
+	return &Output{
+		Started: started,
+		Ended:   ended,
+	}, nil
+}

benchmark/internal/command/xcode/output_test.go

@@ -0,0 +1,37 @@
+package xcode_test
+
+import (
+	"fmt"
+	"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestParseOutput(t *testing.T) {
+	result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec]
+
+System Version: 14.6
+Xcode 15.4
+Hardware Overview
+      Model Name: Apple Virtual Machine 1
+      Model Identifier: VirtualMac2,1
+      Total Number of Cores: 4
+      Memory: 8 GB
+
+✅ XcodeBenchmark has completed
+1️⃣  Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include:
+	- Build Time (See ** BUILD SUCCEEDED ** [XYZ sec])
+	- System Version
+	- Xcode Version
+	- Hardware Overview
+	- Started 13:46:20
+	- Ended   13:50:02
+	- Date Thu Jan 16 13:50:02 UTC 2025
+
+2️⃣  Share your results at https://github.com/devMEremenko/XcodeBenchmark
+`)
+	require.NoError(t, err)
+	fmt.Println(result)
+	require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started))
+}

benchmark/internal/command/xcode/xcode.go

@@ -0,0 +1,108 @@
+package xcode
+
+import (
+	"fmt"
+	executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor"
+	"github.com/gosuri/uitable"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
+	"os"
+	"os/exec"
+)
+
+var debug bool
+var image string
+var prepare string
+
+func NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "xcode",
+		Short: "run XCode benchmarks",
+		RunE:  run,
+	}
+
+	cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
+	cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sequoia-xcode:latest", "image to use for testing")
+	cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark")
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) error {
+	config := zap.NewProductionConfig()
+	if debug {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	logger, err := config.Build()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = logger.Sync()
+	}()
+
+	table := uitable.New()
+	table.AddRow("Name", "Executor", "Time")
+
+	for _, benchmark := range benchmarks {
+		for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) {
+			if prepare != "" {
+				shell := "/bin/sh"
+
+				if shellFromEnv, ok := os.LookupEnv("SHELL"); ok {
+					shell = shellFromEnv
+				}
+
+				logger.Sugar().Infof("running prepare command %q using shell %q",
+					prepare, shell)
+
+				cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare)
+
+				loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
+
+				cmd.Stdout = loggerWriter
+				cmd.Stderr = loggerWriter
+
+				if err := cmd.Run(); err != nil {
+					return fmt.Errorf("failed to run prepare command %q: %v", prepare, err)
+				}
+			}
+
+			logger.Sugar().Infof("initializing executor %s", executorInitializer.Name)
+
+			executor, err := executorInitializer.Fn()
+			if err != nil {
+				return err
+			}
+
+			logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name,
+				executorInitializer.Name)
+
+			stdout, err := executor.Run(cmd.Context(), benchmark.Command)
+			if err != nil {
+				return err
+			}
+
+			output, err := ParseOutput(string(stdout))
+			if err != nil {
+				return err
+			}
+
+			duration := output.Ended.Sub(output.Started)
+
+			logger.Sugar().Infof("Xcode benchmark duration: %s", duration)
+
+			table.AddRow(benchmark.Name, executorInitializer.Name, duration)
+
+			if err := executor.Close(); err != nil {
+				return fmt.Errorf("failed to close executor %s: %w",
+					executorInitializer.Name, err)
+			}
+		}
+	}
+
+	fmt.Println(table.String())
+
+	return nil
+}

benchmark/internal/executor/executor.go

@@ -5,7 +5,6 @@ import (
 )
 
 type Executor interface {
-	Name() string
 	Run(ctx context.Context, command string) ([]byte, error)
 	Close() error
 }

benchmark/internal/executor/initializer.go

@@ -0,0 +1,57 @@
+package executor
+
+import (
+	"context"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/local"
+	"github.com/cirruslabs/tart/benchmark/internal/executor/tart"
+	"go.uber.org/zap"
+)
+
+type Initializer struct {
+	Name string
+	Fn   func() (Executor, error)
+}
+
+func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer {
+	return []Initializer{
+		{
+			Name: "local",
+			Fn: func() (Executor, error) {
+				return local.New(logger)
+			},
+		},
+		{
+			Name: "Tart",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, nil, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"caching=cached",
+				}, logger)
+			},
+		},
+		{
+			Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")",
+			Fn: func() (Executor, error) {
+				return tart.New(ctx, image, []string{
+					"--root-disk-opts",
+					"sync=none,caching=cached",
+				}, logger)
+			},
+		},
+	}
+}

benchmark/internal/executor/tart/tart.go

@@ -7,15 +7,18 @@ import (
 	"fmt"
 	"github.com/avast/retry-go/v4"
 	"github.com/google/uuid"
+	"github.com/shirou/gopsutil/mem"
 	"go.uber.org/zap"
+	"go.uber.org/zap/zapio"
 	"golang.org/x/crypto/ssh"
+	"io"
 	"net"
+	"runtime"
+	"strconv"
 	"strings"
 	"time"
 )
 
-const baseImage = "ghcr.io/cirruslabs/macos-sonoma-base:latest"
-
 type Tart struct {
 	vmRunCancel context.CancelFunc
 	vmName      string
@@ -23,17 +26,34 @@ type Tart struct {
 	logger      *zap.Logger
 }
 
-func New(ctx context.Context, logger *zap.Logger) (*Tart, error) {
+func New(ctx context.Context, image string, runArgsExtra []string, logger *zap.Logger) (*Tart, error) {
 	tart := &Tart{
 		vmName: fmt.Sprintf("tart-benchmark-%s", uuid.NewString()),
 		logger: logger,
 	}
 
-	if err := Cmd(ctx, tart.logger, "pull", baseImage); err != nil {
+	if err := Cmd(ctx, tart.logger, "pull", image); err != nil {
 		return nil, err
 	}
 
-	if err := Cmd(ctx, tart.logger, "clone", baseImage, tart.vmName); err != nil {
+	if err := Cmd(ctx, tart.logger, "clone", image, tart.vmName); err != nil {
+		return nil, err
+	}
+
+	vmStat, err := mem.VirtualMemory()
+	if err != nil {
+		return nil, err
+	}
+
+	cpus := strconv.Itoa(runtime.NumCPU())
+	memory := strconv.FormatUint(vmStat.Total/1024/1024, 10)
+	logger.Info("Setting resources", zap.String("cpus", cpus), zap.String("memory", memory))
+	setResourcesArguments := []string{
+		"set", tart.vmName,
+		"--cpu", cpus,
+		"--memory", memory,
+	}
+	if err := Cmd(ctx, tart.logger, setResourcesArguments...); err != nil {
 		return nil, err
 	}
 
@@ -41,7 +61,11 @@ func New(ctx context.Context, logger *zap.Logger) (*Tart, error) {
 	tart.vmRunCancel = vmRunCancel
 
 	go func() {
-		_ = Cmd(vmRunCtx, tart.logger, "run", "--no-graphics", tart.vmName)
+		runArgs := []string{"run", "--no-graphics", tart.vmName}
+
+		runArgs = append(runArgs, runArgsExtra...)
+
+		_ = Cmd(vmRunCtx, tart.logger, runArgs...)
 	}()
 
 	ip, err := CmdWithOutput(ctx, tart.logger, "ip", "--wait", "60", tart.vmName)
@@ -105,10 +129,12 @@ func (tart *Tart) Run(ctx context.Context, command string) ([]byte, error) {
 	}()
 	defer monitorCancel()
 
+	loggerWriter := &zapio.Writer{Log: tart.logger, Level: zap.DebugLevel}
 	stdoutBuf := &bytes.Buffer{}
 
 	sshSession.Stdin = bytes.NewBufferString(command)
-	sshSession.Stdout = stdoutBuf
+	sshSession.Stdout = io.MultiWriter(stdoutBuf, loggerWriter)
+	sshSession.Stderr = loggerWriter
 
 	if err := sshSession.Shell(); err != nil {
 		return nil, err

benchmark/internal/executor/tart/tart_test.go

@@ -11,7 +11,7 @@ import (
 func TestTart(t *testing.T) {
 	ctx := context.Background()
 
-	tart, err := tart.New(ctx, zap.NewNop())
+	tart, err := tart.New(ctx, "ghcr.io/cirruslabs/macos-sonoma-base:latest", nil, zap.NewNop())
 	require.NoError(t, err)
 
 	output, err := tart.Run(ctx, "echo \"this is a test\"")

docs/.markdownlint.yml

@@ -9,4 +9,6 @@
 "MD033": false # Inline HTML
 "MD041": false # First line in file should be a top level heading
 "MD045": false # OK not to have a description for an image
-"MD046": false #  Code block style [Expected: fenced; Actual: indented]
+"MD046": false #  Code block style [Expected: fenced; Actual: indented]
+"MD059": false # It's OK to have "here" links
+"MD051": false # MkDocs generates "#-no-pki" anchors, but markdownlint expects "#--no-pki" anchors

docs/blog/posts/2023-02-11-changing-tart-license.md

@@ -60,9 +60,9 @@ device without a physical display connected. For example, a Mac Mini with a HDMI
 but a Mac Mini on a desk with a connected physical display is considered a personal computer. **Usage on personal computers
 and before reaching the 100 CPU cores limit is royalty-free and does not have the viral properties of AGPL.**
 
-When an organization surpasses the 100 CPU cores limit, they will be required to obtain a [Gold Tier License](/licensing#license-tiers),
-which costs \$1000 per month. Upon reaching a limit of 500 CPU cores, a [Platinum Tier License](/licensing#license-tiers)
-(\$3000 per month) will be required, and for organizations that exceed 3000 CPU cores, a custom [Diamond Tier License](/licensing#license-tiers)
+When an organization surpasses the 100 CPU cores limit, they will be required to obtain a [Gold Tier License](../../licensing.md#license-tiers),
+which costs \$1000 per month. Upon reaching a limit of 500 CPU cores, a [Platinum Tier License](../../licensing.md#license-tiers)
+(\$3000 per month) will be required, and for organizations that exceed 3000 CPU cores, a custom [Diamond Tier License](../../licensing.md#license-tiers)
 (\$1 per core per month) will be necessary. **All paid license tiers will include priority feature development and SLAs on support with urgent issues.**
 
 ## Have we considered alternatives?

docs/blog/posts/2023-04-25-orchard-ga.md

@@ -89,6 +89,6 @@ orchard dev
 This will launch a development cluster with a single worker on your machine. Refer to [Orchard documentation](https://github.com/cirruslabs/orchard#creating-virtual-machines)
 on how to create your first virtual machine and access it.
 
-In a [separate blog post](/blog/2023/04/28/ssh-over-grpc-or-how-orchard-simplifies-accessing-vms-in-private-networks/)
+In a [separate blog post](2023-04-28-orchard-ssh-over-grpc.md)
 we’ll cover how Orchard implements seamless SSH access over a gRPC connection. Stay tuned and please don’t hesitate to
 [reach out](https://github.com/cirruslabs/orchard/discussions/landing)! 

docs/blog/posts/2023-04-28-orchard-ssh-over-grpc.md

@@ -64,7 +64,7 @@ We’ve also initially considered using [Yamux](https://github.com/hashicorp/yam
 
 First of all, we’ve made the new port-forwarding functionality available for integrations via the Orchard’s REST API:
 
-![OpenAPI documentation for Orchard's port-forwarding endpoint](/assets/images/orchard-port-forwarding-api.png)
+![OpenAPI documentation for Orchard's port-forwarding endpoint](../../assets/images/orchard-port-forwarding-api.png)
 
 All you need is to use a WebSocket client when accessing this endpoint to make it work.
 

docs/blog/posts/2023-09-20-tart-2.0.0.md

@@ -43,7 +43,7 @@ allocate time to continue improving Tart which brings us to the section below.
 In the last 7 months we've had 12 feature releases that brought a lot of features requested by the community. Here are just
 a few of them to highlight:
 
--[Custom GitLab Runner Executor](/integrations/gitlab-runner/).
+-[Custom GitLab Runner Executor](../../integrations/gitlab-runner.md).
 -[Cluster Management via Orchard](2023-04-25-orchard-ga.md).
 -Numerous compatibility improvements for all kinds of OCI-registries.
 -Sonoma Support (see details [below](#macos-sonoma-updates)).

docs/blog/posts/2023-10-06-tart-on-aws.md

@@ -17,7 +17,7 @@ with preconfigured Tart installation that is optimized to work within AWS infras
 EC2 Mac Instances is a gem of engineering powered by AWS Nitro devices. Just imagine there is a physical Mac Mini with
 a plugged in Nitro device that can push the physical power button!
 
-![EC2 M2 Pro](/blog/images/ec2-mac2-m2pro.png)
+![EC2 M2 Pro](../images/ec2-mac2-m2pro.png)
 
 This clever synergy between Apple Hardware and Nitro System allows seamless integration with VPC networking and booting macOS from an EBS volume.
 

docs/blog/posts/2023-11-03-cirrus-runners-dashboard.md

@@ -34,7 +34,7 @@ than recently announced Apple Silicon GitHub-manged runners that cost $0.16 per
 Now lets take a look at the new Cirrus Runners dashboard of a real customers that run their workflows on Cirrus Runners
 and **practically pushing the price performance pretty close to the theoretical minimum**.
 
-![Cirrus Runners Dashboard](/blog/images/runners-price-performance-2.png)
+![Cirrus Runners Dashboard](../images/runners-price-performance-2.png)
 
 As you can see above Cirrus Runners Dashboard focuses on 4 core metrics:
 
@@ -50,7 +50,7 @@ we can see that the downside of such great price performance is that jobs are wa
 Here is another example of Cirrus Runners Dashboard for a different customer that has a slightly higher price performance of $0.017 per minute
 but at the same time doesn't experience queue time at all. **Note that $0.017 is still 10 times cheaper than GitHub-managed Apple Silicon runners**.
 
-![Cirrus Runners Dashboard](/blog/images/runners-price-performance-3.png)
+![Cirrus Runners Dashboard](../images/runners-price-performance-3.png)
 
 ## Conclusion