Revert "Do not magically set --no-graphics when --vnc is passed (#763)

* Revert "Do not magically set `--no-graphics` when `--vnc` is passed (#732)"

This reverts commit a48f4d4ec9.

* Mark `--graphics` as private

.ci/create-pkg.sh

@@ -5,8 +5,10 @@ set -e
 export VERSION="${CIRRUS_TAG:-0}"
 
 mkdir -p .ci/pkg/
-cp .build/arm64-apple-macosx/debug/tart .ci/pkg/tart
+cp .build/arm64-apple-macosx/release/tart .ci/pkg/tart
 cp Resources/embedded.provisionprofile .ci/pkg/embedded.provisionprofile
+cp Resources/AppIcon.png .ci/pkg/AppIcon.png
+cp Resources/Info.plist .ci/pkg/Info.plist
 pkgbuild --root .ci/pkg/ --identifier com.github.cirruslabs.tart --version $VERSION \
   --scripts .ci/pkg/scripts \
   --install-location "/Library/Application Support/Tart" \

.ci/pkg/scripts/postinstall

@@ -3,9 +3,11 @@
 set -e
 
 # fix structure
-mkdir -p "$2/tart.app/Contents/MacOS"
+mkdir -p "$2/tart.app/Contents/MacOS" "$2/tart.app/Resources"
 mv "$2/tart" "$2/tart.app/Contents/MacOS/tart"
 mv "$2/embedded.provisionprofile" "$2/tart.app/Contents/embedded.provisionprofile"
+mv "$2/AppIcon.png" "$2/tart.app/Resources/AppIcon.png"
+mv "$2/Info.plist" "$2/tart.app/Contents/Info.plist"
 
 echo "#!/bin/sh" > /usr/local/bin/tart
 echo "exec '$2/tart.app/Contents/MacOS/tart' \"\$@\"" >> /usr/local/bin/tart

.cirrus.yml

@@ -1,11 +1,16 @@
 use_compute_credits: true
 
+env: 
+  XCODE_TAG: 15.2
+
 task:
-  name: Test on Ventura
+  name: Test on Sonoma
   alias: test
   persistent_worker:
     labels:
       name: dev-mini
+    resources:
+      tart-vms: 1
   test_script:
     - swift test
   integration_test_script:
@@ -24,11 +29,19 @@ task:
     path: "integration-tests/pytest-junit.xml"
     format: junit
 
+task:
+  name: Markdown Lint
+  only_if: $CIRRUS_BRANCH != 'gh-pages' && changesInclude('**.md')
+  container:
+    image: node:latest
+  install_script: npm install -g markdownlint-cli
+  lint_script: markdownlint --config=docs/.markdownlint.yml docs/
+
 task:
   name: Lint
   alias: lint
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   lint_script:
     - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
   always:
@@ -37,24 +50,28 @@ task:
       format: swiftformat
 
 task:
-  name: Build
-  alias: build
   only_if: $CIRRUS_TAG == ''
+  env:
+    matrix:
+      BUILD_ARCH: arm64
+      BUILD_ARCH: x86_64
+  name: Build ($BUILD_ARCH)
+  alias: build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
-  build_script: swift build --product tart
+  build_script: swift build --arch $BUILD_ARCH --product tart
-  sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
+  sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/$BUILD_ARCH-apple-macosx/debug/tart
   binary_artifacts:
-    path: .build/debug/tart
+    path: .build/$BUILD_ARCH-apple-macosx/debug/tart
 
 task:
-  name: Release (Dry Run)
   only_if: $CIRRUS_TAG == '' && ($CIRRUS_USER_PERMISSION == 'write' || $CIRRUS_USER_PERMISSION == 'admin')
+  name: Release (Dry Run)
   depends_on:
     - lint
     - build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   env:
     MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -70,7 +87,7 @@ task:
     - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
     - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
   install_script:
-    - brew install go goreleaser/tap/goreleaser-pro getsentry/tools/sentry-cli
+    - brew install go goreleaser/tap/goreleaser-pro
     - brew install mitchellh/gon/gon
   info_script:
     - security find-identity -v
@@ -89,7 +106,7 @@ task:
     - test
     - build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   env:
     MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -116,13 +133,15 @@ task:
     - swift -version
   release_script: goreleaser
   upload_sentry_debug_files_script:
-    - cd .build/arm64-apple-macosx/debug/
+    - cd .build/arm64-apple-macosx/release/
     # Generate and upload symbols
     - dsymutil tart
     - sentry-cli debug-files upload tart.dSYM/
+    - SENTRY_PROJECT=tart sentry-cli debug-files upload tart.dSYM/
     # Bundle and upload sources
     - sentry-cli debug-files bundle-sources tart.dSYM
     - sentry-cli debug-files upload tart.src.zip
+    - SENTRY_PROJECT=tart sentry-cli debug-files upload tart.src.zip
   create_sentry_release_script:
     - export SENTRY_RELEASE="tart@$CIRRUS_TAG"
     - sentry-cli releases new $SENTRY_RELEASE

.goreleaser.yml

@@ -3,42 +3,49 @@ project_name: tart
 before:
   hooks:
     - .ci/set-version.sh
-    - swift build -c debug --product tart
+    - swift build --arch x86_64 -c release --product tart
+    - swift build --arch arm64 -c release --product tart
+    - gon gon.hcl
 
 builds:
-  - builder: prebuilt
+  - id: tart
+    builder: prebuilt
+    goamd64: [v1]
     goos:
       - darwin
     goarch:
       - arm64
+      - amd64
+    binary: tart.app/Contents/MacOS/tart
     prebuilt:
-      path: .build/arm64-apple-macosx/debug/tart
+      path: '.build/{{- if eq .Arch "arm64" }}arm64{{- else }}x86_64{{ end }}-apple-macosx/release/tart'
-    hooks:
-      post:
-        - gon gon.hcl
 
 archives:
-  - name_template: "{{ .ProjectName }}"
+  - name_template: "{{ .ProjectName }}-{{ .Arch }}"
     files:
       - src: Resources/embedded.provisionprofile
         dst: tart.app/Contents
         strip_parent: true
-      - src: ".build/arm64-apple-macosx/debug/tart"
+      - src: Resources/Info.plist
-        dst: tart.app/Contents/MacOS
+        dst: tart.app/Contents
         strip_parent: true
+      - src: Resources/AppIcon.png
+        dst: tart.app/Contents/Resources
+        strip_parent: true
+      - LICENSE
 
 release:
   prerelease: auto
 
 brews:
   - name: tart
-    tap:
+    repository:
       owner: cirruslabs
       name: homebrew-cli
     caveats: See the GitHub repository for more information
     homepage: https://github.com/cirruslabs/tart
     license: "Fair Source"
-    description: Run macOS VMs on Apple Silicon
+    description: Run macOS and Linux VMs on Apple Hardware
     skip_upload: auto
     dependencies:
       - "cirruslabs/cli/softnet"
@@ -46,10 +53,4 @@ brews:
       libexec.install Dir["*"]
       bin.write_exec_script "#{libexec}/tart.app/Contents/MacOS/tart"
     custom_block: |
-      depends_on :macos => :monterey
+      depends_on :macos => :ventura
-      
-      on_macos do
-        unless Hardware::CPU.arm?
-          odie "Tart only works on Apple Silicon!"
-        end
-      end

.run/sign debug.run.xml

@@ -1,17 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="sign debug" type="ShConfigurationType">
-    <option name="SCRIPT_TEXT" value="codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart" />
-    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
-    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/scripts/sign.sh" />
-    <option name="SCRIPT_OPTIONS" value="" />
-    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
-    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
-    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
-    <option name="INTERPRETER_PATH" value="/bin/zsh" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="EXECUTE_IN_TERMINAL" value="true" />
-    <option name="EXECUTE_SCRIPT_FILE" value="false" />
-    <envs />
-    <method v="2" />
-  </configuration>
-</component>

.run/tart create.run.xml

@@ -1,8 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="tart create" type="SwiftPackageManagerRunConfiguration" factoryName="Swift Package Run" PROGRAM_PARAMS="create latest --from-ipsw=latest" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="tart" TARGET_NAME="tart" CONFIG_NAME="tart" RUN_TARGET_PROJECT_NAME="tart" RUN_TARGET_NAME="tart" WAS_MODIFIED="">
-    <method v="2">
-      <option name="SPM.BUILD_TASK_PROVIDER" enabled="true" />
-      <option name="RunConfigurationTask" enabled="true" run_configuration_name="sign debug" run_configuration_type="ShConfigurationType" />
-    </method>
-  </configuration>
-</component>

.run/tart run.run.xml

@@ -1,8 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="tart run" type="SwiftPackageManagerRunConfiguration" factoryName="Swift Package Run" PROGRAM_PARAMS="run latest" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="tart" TARGET_NAME="tart" CONFIG_NAME="tart" RUN_TARGET_PROJECT_NAME="tart" RUN_TARGET_NAME="tart" WAS_MODIFIED="">
-    <method v="2">
-      <option name="SPM.BUILD_TASK_PROVIDER" enabled="true" />
-      <option name="RunConfigurationTask" enabled="true" run_configuration_name="sign debug" run_configuration_type="ShConfigurationType" />
-    </method>
-  </configuration>
-</component>

CONTRIBUTING.md

@@ -0,0 +1,40 @@
+# Contributing to Tart
+
+Table of Contents
+-----------------
+
+- [How to Build](#how-to-build)
+- [How to Create an Issue/Enhancement](#how-to-create-an-issueenhancement)
+- [Style Guidelines](#style-guidelines)
+- [Pull Requests](#Pull-Requests)
+
+## How to Build
+
+1. Fork the repository to your own GitHub account
+2. Clone the forked repository to your local machine
+3. If using Xcode, use from Xcode 15 or newer
+4. Run ./scripts/run-signed.sh from the root of your repository
+
+```bash
+./scripts/run-signed.sh list
+```
+## How to Create an Issue/Enhancement
+
+1. Go to the [Issue page](https://github.com/cirruslabs/tart/issues) of the repository
+2. Click on the "New Issue" button
+3. Provide a descriptive title and detailed description of the issue or enhancement you're suggesting
+4. Submit the issue
+
+## Style Guidelines
+
+1. Code should follow camel case
+2. Code should follow [SwiftFormat](https://github.com/nicklockwood/SwiftFormat#swift-package-manager-plugin) guidelines. You can auto-format the code by running the following command:
+```bash
+swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore .
+```
+
+## Pull Requests
+
+1. Provide a detailed description of the changes you made in the pull request
+2. Wait for pull request to be reviewed 
+3. Make adjustments if necessary

Package.resolved

@@ -23,8 +23,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/getsentry/sentry-cocoa",
       "state" : {
-        "revision" : "ac224c437a3070ffe34460137ac8761eddaf2852",
+        "revision" : "d277532e1c8af813981ba01f591b15bbdd735615",
-        "version" : "8.3.3"
+        "version" : "8.8.0"
       }
     },
     {
@@ -81,6 +81,15 @@
         "version" : "1.0.2"
       }
     },
+    {
+      "identity" : "swift-sysctl",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/sersoft-gmbh/swift-sysctl.git",
+      "state" : {
+        "revision" : "71fd64ee84819bb19fbecfb36d5a4503726b6fb7",
+        "version" : "1.6.0"
+      }
+    },
     {
       "identity" : "swiftdate",
       "kind" : "remoteSourceControl",
@@ -99,6 +108,15 @@
         "version" : "0.50.6"
       }
     },
+    {
+      "identity" : "swiftradix",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/orchetect/SwiftRadix",
+      "state" : {
+        "revision" : "a52c37a4c213403f7377ae77b4c68451bcab8330",
+        "version" : "1.3.1"
+      }
+    },
     {
       "identity" : "texttable",
       "kind" : "remoteSourceControl",

Package.swift

@@ -4,7 +4,7 @@ import PackageDescription
 let package = Package(
   name: "Tart",
   platforms: [
-    .macOS(.v12)
+    .macOS(.v13)
   ],
   products: [
     .executable(name: "tart", targets: ["tart"])
@@ -18,8 +18,10 @@ let package = Package(
     .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
     .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.0.0")),
     .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.50.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.3.3"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.8.0"),
     .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
+    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.0.0"),
+    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.0")
   ],
   targets: [
     .executableTarget(name: "tart", dependencies: [
@@ -32,6 +34,8 @@ let package = Package(
       .product(name: "Atomics", package: "swift-atomics"),
       .product(name: "Sentry", package: "sentry-cocoa"),
       .product(name: "TextTable", package: "TextTable"),
+      .product(name: "Sysctl", package: "swift-sysctl"),
+      .product(name: "SwiftRadix", package: "SwiftRadix"),
     ], exclude: [
       "OCI/Reference/Makefile",
       "OCI/Reference/Reference.g4",

README.md

@@ -6,23 +6,18 @@ Built by CI engineers for your automation needs. Here are some highlights of Tar
 * Tart uses Apple's own `Virtualization.Framework` for [near-native performance](https://browser.geekbench.com/v5/cpu/compare/20382844?baseline=20382722).
 * Push/Pull virtual machines from any OCI-compatible container registry.
 * Use Tart Packer Plugin to automate VM creation.
-* Built-in CI integration.
+* Easily integrates with any CI system.
 
-*Tart* is already adopted by several automation services:
+Tart powers [Cirrus Runners](https://cirrus-runners.app/)
+service — a drop-in replacement for the standard GitHub-hosted runners, offering 2-3 times better performance for a fraction of the price.
 
 <p align="center">
-  <a href="https://cirrus-ci.org/guide/macOS/" target=_blank>
+  <a href="https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/CirrusCI.png" height="65"/>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/CirrusRunnersForGHA.png" height="65"/>
-  </a>
-  <a href="https://codemagic.io/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Codemagic.png" height="65"/>
-  </a>
-  <a href="https://testingbot.com/" target=_blank>
-    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/TestingBot.png" height="65"/>
   </a>
 </p>
 
-Many more companies are using Tart in their internal setups. Here are a few of them:
+Many companies are using Tart in their internal setups. Here are a few of them:
 
 <p align="center">
   <a href="https://ahrefs.com/" target=_blank>
@@ -31,6 +26,12 @@ Many more companies are using Tart in their internal setups. Here are a few of t
   <a href="https://krisp.ai/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
   </a>
+  <a href="https://mullvad.net/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Mullvad.png" height="65"/>
+  </a>
+  <a href="https://shape.dk/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/shape.png" height="65"/>
+  </a>
   <a href="https://suran.com/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Suran.png" height="65"/>
   </a>
@@ -40,18 +41,30 @@ Many more companies are using Tart in their internal setups. Here are a few of t
   <a href="https://transloadit.com/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png" height="65"/>
   </a>
+  <a href="https://uphold.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png" height="65"/>
+  </a>
+  <a href="https://www.pitsdatarecovery.net/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
+  </a>
 </p>
 
 **Note:** If your company or project is using Tart please consider [adding yourself to the list above](/Resources/Users/HowToAddYourself.md).
 
+<p align="center">
+  <a href="https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws?utm_source=github&utm_medium=referral" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/AWSMarkeplaceLogo.png" height="90"/>
+  </a>
+</p>
+
 ## Usage
 
-Try running a Tart VM on your Apple Silicon device running macOS 12.0 (Monterey) or later (will download a 25 GB image):
+Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura) or later (will download a 25 GB image):
 
 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
+tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
-tart run ventura-base
+tart run sonoma-base
 ```
 
 Please check the [official documentation](https://tart.run) for more information and/or feel free to use [discussions](https://github.com/cirruslabs/tart/discussions)

Resources/Info.plist

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleName</key>
+    <string>tart</string>
+    <key>CFBundleIdentifier</key>
+    <string>org.cirruslabs.tart</string>
+    <key>CFBundleExecutable</key>
+    <string>tart</string>
+    <key>LSBackgroundOnly</key>
+    <string>1</string>
+    <key>CFBundleIconFiles</key>
+    <array>
+        <string>AppIcon.png</string>
+    </array>
+    <key>NSAppTransportSecurity</key>
+    <dict>
+        <key>NSAllowsArbitraryLoads</key>
+        <true/>
+    </dict>
+</dict>
+</plist>

Resources/tart-dev.entitlements

@@ -4,5 +4,7 @@
 <dict>
 	<key>com.apple.security.virtualization</key>
 	<true/>
+	<key>com.apple.security.get-task-allow</key>
+	<true/>
 </dict>
 </plist>

Sources/tart/Commands/Clone.swift

@@ -3,7 +3,20 @@ import Foundation
 import SystemConfiguration
 
 struct Clone: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Clone a VM")
+  static var configuration = CommandConfiguration(
+    abstract: "Clone a VM",
+    discussion: """
+    Creates a local virtual machine by cloning either a remote or another local virtual machine.
+
+    Due to copy-on-write magic in Apple File System a cloned VM won't actually claim all the space right away.
+    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
+    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
+    will be modified.
+
+    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable. This might be helpful
+    for use cases when the original image is very big and a workload is known to only modify a fraction of the cloned disk.
+    """
+  )
 
   @Argument(help: "source VM name")
   var sourceName: String
@@ -14,10 +27,17 @@ struct Clone: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+  var concurrency: UInt = 4
+
   func validate() throws {
     if newName.contains("/") {
       throw ValidationError("<new-name> should be a local name")
     }
+
+    if concurrency < 1 {
+      throw ValidationError("network concurrency cannot be less than 1")
+    }
   }
 
   func run() async throws {
@@ -27,11 +47,10 @@ struct Clone: AsyncParsableCommand {
     if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
       // Pull the VM in case it's OCI-based and doesn't exist locally yet
       let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
     }
 
     let sourceVM = try VMStorageHelper.open(sourceName)
-
     let tmpVMDir = try VMDirectory.temporary()
 
     // Lock the temporary VM directory to prevent it's garbage collection
@@ -44,10 +63,17 @@ struct Clone: AsyncParsableCommand {
       try lock.lock()
 
       let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
+        && sourceVM.state() != "suspended"
       try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)
+
       try localStorage.move(newName, from: tmpVMDir)
 
       try lock.unlock()
+
+      // APFS is doing copy-on-write so the above cloning operation (just copying files on disk)
+      // is not actually claiming new space until the VM is started and it writes something to disk.
+      // So once we clone the VM let's try to claim a little bit of space for the VM to run.
+      try Prune.reclaimIfNeeded(UInt64(sourceVM.allocatedSizeBytes()), sourceVM)
     }, onCancel: {
       try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
     })

Sources/tart/Commands/Create.swift

@@ -1,7 +1,8 @@
 import ArgumentParser
 import Dispatch
-import SwiftUI
 import Foundation
+import SwiftUI
+import Virtualization
 
 struct Create: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Create a VM")
@@ -22,6 +23,11 @@ struct Create: AsyncParsableCommand {
     if fromIPSW == nil && !linux {
       throw ValidationError("Please specify either a --from-ipsw or --linux option!")
     }
+    #if arch(x86_64)
+      if fromIPSW != nil {
+        throw ValidationError("Only Linux VMs are supported on Intel!")
+      }
+    #endif
   }
 
   func run() async throws {
@@ -32,26 +38,32 @@ struct Create: AsyncParsableCommand {
     try tmpVMDirLock.lock()
 
     try await withTaskCancellationHandler(operation: {
-      if let fromIPSW = fromIPSW {
+      #if arch(arm64)
-        let ipswURL: URL
+        if let fromIPSW = fromIPSW {
+          let ipswURL: URL
 
-        if fromIPSW == "latest" {
+          if fromIPSW == "latest" {
-          ipswURL = try await VM.latestIPSWURL()
+            defaultLogger.appendNewLine("Looking up the latest supported IPSW...")
-        } else if fromIPSW.starts(with: "http://") || fromIPSW.starts(with: "https://") {
+
-          ipswURL = URL(string: fromIPSW)!
+            let image = try await withCheckedThrowingContinuation { continuation in
-        } else {
+              VZMacOSRestoreImage.fetchLatestSupported() { result in
-          ipswURL = URL(fileURLWithPath: fromIPSW)
+                continuation.resume(with: result)
+              }
+            }
+
+            ipswURL = image.url
+          } else if fromIPSW.starts(with: "http://") || fromIPSW.starts(with: "https://") {
+            ipswURL = URL(string: fromIPSW)!
+          } else {
+            ipswURL = URL(fileURLWithPath: NSString(string: fromIPSW).expandingTildeInPath)
+          }
+
+          _ = try await VM(vmDir: tmpVMDir, ipswURL: ipswURL, diskSizeGB: diskSize)
         }
-
+      #endif
-        _ = try await VM(vmDir: tmpVMDir, ipswURL: ipswURL, diskSizeGB: diskSize)
-      }
 
       if linux {
-        if #available(macOS 13, *) {
+        _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
-          _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
-        } else {
-          throw UnsupportedOSError("Linux VMs", "are")
-        }
       }
 
       try VMStorageLocal().move(name, from: tmpVMDir)

Sources/tart/Commands/FQN.swift

@@ -0,0 +1,22 @@
+import ArgumentParser
+import Foundation
+import SystemConfiguration
+
+struct FQN: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(abstract: "Get a fully-qualified VM name", shouldDisplay: false)
+
+  @Argument(help: "VM name")
+  var name: String
+
+  func run() async throws {
+    if var remoteName = try? RemoteName(name) {
+      let digest = try VMStorageOCI().digest(remoteName)
+
+      remoteName.reference = Reference(digest: digest)
+
+      print(remoteName)
+    } else {
+      print(name)
+    }
+  }
+}

Sources/tart/Commands/Get.swift

@@ -5,8 +5,10 @@ fileprivate struct VMInfo: Encodable {
   let CPU: Int
   let Memory: UInt64
   let Disk: Int
+  let Size: String
   let Display: String
   let Running: Bool
+  let State: String
 }
 
 struct Get: AsyncParsableCommand {
@@ -21,11 +23,10 @@ struct Get: AsyncParsableCommand {
   func run() async throws {
     let vmDir = try VMStorageLocal().open(name)
     let vmConfig = try VMConfig(fromURL: vmDir.configURL)
-    let diskSizeInGb = try vmDir.sizeGB()
     let memorySizeInMb = vmConfig.memorySize / 1024 / 1024
 
-    let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: diskSizeInGb,
+    let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: try vmDir.sizeGB(), Size: String(format: "%.3f", Float(try vmDir.allocatedSizeBytes()) / 1000 / 1000 / 1000),
-                      Display: vmConfig.display.description, Running: try vmDir.running())
+                      Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state())
     print(format.renderSingle(info))
   }
 }

Sources/tart/Commands/IP.swift

@@ -35,8 +35,19 @@ struct IP: AsyncParsableCommand {
     let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!
 
     guard let ip = try await IP.resolveIP(vmMACAddress, resolutionStrategy: resolver, secondsToWait: wait) else {
-      throw RuntimeError.NoIPAddressFound("no IP address found, is your VM running?")
+      var message = "no IP address found"
+
+      if try !vmDir.running() {
+        message += ", is your VM running?"
+      }
+
+      if (vmConfig.os == .linux && resolver == .arp) {
+        message += " (not all Linux distributions are compatible with the ARP resolver)"
+      }
+
+      throw RuntimeError.NoIPAddressFound(message)
     }
+
     print(ip)
   }
 

Sources/tart/Commands/List.swift

@@ -5,8 +5,10 @@ import SwiftUI
 fileprivate struct VMInfo: Encodable {
   let Source: String
   let Name: String
+  let Disk: Int
   let Size: Int
   let Running: Bool
+  let State: String
 }
 
 struct List: AsyncParsableCommand {
@@ -36,13 +38,13 @@ struct List: AsyncParsableCommand {
 
     if source == nil || source == "local" {
       infos += sortedInfos(try VMStorageLocal().list().map { (name, vmDir) in
-        try VMInfo(Source: "local", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running())
+        try VMInfo(Source: "local", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state())
       })
     }
 
     if source == nil || source == "oci" {
       infos += sortedInfos(try VMStorageOCI().list().map { (name, vmDir, _) in
-        try VMInfo(Source: "oci", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running())
+        try VMInfo(Source: "oci", Name: name, Disk: vmDir.sizeGB(), Size: vmDir.allocatedSizeGB(), Running: vmDir.running(), State: vmDir.state())
       })
     }
 

Sources/tart/Commands/Login.swift

@@ -17,6 +17,9 @@ struct Login: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Flag(help: "skip validation of the registry's credentials before logging-in")
+  var noValidate: Bool = false
+
   func validate() throws {
     let usernameProvided = username != nil
     let passwordProvided = passwordStdin
@@ -35,6 +38,9 @@ struct Login: AsyncParsableCommand {
 
       let passwordData = FileHandle.standardInput.readDataToEndOfFile()
       password = String(decoding: passwordData, as: UTF8.self)
+
+      // Support "echo $PASSWORD | tart login --username $USERNAME --password-stdin $REGISTRY"
+      password.trimSuffix { c in c.isNewline }
     } else {
       (user, password) = try StdinCredentials.retrieve()
     }
@@ -42,12 +48,14 @@ struct Login: AsyncParsableCommand {
       host: (user, password)
     ])
 
-    do {
+    if !noValidate {
-      let registry = try Registry(host: host, namespace: "", insecure: insecure,
+      do {
-                                  credentialsProviders: [credentialsProvider])
+        let registry = try Registry(host: host, namespace: "", insecure: insecure,
-      try await registry.ping()
+                                    credentialsProviders: [credentialsProvider])
-    } catch {
+        try await registry.ping()
-      throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
+      } catch {
+        throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
+      }
     }
 
     try KeychainCredentialsProvider().store(host: host, user: user, password: password)

Sources/tart/Commands/Logout.swift

@@ -0,0 +1,14 @@
+import ArgumentParser
+import Dispatch
+import SwiftUI
+
+struct Logout: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(abstract: "Logout from a registry")
+
+  @Argument(help: "host")
+  var host: String
+
+  func run() async throws {
+    try KeychainCredentialsProvider().remove(host: host)
+  }
+}

Sources/tart/Commands/Prune.swift

@@ -5,23 +5,40 @@ import SwiftUI
 import SwiftDate
 
 struct Prune: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches")
+  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches or local VMs")
 
-  @Option(help: ArgumentHelp("Remove cache entries last accessed more than n days ago",
+  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."))
+  var entries: String = "caches"
+
+  @Option(help: ArgumentHelp("Remove entries that were last accessed more than n days ago",
                              discussion: "For example, --older-than=7 will remove entries that weren't accessed by Tart in the last 7 days.",
                              valueName: "n"))
   var olderThan: UInt?
 
-  @Option(help: ArgumentHelp("Remove least recently used cache entries that do not fit the specified cache size budget n, expressed in gigabytes",
+  @Option(help: .hidden)
-                             discussion: "For example, --cache-budget=50 will effectively shrink all caches to a total size of 50 gigabytes.",
-                             valueName: "n"))
   var cacheBudget: UInt?
 
+  @Option(help: ArgumentHelp("Remove the least recently used entries that do not fit the specified space size budget n, expressed in gigabytes",
+                             discussion: "For example, --space-budget=50 will effectively shrink all entries to a total size of 50 gigabytes.",
+                             valueName: "n"))
+  var spaceBudget: UInt?
+
   @Flag(help: .hidden)
   var gc: Bool = false
 
-  func validate() throws {
+  mutating func validate() throws {
-    if olderThan == nil && cacheBudget == nil && !gc {
+    // --cache-budget deprecation logic
+    if let cacheBudget = cacheBudget {
+      fputs("--cache-budget is deprecated, please use --space-budget\n", stderr)
+
+      if spaceBudget != nil {
+        throw ValidationError("--cache-budget is deprecated, please use --space-budget")
+      }
+
+      spaceBudget = cacheBudget
+    }
+
+    if olderThan == nil && spaceBudget == nil && !gc {
       throw ValidationError("at least one pruning criteria must be specified")
     }
   }
@@ -31,43 +48,53 @@ struct Prune: AsyncParsableCommand {
       try VMStorageOCI().gc()
     }
 
+    // Build a list of prunable storages that we're going to prune based on user's request
+    let prunableStorages: [PrunableStorage]
+
+    switch entries {
+    case "caches":
+      prunableStorages = [VMStorageOCI(), try IPSWCache()]
+    case "vms":
+      prunableStorages = [VMStorageLocal()]
+    default:
+      throw ValidationError("unsupported --entries value, please specify either \"caches\" or \"vms\"")
+    }
+
     // Clean up cache entries based on last accessed date
     if let olderThan = olderThan {
       let olderThanInterval = Int(exactly: olderThan)!.days.timeInterval
-      let olderThanDate = Date().addingTimeInterval(olderThanInterval)
+      let olderThanDate = Date() - olderThanInterval
 
-      try Prune.pruneOlderThan(olderThanDate: olderThanDate)
+      try Prune.pruneOlderThan(prunableStorages: prunableStorages, olderThanDate: olderThanDate)
     }
 
     // Clean up cache entries based on imposed cache size limit and entry's last accessed date
-    if let cacheBudget = cacheBudget {
+    if let spaceBudget = spaceBudget {
-      try Prune.pruneCacheBudget(cacheBudgetBytes: UInt64(cacheBudget) * 1024 * 1024 * 1024)
+      try Prune.pruneSpaceBudget(prunableStorages: prunableStorages, spaceBudgetBytes: UInt64(spaceBudget) * 1024 * 1024 * 1024)
     }
   }
 
-  static func pruneOlderThan(olderThanDate: Date) throws {
+  static func pruneOlderThan(prunableStorages: [PrunableStorage], olderThanDate: Date) throws {
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
     let prunables: [Prunable] = try prunableStorages.flatMap { try $0.prunables() }
 
     try prunables.filter { try $0.accessDate() <= olderThanDate }.forEach { try $0.delete() }
   }
 
-  static func pruneCacheBudget(cacheBudgetBytes: UInt64) throws {
+  static func pruneSpaceBudget(prunableStorages: [PrunableStorage], spaceBudgetBytes: UInt64) throws {
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
     let prunables: [Prunable] = try prunableStorages
       .flatMap { try $0.prunables() }
       .sorted { try $0.accessDate() > $1.accessDate() }
 
-    var cacheBudgetBytes = cacheBudgetBytes
+    var spaceBudgetBytes = spaceBudgetBytes
     var prunablesToDelete: [Prunable] = []
 
     for prunable in prunables {
-      let prunableSizeBytes = UInt64(try prunable.sizeBytes())
+      let prunableSizeBytes = UInt64(try prunable.allocatedSizeBytes())
 
-      if prunableSizeBytes <= cacheBudgetBytes {
+      if prunableSizeBytes <= spaceBudgetBytes {
         // Don't mark for deletion as
         // there's a budget available
-        cacheBudgetBytes -= prunableSizeBytes
+        spaceBudgetBytes -= prunableSizeBytes
       } else {
         // Mark for deletion
         prunablesToDelete.append(prunable)
@@ -77,14 +104,61 @@ struct Prune: AsyncParsableCommand {
     try prunablesToDelete.forEach { try $0.delete() }
   }
 
-  static func pruneReclaim(reclaimBytes: UInt64) throws {
+  static func reclaimIfNeeded(_ requiredBytes: UInt64, _ initiator: Prunable? = nil) throws {
+    if ProcessInfo.processInfo.environment.keys.contains("TART_NO_AUTO_PRUNE") {
+      return
+    }
+
+    SentrySDK.configureScope { scope in
+      scope.setContext(value: ["requiredBytes": requiredBytes], key: "Prune")
+    }
+
+    // Figure out how much disk space is available
+    let attrs = try Config().tartCacheDir.resourceValues(forKeys: [
+      .volumeAvailableCapacityKey,
+      .volumeAvailableCapacityForImportantUsageKey
+    ])
+    let volumeAvailableCapacityCalculated = max(
+      UInt64(attrs.volumeAvailableCapacity!),
+      UInt64(attrs.volumeAvailableCapacityForImportantUsage!)
+    )
+
+    SentrySDK.configureScope { scope in
+      scope.setContext(value: [
+        "volumeAvailableCapacity": attrs.volumeAvailableCapacity!,
+        "volumeAvailableCapacityForImportantUsage": attrs.volumeAvailableCapacityForImportantUsage!,
+        "volumeAvailableCapacityCalculated": volumeAvailableCapacityCalculated
+      ], key: "Prune")
+    }
+
+    if volumeAvailableCapacityCalculated <= 0 {
+      SentrySDK.capture(message: "Zero volume capacity reported") { scope in
+        scope.setLevel(.warning)
+      }
+
+      return
+    }
+
+    // Now that we know how much free space is left,
+    // check if we even need to reclaim anything
+    if requiredBytes < volumeAvailableCapacityCalculated {
+      return
+    }
+
+    try Prune.reclaimIfPossible(requiredBytes - volumeAvailableCapacityCalculated, initiator)
+  }
+
+  private static func reclaimIfPossible(_ reclaimBytes: UInt64, _ initiator: Prunable? = nil) throws {
+    let transaction = SentrySDK.startTransaction(name: "Pruning cache", operation: "prune", bindToScope: true)
+    defer { transaction.finish() }
+
     let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
     let prunables: [Prunable] = try prunableStorages
       .flatMap { try $0.prunables() }
       .sorted { try $0.accessDate() < $1.accessDate() }
 
     // Does it even make sense to start?
-    let cacheUsedBytes = try prunables.map { try $0.sizeBytes() }.reduce(0, +)
+    let cacheUsedBytes = try prunables.map { try $0.allocatedSizeBytes() }.reduce(0, +)
     if cacheUsedBytes < reclaimBytes {
       return
     }
@@ -98,10 +172,16 @@ struct Prune: AsyncParsableCommand {
         break
       }
 
-      cacheReclaimedBytes += try prunable.sizeBytes()
+      if prunable.url == initiator?.url.resolvingSymlinksInPath() {
-      try prunable.delete()
+        // do not prune the initiator
+        continue
+      }
 
-      try SentrySDK.span?.setExtra(value: prunable.sizeBytes(), key: prunable.url.path);
+      try SentrySDK.span?.setData(value: prunable.allocatedSizeBytes(), key: prunable.url.path)
+
+      cacheReclaimedBytes += try prunable.allocatedSizeBytes()
+
+      try prunable.delete()
     }
 
     SentrySDK.span?.setMeasurement(name: "gc_disk_reclaimed", value: cacheReclaimedBytes as NSNumber, unit: MeasurementUnitInformation.byte);

Sources/tart/Commands/Pull.swift

@@ -3,7 +3,16 @@ import Dispatch
 import SwiftUI
 
 struct Pull: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Pull a VM from a registry")
+  static var configuration = CommandConfiguration(
+    abstract: "Pull a VM from a registry",
+    discussion: """
+    Pulls a virtual machine from a remote OCI-compatible registry. Supports authorization via Keychain (see "tart login --help"),
+    Docker credential helpers defined in ~/.docker/config.json or via TART_REGISTRY_USERNAME/TART_REGISTRY_PASSWORD environment variables.
+
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image to fit via "tart prune".
+    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
+    """
+  )
 
   @Argument(help: "remote VM name")
   var remoteName: String
@@ -11,6 +20,15 @@ struct Pull: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+  var concurrency: UInt = 4
+
+  func validate() throws {
+    if concurrency < 1 {
+      throw ValidationError("network concurrency cannot be less than 1")
+    }
+  }
+
   func run() async throws {
     // Be more liberal when accepting local image as argument,
     // see https://github.com/cirruslabs/tart/issues/36
@@ -25,6 +43,6 @@ struct Pull: AsyncParsableCommand {
 
     defaultLogger.appendNewLine("pulling \(remoteName)...")
 
-    try await VMStorageOCI().pull(remoteName, registry: registry)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
   }
 }

Sources/tart/Commands/Push.swift

@@ -6,7 +6,7 @@ import Compression
 struct Push: AsyncParsableCommand {
   static var configuration = CommandConfiguration(abstract: "Push a VM to a registry")
 
-  @Argument(help: "local VM name")
+  @Argument(help: "local or remote VM name")
   var localName: String
 
   @Argument(help: "remote VM name(s)")
@@ -23,12 +23,16 @@ struct Push: AsyncParsableCommand {
                              """))
   var chunkSize: Int = 0
 
+  @Option(help: .hidden)
+  var diskFormat: String = "v2"
+
   @Flag(help: ArgumentHelp("cache pushed images locally",
                            discussion: "Increases disk usage, but saves time if you're going to pull the pushed images later."))
   var populateCache: Bool = false
 
   func run() async throws {
-    let localVMDir = try VMStorageLocal().open(localName)
+    let ociStorage = VMStorageOCI()
+    let localVMDir = try VMStorageHelper.open(localName)
 
     // Parse remote names supplied by the user
     let remoteNames = try remoteNames.map{
@@ -53,23 +57,56 @@ struct Push: AsyncParsableCommand {
       defaultLogger.appendNewLine("pushing \(localName) to "
         + "\(registryIdentifier.host)/\(registryIdentifier.namespace)\(remoteNamesForRegistry.referenceNames())...")
 
-      let pushedRemoteName = try await localVMDir.pushToRegistry(
+      let references = remoteNamesForRegistry.map{ $0.reference.value }
-        registry: registry,
-        references: remoteNamesForRegistry.map{ $0.reference.value },
-        chunkSizeMb: chunkSize
-      )
 
-      // Populate the local cache (if requested)
+      let pushedRemoteName: RemoteName
+      // If we're pushing a local OCI VM, check if points to an already existing registry manifest
+      // and if so, only upload manifests (without config, disk and NVRAM) to the user-specified references
+      if let remoteName = try? RemoteName(localName) {
+        pushedRemoteName = try await lightweightPushToRegistry(
+          registry: registry, 
+          remoteName: remoteName,
+          references: references
+        )
+      } else {
+        pushedRemoteName = try await localVMDir.pushToRegistry(
+          registry: registry,
+          references: references,
+          chunkSizeMb: chunkSize,
+          diskFormat: diskFormat
+        )
+        // Populate the local cache (if requested)
+        if populateCache {
+          let expectedPushedVMDir = try ociStorage.create(pushedRemoteName)
+          try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false)
+        }
+      }
+
+      // link the rest remote names
       if populateCache {
-        let ociStorage = VMStorageOCI()
-        let expectedPushedVMDir = try ociStorage.create(pushedRemoteName)
-        try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false)
         for remoteName in remoteNamesForRegistry {
           try ociStorage.link(from: remoteName, to: pushedRemoteName)
         }
       }
     }
   }
+
+  func lightweightPushToRegistry(registry: Registry, remoteName: RemoteName, references: [String]) async throws -> RemoteName {
+    // Is the local OCI VM already present in the registry?
+    let digest = try VMStorageOCI().digest(remoteName)
+
+    let (remoteManifest, _) = try await registry.pullManifest(reference: digest)
+
+    // Overwrite registry's references with the retrieved manifest
+    for reference in references {
+      defaultLogger.appendNewLine("pushing manifest for \(reference)...")
+
+      _ = try await registry.pushManifest(reference: reference, manifest: remoteManifest)
+    }
+
+    return RemoteName(host: registry.host!, namespace: registry.namespace,
+                      reference: Reference(digest: digest))
+  }
 }
 
 extension Collection where Element == RemoteName {

Sources/tart/Commands/Rename.swift

@@ -24,7 +24,7 @@ struct Rename: AsyncParsableCommand {
     }
 
     if localStorage.exists(newName) {
-      throw ValidationError("failed to rename VM \(name), target VM \(name) already exists, delete it first!")
+      throw ValidationError("failed to rename VM \(name), target VM \(newName) already exists, delete it first!")
     }
 
     try localStorage.rename(name, newName)

Sources/tart/Commands/Run.swift

@@ -1,4 +1,6 @@
 import ArgumentParser
+import Cocoa
+import Darwin
 import Dispatch
 import SwiftUI
 import Virtualization
@@ -31,54 +33,77 @@ struct Run: AsyncParsableCommand {
   ))
   var serialPath: String?
 
-  @Flag(help: "Force open a UI window, even when VNC is enabled.")
+  @Flag(help: ArgumentHelp("Force open a UI window, even when VNC is enabled.", visibility: .private))
   var graphics: Bool = false
 
-  @Flag(help: "Boot into recovery mode")
+  @Flag(help: "Disable audio pass-through to host.")
+  var noAudio: Bool = false
+
+  #if arch(arm64)
+    @Flag(help: "Boot into recovery mode")
+  #endif
   var recovery: Bool = false
 
-  @Flag(help: ArgumentHelp(
+  #if arch(arm64)
-    "Use screen sharing instead of the built-in UI.",
+    @Flag(help: ArgumentHelp(
-    discussion: "Useful since Screen Sharing supports copy/paste, drag and drop, etc.\n"
+      "Use screen sharing instead of the built-in UI.",
-      + "Note that Remote Login option should be enabled inside the VM."))
+      discussion: "Useful since Screen Sharing supports copy/paste, drag and drop, etc.\n"
+        + "Note that Remote Login option should be enabled inside the VM."))
+  #endif
   var vnc: Bool = false
 
-  @Flag(help: ArgumentHelp(
+  #if arch(arm64)
-    "Use Virtualization.Framework's VNC server instead of the build-in UI.",
+    @Flag(help: ArgumentHelp(
-    discussion: "Useful since this type of VNC is available in recovery mode and in macOS installation.\n"
+      "Use Virtualization.Framework's VNC server instead of the build-in UI.",
-      + "Note that this feature is experimental and there may be bugs present when using VNC."))
+      discussion: "Useful since this type of VNC is available in recovery mode and in macOS installation.\n"
+        + "Note that this feature is experimental and there may be bugs present when using VNC."))
+  #endif
   var vncExperimental: Bool = false
 
   @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\")
+  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\" --disk=\"/dev/disk0\")
   """, discussion: """
+  Can be either a disk image file or a block device like a local SSD on AWS EC2 Mac instances.
+
   Learn how to create a disk image using Disk Utility here:
   https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+
+  To work with block devices 'tart' binary must be executed as root which affects locating Tart VMs.
+  To workaround this issue pass TART_HOME explicitly:
+
+  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
   """, valueName: "path[:ro]"))
   var disk: [String] = []
 
-  @Option(name: [.customLong("rosetta")], help: ArgumentHelp(
+  #if arch(arm64)
-    "Attaches a Rosetta share to the guest Linux VM with a specific tag (e.g. --rosetta=\"rosetta\")",
+    @Option(name: [.customLong("rosetta")], help: ArgumentHelp(
-    discussion: """
+      "Attaches a Rosetta share to the guest Linux VM with a specific tag (e.g. --rosetta=\"rosetta\")",
-    Requires host to be macOS 13.0 (Ventura) with Rosetta installed. The latter can be done
+      discussion: """
-    by running "softwareupdate --install-rosetta" (without quotes) in the Terminal.app.
+      Requires host to be macOS 13.0 (Ventura) with Rosetta installed. The latter can be done
+      by running "softwareupdate --install-rosetta" (without quotes) in the Terminal.app.
 
-    Note that you also have to configure Rosetta in the guest Linux VM by following the
+      Note that you also have to configure Rosetta in the guest Linux VM by following the
-    steps from "Mount the Shared Directory and Register Rosetta" section here:
+      steps from "Mount the Shared Directory and Register Rosetta" section here:
-    https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta#3978496
+      https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta#3978496
-    """,
+      """,
-    valueName: "tag"
+      valueName: "tag"
-  ))
+    ))
+  #endif
   var rosettaTag: String?
 
-  @Option(help: ArgumentHelp("""
+  @Option(help: ArgumentHelp("Additional directory shares with an optional read-only and mount tag options (e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")", discussion: """
-  Additional directory shares with an optional read-only specifier\n(e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\")
+  Requires host to be macOS 13.0 (Ventura) or newer. macOS guests must be running macOS 13.0 (Ventura) or newer too.
-  """, discussion: """
+
-  Requires host to be macOS 13.0 (Ventura) or newer.
+  Options are comma-separated and are as follows:
-  All shared directories are automatically mounted to "/Volumes/My Shared Files" directory on macOS,
+
-  while on Linux you have to do it manually: "mount -t virtiofs com.apple.virtio-fs.automount /mount/point".
+  * ro — mount this directory share in read-only mode instead of the default read-write (e.g. --dir=\"~/src/sources:ro\")
-  For macOS guests, they must be running macOS 13.0 (Ventura) or newer.
+
-  """, valueName: "name:path[:ro]"))
+  * tag=<TAG> — by default, the \"com.apple.virtio-fs.automount\" mount tag is used for all directory shares. On macOS, this causes the directories to be automatically mounted to "/Volumes/My Shared Files" directory. On Linux, you have to do it manually: "mount -t virtiofs com.apple.virtio-fs.automount /mount/point".
+
+  Mount tag can be overridden by appending tag property to the directory share (e.g. --dir=\"~/src/build:tag=build\" or --dir=\"~/src/build:ro,tag=build\"). Then it can be mounted via "mount_virtiofs build ~/build" inside guest macOS and "mount -t virtiofs build ~/build" inside guest Linux.
+
+  In case of passing multiple directories per mount tag it is required to prefix them with names e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\". These names will be used as directory names under the mounting point inside guests. For the example above it will be "/Volumes/My Shared Files/build" and "/Volumes/My Shared Files/sources" respectively.
+  """, valueName: "[name:]path[:options]"))
   var dir: [String] = []
 
   @Option(help: ArgumentHelp("""
@@ -86,49 +111,97 @@ struct Run: AsyncParsableCommand {
   """, discussion: """
   Specify "list" as an interface name (--net-bridged=list) to list the available bridged interfaces.
   """, valueName: "interface name"))
-  var netBridged: String?
+  var netBridged: [String] = []
 
   @Flag(help: ArgumentHelp("Use software networking instead of the default shared (NAT) networking",
                            discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
   var netSoftnet: Bool = false
 
-  func validate() throws {
+  @Option(help: ArgumentHelp("Comma-separated list of CIDRs to allow the traffic to when using Softnet isolation\n(e.g. --net-softnet-allow=192.168.0.0/24)", valueName: "comma-separated CIDRs"))
+  var netSoftnetAllow: String?
+
+  @Flag(help: ArgumentHelp("Restrict network access to the host-only network"))
+  var netHost: Bool = false
+
+  #if arch(arm64)
+    @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
+  #endif
+  var suspendable: Bool = false
+
+  #if arch(arm64)
+    @Flag(help: ArgumentHelp("Whether system hot keys should be sent to the guest instead of the host",
+                             discussion: "If enabled then system hot keys like Cmd+Tab will be sent to the guest instead of the host."))
+  #endif
+  var captureSystemKeys: Bool = false
+
+  mutating func validate() throws {
     if vnc && vncExperimental {
       throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
     }
-    if netBridged != nil && netSoftnet {
+
-      throw ValidationError("--net-bridged and --net-softnet are mutually exclusive")
+    // check that not more than one network option is specified
+    var netFlags = 0
+    if netBridged.count > 0 { netFlags += 1 }
+    if netSoftnet { netFlags += 1 }
+    if netHost { netFlags += 1 }
+
+    if netFlags > 1 {
+      throw ValidationError("--net-bridged, --net-softnet and --net-host are mutually exclusive")
     }
 
     if graphics && noGraphics {
       throw ValidationError("--graphics and --no-graphics are mutually exclusive")
     }
+
+    if (noGraphics || vnc || vncExperimental) && captureSystemKeys {
+      throw ValidationError("--captures-system-keys can only be used with the default VM view")
+    }
+
+    let localStorage = VMStorageLocal()
+    let vmDir = try localStorage.open(name)
+    if try vmDir.state() == "suspended" {
+      suspendable = true
+    }
+
+    if suspendable {
+      if dir.count > 0 {
+        throw ValidationError("Suspending VMs with shared directories is not supported")
+      }
+    }
+
+    for disk in disk {
+      if disk.hasSuffix("-amd64.iso") {
+        throw ValidationError("Seems you have a disk targeting x86 architecture (hence amd64 in the name). Please use an 'arm64' version of the disk.")
+      }
+    }
   }
 
   @MainActor
   func run() async throws {
-    let vmDir = try VMStorageLocal().open(name)
+    let localStorage = VMStorageLocal()
+    let vmDir = try localStorage.open(name)
 
-    if netSoftnet && isInteractiveSession() {
+    let storageLock = try FileLock(lockURL: Config().tartHomeDir)
+    if try vmDir.state() == "suspended" {
+      try storageLock.lock() // lock before checking
+      let needToGenerateNewMac = try localStorage.list().contains {
+        // check if there is a running VM with the same MAC but different name
+        try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
+      }
+
+      if needToGenerateNewMac {
+        print("There is already a running VM with the same MAC address!")
+        print("Resetting VM to assign a new MAC address...")
+        try vmDir.regenerateMACAddress()
+      }
+    }
+
+    if (netSoftnet || netHost) && isInteractiveSession() {
       try Softnet.configureSUIDBitIfNeeded()
     }
 
     let additionalDiskAttachments = try additionalDiskAttachments()
 
-    // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-    // see https://github.com/cirruslabs/tart/issues/323 for more details.
-    for additionalDiskAttachment in additionalDiskAttachments {
-      // Read-only attachments do not seem to acquire the lock
-      if additionalDiskAttachment.isReadOnly {
-        continue
-      }
-
-      if try !FileLock(lockURL: additionalDiskAttachment.url).trylock() {
-        throw RuntimeError.DiskAlreadyInUse("disk \(additionalDiskAttachment.url.path) seems to be already in use, "
-          + "unmount it first in Finder")
-      }
-    }
-
     var serialPorts: [VZSerialPortConfiguration] = []
     if serial {
       let tty_fd = createPTY()
@@ -150,9 +223,11 @@ struct Run: AsyncParsableCommand {
     vm = try VM(
       vmDir: vmDir,
       network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalDiskAttachments: additionalDiskAttachments,
+      additionalStorageDevices: additionalDiskAttachments,
       directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
-      serialPorts: serialPorts
+      serialPorts: serialPorts,
+      suspendable: suspendable,
+      audio: !noAudio
     )
 
     let vncImpl: VNC? = try {
@@ -178,13 +253,32 @@ struct Run: AsyncParsableCommand {
     // configuration file, otherwise we will loose the lock.
     //
     // [1]: https://man.openbsd.org/fcntl
-    let lock = try PIDLock(lockURL: vmDir.configURL)
+    let lock = try vmDir.lock()
     if try !lock.trylock() {
       throw RuntimeError.VMAlreadyRunning("VM \"\(name)\" is already running!")
     }
 
+    // now VM state will return "running" so we can unlock
+    try storageLock.unlock()
+
     let task = Task {
       do {
+        var resume = false
+
+        #if arch(arm64)
+          if #available(macOS 14, *) {
+            if FileManager.default.fileExists(atPath: vmDir.stateURL.path) {
+              print("restoring VM state from a snapshot...")
+              try await vm!.virtualMachine.restoreMachineStateFrom(url: vmDir.stateURL)
+              try FileManager.default.removeItem(at: vmDir.stateURL)
+              resume = true
+              print("resuming VM...")
+            }
+          }
+        #endif
+
+        try await vm!.start(recovery: recovery, resume: resume)
+
         if let vncImpl = vncImpl {
           let vncURL = try await vncImpl.waitForURL()
 
@@ -196,7 +290,7 @@ struct Run: AsyncParsableCommand {
           }
         }
 
-        try await vm!.run(recovery)
+        try await vm!.run()
 
         if let vncImpl = vncImpl {
           try vncImpl.stop()
@@ -208,23 +302,60 @@ struct Run: AsyncParsableCommand {
         SentrySDK.capture(error: error)
         SentrySDK.flush(timeout: 2.seconds.timeInterval)
 
-        print(error)
+        fputs("\(error)\n", stderr)
 
         Foundation.exit(1)
       }
     }
 
+    // "tart stop" support
     let sigintSrc = DispatchSource.makeSignalSource(signal: SIGINT)
     sigintSrc.setEventHandler {
       task.cancel()
     }
     sigintSrc.activate()
 
+    // "tart suspend" / UI window closing support
+    signal(SIGUSR1, SIG_IGN)
+    let sigusr1Src = DispatchSource.makeSignalSource(signal: SIGUSR1)
+    sigusr1Src.setEventHandler {
+      Task {
+        do {
+          #if arch(arm64)
+            if #available(macOS 14, *) {
+              try vm!.configuration.validateSaveRestoreSupport()
+
+              print("pausing VM to take a snapshot...")
+              try await vm!.virtualMachine.pause()
+
+              print("creating a snapshot...")
+              try await vm!.virtualMachine.saveMachineStateTo(url: vmDir.stateURL)
+
+              print("snapshot created successfully! shutting down the VM...")
+
+              task.cancel()
+            } else {
+              print(RuntimeError.SuspendFailed("this functionality is only supported on macOS 14 (Sonoma) or newer"))
+
+              Foundation.exit(1)
+            }
+          #endif
+        } catch (let e) {
+          print(RuntimeError.SuspendFailed(e.localizedDescription))
+
+          Foundation.exit(1)
+        }
+      }
+    }
+    sigusr1Src.activate()
+
     let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
     if noGraphics || useVNCWithoutGraphics {
-      dispatchMain()
+      // enter the main even loop, without bringing up any UI,
+      // and just wait for the VM to exit.
+      NSApplication.shared.run()
     } else {
-      runUI()
+      runUI(suspendable, captureSystemKeys)
     }
   }
 
@@ -246,26 +377,33 @@ struct Run: AsyncParsableCommand {
     if netSoftnet {
       let config = try VMConfig.init(fromURL: vmDir.configURL)
 
-      return try Softnet(vmMACAddress: config.macAddress.string)
+      var extraArguments: [String] = []
+
+      if let netSoftnetAllow = netSoftnetAllow {
+        extraArguments += ["--allow", netSoftnetAllow]
+      }
+
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: extraArguments)
     }
 
-    if let netBridged = netBridged {
+    if netHost {
-      let matchingInterfaces = VZBridgedNetworkInterface.networkInterfaces.filter { interface in
+      let config = try VMConfig.init(fromURL: vmDir.configURL)
-        interface.identifier == netBridged || interface.localizedDisplayName == netBridged
+      return try Softnet(vmMACAddress: config.macAddress.string, extraArguments: ["--vm-net-type", "host"])
+    }
+
+    if netBridged.count > 0 {
+      func findBridgedInterface(_ name: String) throws -> VZBridgedNetworkInterface {
+        let interface = VZBridgedNetworkInterface.networkInterfaces.first { interface in
+          interface.identifier == name || interface.localizedDisplayName == name
+        }
+        if (interface == nil) {
+          throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
+            + "available interfaces: \(bridgeInterfaces())")
+        }
+        return interface!
       }
 
-      if matchingInterfaces.isEmpty {
+      return NetworkBridged(interfaces: try netBridged.map { try findBridgedInterface($0) })
-        let available = bridgeInterfaces().joined(separator: ", ")
-        throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
-          + "available interfaces: \(available)")
-      }
-
-      if matchingInterfaces.count > 1 {
-        throw ValidationError("more than one bridge interface matched \"\(netBridged)\", "
-          + "consider refining the search criteria")
-      }
-
-      return NetworkBridged(interface: matchingInterfaces.first!)
     }
 
     return nil
@@ -283,22 +421,43 @@ struct Run: AsyncParsableCommand {
     }
   }
 
-  func additionalDiskAttachments() throws -> [VZDiskImageStorageDeviceAttachment] {
+  func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
-    var result: [VZDiskImageStorageDeviceAttachment] = []
+    var result: [VZStorageDeviceConfiguration] = []
     let readOnlySuffix = ":ro"
     let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
 
     for rawDisk in expandedDiskPaths {
-      if rawDisk.hasSuffix(readOnlySuffix) {
+      let diskReadOnly = rawDisk.hasSuffix(readOnlySuffix)
-        result.append(try VZDiskImageStorageDeviceAttachment(
+      let diskPath = diskReadOnly ? String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count)) : rawDisk
-          url: URL(fileURLWithPath: String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count))),
+      let diskURL = URL(fileURLWithPath: diskPath)
-          readOnly: true
+
-        ))
+      // check if `diskPath` is a block device or a directory
+      if pathHasMode(diskPath, mode: S_IFBLK) || pathHasMode(diskPath, mode: S_IFDIR) {
+        print("Using block device\n")
+        guard #available(macOS 14, *) else {
+          throw UnsupportedOSError("attaching block devices", "are")
+        }
+        let fileHandle = FileHandle(forUpdatingAtPath: diskPath)
+        guard fileHandle != nil else {
+          if ProcessInfo.processInfo.userName != "root" {
+            throw RuntimeError.VMConfigurationError("need to run as root to work with block devices")
+          }
+          throw RuntimeError.VMConfigurationError("block device \(diskURL.url.path) seems to be already in use, unmount it first via 'diskutil unmount'")
+        }
+        let attachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: fileHandle!, readOnly: diskReadOnly, synchronizationMode: .full)
+        result.append(VZVirtioBlockDeviceConfiguration(attachment: attachment))
       } else {
-        result.append(try VZDiskImageStorageDeviceAttachment(
+        // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-          url: URL(fileURLWithPath: rawDisk),
+        // see https://github.com/cirruslabs/tart/issues/323 for more details.
-          readOnly: false
+        if try !diskReadOnly && !FileLock(lockURL: diskURL).trylock() {
-        ))
+          throw RuntimeError.DiskAlreadyInUse("disk \(diskURL.url.path) seems to be already in use, unmount it first in Finder")
+        }
+
+        let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
+          url: diskURL,
+          readOnly: diskReadOnly
+        )
+        result.append(VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
       }
     }
 
@@ -314,90 +473,84 @@ struct Run: AsyncParsableCommand {
       throw UnsupportedOSError("directory sharing", "is")
     }
 
-    struct DirectoryShare {
+    var allDirectoryShares: [DirectoryShare] = []
-      let name: String
-      let path: URL
-      let readOnly: Bool
-    }
-
-    var directoryShares: [DirectoryShare] = []
 
     for rawDir in dir {
-      let splits = rawDir.split(maxSplits: 2) { $0 == ":" }
+      allDirectoryShares.append(try DirectoryShare(parseFrom: rawDir))
-
-      if splits.count < 2 {
-        throw ValidationError("invalid --dir syntax: should at least include name and path, colon-separated")
-      }
-
-      var readOnly: Bool = false
-
-      if splits.count == 3 {
-        if splits[2] == "ro" {
-          readOnly = true
-        } else {
-          throw ValidationError("invalid --dir syntax: optional read-only specifier can only be \"ro\"")
-        }
-      }
-
-      let (name, path) = (String(splits[0]), String(splits[1]))
-
-      directoryShares.append(DirectoryShare(
-        name: name,
-        path: URL(fileURLWithPath: NSString(string: path).expandingTildeInPath),
-        readOnly: readOnly)
-      )
     }
 
-    var directories: [String : VZSharedDirectory] = Dictionary()
+    return try Dictionary(grouping: allDirectoryShares, by: {$0.mountTag}).map { mountTag, directoryShares in
-    directoryShares.forEach { directories[$0.name] = VZSharedDirectory(url: $0.path, readOnly: $0.readOnly) }
+      let sharingDevice = VZVirtioFileSystemDeviceConfiguration(tag: mountTag)
 
-    let automountTag = VZVirtioFileSystemDeviceConfiguration.macOSGuestAutomountTag
+      var allNamedShares = true
-    let sharingDevice = VZVirtioFileSystemDeviceConfiguration(tag: automountTag)
+      for directoryShare in directoryShares {
-    sharingDevice.share = VZMultipleDirectoryShare(directories: directories)
+        if directoryShare.name == nil {
+          allNamedShares = false
+        }
+      }
+      if directoryShares.count == 1 && directoryShares.first!.name == nil {
+        let directoryShare = directoryShares.first!
+        let singleDirectoryShare = VZSingleDirectoryShare(directory: try directoryShare.createConfiguration())
+        sharingDevice.share = singleDirectoryShare
+      } else if !allNamedShares {
+        throw ValidationError("invalid --dir syntax: for multiple directory shares each one of them should be named")
+      } else {
+        var directories: [String : VZSharedDirectory] = Dictionary()
+        try directoryShares.forEach { directories[$0.name!] = try $0.createConfiguration() }
+        sharingDevice.share = VZMultipleDirectoryShare(directories: directories)
+      }
 
-    return [sharingDevice]
+      return sharingDevice
+    }
   }
 
   private func rosettaDirectoryShare() throws -> [VZDirectorySharingDeviceConfiguration] {
     guard let rosettaTag = rosettaTag else {
       return []
     }
+    #if arch(arm64)
+      guard #available(macOS 13, *) else {
+        throw UnsupportedOSError("Rosetta directory share", "is")
+      }
 
-    guard #available(macOS 13, *) else {
+      switch VZLinuxRosettaDirectoryShare.availability {
-      throw UnsupportedOSError("Rosetta directory share", "is")
+      case .notInstalled:
-    }
+        throw UnsupportedOSError("Rosetta directory share", "is", "that have Rosetta installed")
+      case .notSupported:
+        throw UnsupportedOSError("Rosetta directory share", "is", "running Apple silicon")
+      default:
+        break
+      }
 
-    switch VZLinuxRosettaDirectoryShare.availability {
+      try VZVirtioFileSystemDeviceConfiguration.validateTag(rosettaTag)
-    case .notInstalled:
+      let device = VZVirtioFileSystemDeviceConfiguration(tag: rosettaTag)
-      throw UnsupportedOSError("Rosetta directory share", "is", "that have Rosetta installed")
+      device.share = try VZLinuxRosettaDirectoryShare()
-    case .notSupported:
-      throw UnsupportedOSError("Rosetta directory share", "is", "running Apple silicon")
-    default:
-      break
-    }
 
-    try VZVirtioFileSystemDeviceConfiguration.validateTag(rosettaTag)
+      return [device]
-    let device = VZVirtioFileSystemDeviceConfiguration(tag: rosettaTag)
+    #elseif arch(x86_64)
-    device.share = try VZLinuxRosettaDirectoryShare()
+      // there is no Rosetta on Intel
-
+      return []
-    return [device]
+    #endif
   }
 
-  private func runUI() {
+  private func runUI(_ suspendable: Bool, _ captureSystemKeys: Bool) {
     let nsApp = NSApplication.shared
     nsApp.setActivationPolicy(.regular)
     nsApp.activate(ignoringOtherApps: true)
 
-    nsApp.applicationIconImage = NSImage(data: AppIconData)
-
     struct MainApp: App {
+      static var disappearSignal: Int32 = SIGINT
+      static var capturesSystemKeys: Bool = false
+
+      @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
+
       var body: some Scene {
         WindowGroup(vm!.name) {
           Group {
-            VMView(vm: vm!).onAppear {
+            VMView(vm: vm!, capturesSystemKeys: MainApp.capturesSystemKeys).onAppear {
               NSWindow.allowsAutomaticWindowTabbing = false
             }.onDisappear {
-              let ret = kill(getpid(), SIGINT)
+              let ret = kill(getpid(), MainApp.disappearSignal)
               if ret != 0 {
                 // Fallback to the old termination method that doesn't
                 // propagate the cancellation to Task's in case graceful
@@ -405,7 +558,14 @@ struct Run: AsyncParsableCommand {
                 NSApplication.shared.terminate(self)
               }
             }
-          }.frame(width: CGFloat(vm!.config.display.width), height: CGFloat(vm!.config.display.height))
+          }.frame(
+            minWidth: CGFloat(vm!.config.display.width),
+            idealWidth: CGFloat(vm!.config.display.width),
+            maxWidth: .infinity,
+            minHeight: CGFloat(vm!.config.display.height),
+            idealHeight: CGFloat(vm!.config.display.height),
+            maxHeight: .infinity
+          )
         }.commands {
           // Remove some standard menu options
           CommandGroup(replacing: .help, addition: {})
@@ -415,23 +575,69 @@ struct Run: AsyncParsableCommand {
           CommandGroup(replacing: .undoRedo, addition: {})
           CommandGroup(replacing: .windowSize, addition: {})
           // Replace some standard menu options
-          CommandGroup(replacing: .appInfo) { AboutTart() }
+          CommandGroup(replacing: .appInfo) { AboutTart(config: vm!.config) }
+          CommandMenu("Control") {
+            Button("Start") {
+              Task { try await vm!.virtualMachine.start() }
+            }
+            Button("Stop") {
+              Task { try await vm!.virtualMachine.stop() }
+            }
+            Button("Request Stop") {
+              Task { try vm!.virtualMachine.requestStop() }
+            }
+            if #available(macOS 14, *) {
+              Button("Suspend") {
+                kill(getpid(), SIGUSR1)
+              }
+            }
+          }
         }
       }
     }
 
+    MainApp.disappearSignal = suspendable ? SIGUSR1 : SIGINT
+    MainApp.capturesSystemKeys = captureSystemKeys
     MainApp.main()
   }
 }
 
+// The only way to fully remove Edit menu item.
+class MinimalMenuAppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
+  let indexOfEditMenu = 2
+
+  func applicationDidFinishLaunching(_ : Notification) {
+    NSApplication.shared.mainMenu?.removeItem(at: indexOfEditMenu)
+  }
+}
+
 struct AboutTart: View {
+  var credits: NSAttributedString
+
+  init(config: VMConfig) {
+    let mutableAttrStr = NSMutableAttributedString()
+    let style = NSMutableParagraphStyle()
+    style.alignment = NSTextAlignment.center
+    let attrCenter: [NSAttributedString.Key : Any] = [
+      .paragraphStyle: style,
+    ]
+    mutableAttrStr.append(NSAttributedString(string: "CPU: \(config.cpuCount) cores\n", attributes: attrCenter))
+    mutableAttrStr.append(NSAttributedString(string: "Memory: \(config.memorySize / 1024 / 1024) MB\n", attributes: attrCenter))
+    mutableAttrStr.append(NSAttributedString(string: "Display: \(config.display.description)\n", attributes: attrCenter))
+    mutableAttrStr.append(NSAttributedString(string: "https://github.com/cirruslabs/tart", attributes: [
+      .paragraphStyle: style,
+      .link : "https://github.com/cirruslabs/tart"
+    ]))
+    credits = mutableAttrStr
+  }
+
   var body: some View {
     Button("About Tart") {
       NSApplication.shared.orderFrontStandardAboutPanel(options: [
         NSApplication.AboutPanelOptionKey.applicationIcon: NSApplication.shared.applicationIconImage as Any,
         NSApplication.AboutPanelOptionKey.applicationName: "Tart",
         NSApplication.AboutPanelOptionKey.applicationVersion: CI.version,
-        NSApplication.AboutPanelOptionKey.credits: try! NSAttributedString(markdown: "https://github.com/cirruslabs/tart"),
+        NSApplication.AboutPanelOptionKey.credits: credits,
       ])
     }
   }
@@ -441,10 +647,22 @@ struct VMView: NSViewRepresentable {
   typealias NSViewType = VZVirtualMachineView
 
   @ObservedObject var vm: VM
+  var capturesSystemKeys: Bool
 
   func makeNSView(context: Context) -> NSViewType {
     let machineView = VZVirtualMachineView()
-    machineView.capturesSystemKeys = true
+
+    machineView.capturesSystemKeys = capturesSystemKeys
+
+    // Enable automatic display reconfiguration
+    // for guests that support it
+    //
+    // This is disabled for Linux because of poor HiDPI
+    // support, which manifests in fonts being too small
+    if #available(macOS 14.0, *), vm.config.os != .linux {
+      machineView.automaticallyReconfiguresDisplay = true
+    }
+
     return machineView
   }
 
@@ -452,3 +670,152 @@ struct VMView: NSViewRepresentable {
     nsView.virtualMachine = vm.virtualMachine
   }
 }
+
+struct DirectoryShare {
+  let name: String?
+  let path: URL
+  let readOnly: Bool
+  let mountTag: String
+
+  init(parseFrom: String) throws {
+    var parseFrom = parseFrom
+
+    // Consume options
+    (self.readOnly, self.mountTag, parseFrom) = Self.parseOptions(parseFrom)
+
+    // Special case for URLs
+    if parseFrom.hasPrefix("http:") || parseFrom.hasPrefix("https:") {
+      self.name = nil
+      self.path = URL(string: parseFrom)!
+
+      return
+    }
+
+    let arguments = parseFrom.split(separator: ":", maxSplits: 1)
+
+    if arguments.count == 2 {
+      self.name = String(arguments[0])
+      self.path = String(arguments[1]).toRemoteOrLocalURL()
+    } else {
+      self.name = nil
+      self.path = String(arguments[0]).toRemoteOrLocalURL()
+    }
+  }
+
+  static func parseOptions(_ parseFrom: String) -> (Bool, String, String) {
+    var arguments = parseFrom.split(separator: ":")
+    let options = arguments.last!.split(separator: ",")
+
+    var readOnly: Bool = false
+    var mountTag: String = VZVirtioFileSystemDeviceConfiguration.macOSGuestAutomountTag
+
+    var found: Bool = false
+
+    for option in options {
+      switch true {
+      case option == "ro":
+        readOnly = true
+        found = true
+      case option.hasPrefix("tag="):
+        mountTag = String(option.dropFirst(4))
+        found = true
+      default:
+        continue
+      }
+    }
+
+    if found {
+      arguments.removeLast()
+    }
+
+    return (readOnly, mountTag, arguments.joined(separator: ":"))
+  }
+
+  func createConfiguration() throws -> VZSharedDirectory {
+    if (path.isFileURL) {
+      return VZSharedDirectory(url: path, readOnly: readOnly)
+    }
+
+    let urlCache = URLCache(memoryCapacity: 0, diskCapacity: 1 * 1024 * 1024 * 1024)
+
+    let archiveRequest = URLRequest(url: path, cachePolicy: .returnCacheDataElseLoad)
+    var response: CachedURLResponse? = urlCache.cachedResponse(for: archiveRequest)
+    if (response == nil || response?.data.isEmpty == true) {
+      print("Downloading \(path)...")
+      // download and unarchive remote directories if needed here
+      // use old school API to prevent deadlocks since we are running via MainActor
+      let downloadSemaphore = DispatchSemaphore(value: 0)
+      Task {
+        do {
+          let (archiveData, archiveResponse) = try await URLSession.shared.data(for: archiveRequest)
+          if archiveData.isEmpty {
+            print("Remote archive is empty!")
+          } else {
+            urlCache.storeCachedResponse(CachedURLResponse(response: archiveResponse, data: archiveData, storagePolicy: .allowed), for: archiveRequest)
+            print("Cached for future invocations!")
+          }
+        } catch {
+          print("Download failed: \(error)")
+        }
+        downloadSemaphore.signal()
+      }
+      downloadSemaphore.wait()
+      response = urlCache.cachedResponse(for: archiveRequest)
+    } else {
+      print("Using cached archive for \(path)...")
+    }
+
+    if (response == nil) {
+      throw ValidationError("Failed to fetch a remote archive!")
+    }
+
+    let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".volume")
+    try FileManager.default.createDirectory(atPath: temporaryLocation.path, withIntermediateDirectories: true)
+    let lock = try FileLock(lockURL: temporaryLocation)
+    try lock.lock()
+
+    guard let executableURL = resolveBinaryPath("tar") else {
+      throw ValidationError("tar not found in PATH")
+    }
+
+    let process = Process.init()
+    process.executableURL = executableURL
+    process.currentDirectoryURL = temporaryLocation
+    process.arguments = ["-xz"]
+
+    let inPipe = Pipe()
+    process.standardInput = inPipe
+    process.launch()
+
+    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.close()
+    process.waitUntilExit()
+
+    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
+      throw ValidationError("Unarchiving failed!")
+    }
+
+    print("Unarchived into a temporary directory!")
+
+    return VZSharedDirectory(url: temporaryLocation, readOnly: readOnly)
+  }
+}
+
+extension String {
+  func toRemoteOrLocalURL() -> URL {
+    if (starts(with: "https://") || starts(with: "https://")) {
+      URL(string: self)!
+    } else {
+      URL(fileURLWithPath: NSString(string: self).expandingTildeInPath)
+    }
+  }
+}
+
+func pathHasMode(_ path: String, mode: mode_t) -> Bool {
+  var st = stat()
+  let statRes = stat(path, &st)
+  guard statRes != -1 else {
+    return false
+  }
+  return (Int32(st.st_mode) & Int32(mode)) == Int32(mode)
+}

Sources/tart/Commands/Set.swift

@@ -16,7 +16,18 @@ struct Set: AsyncParsableCommand {
   @Option(help: "VM display resolution in a format of <width>x<height>. For example, 1200x800")
   var display: VMDisplayConfig?
 
-  @Option(help: .hidden)
+  @Option(help: ArgumentHelp("Resize the VMs disk to the specified size in GB (note that the disk size can only be increased to avoid losing data)",
+                             discussion: """
+                             Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images
+                             have the \"cloud-initramfs-growroot\" package installed that runs on boot) and on the rest of the
+                             distributions by running the \"growpart\" or \"resize2fs\" commands.
+
+                             For macOS, however, things are a bit more complicated: you need to remove the recovery partition
+                             first and then run various \"diskutil\" commands, see Tart's packer plugin source code for more
+                             details[1].
+
+                             [1]: https://github.com/cirruslabs/packer-plugin-tart/blob/main/builder/tart/step_disk_resize.go
+                             """))
   var diskSize: UInt16?
 
   func run() async throws {

Sources/tart/Commands/Stop.swift

@@ -14,12 +14,27 @@ struct Stop: AsyncParsableCommand {
 
   func run() async throws {
     let vmDir = try VMStorageLocal().open(name)
-    let lock = try PIDLock(lockURL: vmDir.configURL)
+    switch try vmDir.state() {
+    case "suspended":
+      try stopSuspended(vmDir)
+    case "running":
+      try await stopRunning(vmDir)
+    default:
+      return
+    }
+  }
+
+  func stopSuspended(_ vmDir: VMDirectory) throws {
+    try? FileManager.default.removeItem(at: vmDir.stateURL)
+  }
+
+  func stopRunning(_ vmDir: VMDirectory) async throws {
+    let lock = try vmDir.lock()
 
     // Find the VM's PID
     var pid = try lock.pid()
     if pid == 0 {
-      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
+      throw RuntimeError.VMNotRunning(name)
     }
 
     // Try to gracefully terminate the VM

Sources/tart/Commands/Suspend.swift

@@ -0,0 +1,28 @@
+import ArgumentParser
+import Foundation
+import System
+import SwiftDate
+
+struct Suspend: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(commandName: "suspend", abstract: "Suspend a VM")
+
+  @Argument(help: "VM name")
+  var name: String
+
+  func run() async throws {
+    let vmDir = try VMStorageLocal().open(name)
+    let lock = try vmDir.lock()
+
+    // Find the VM's PID
+    var pid = try lock.pid()
+    if pid == 0 {
+      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
+    }
+
+    // Tell the "tart run" process to suspend the VM
+    let ret = kill(pid, SIGUSR1)
+    if ret != 0 {
+      throw RuntimeError.SuspendFailed("failed to send SIGUSR1 signal to the \"tart run\" process running VM \"\(name)\"")
+    }
+  }
+}

Sources/tart/Credentials/DockerConfigCredentialsProvider.swift

@@ -11,7 +11,7 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
     if let credentialsFromAuth = config.auths?[host]?.decodeCredentials() {
       return credentialsFromAuth
     }
-    if let helperProgram = config.credHelpers?[host] {
+    if let helperProgram = try config.findCredHelper(host: host) {
       return try executeHelper(binaryName: "docker-credential-\(helperProgram)", host: host)
     }
 
@@ -41,13 +41,18 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 
     process.waitUntilExit()
 
+    let outputData = try outPipe.fileHandleForReading.readToEnd()
     if !(process.terminationReason == .exit && process.terminationStatus == 0) {
+      if let outputData = outputData {
+        print(String(decoding: outputData, as: UTF8.self))
+      }
       throw CredentialsProviderError.Failed(message: "Docker helper failed!")
     }
+    if outputData == nil || outputData?.count == 0 {
+      throw CredentialsProviderError.Failed(message: "Docker helper output is empty!")
+    }
 
-    let getOutput = try JSONDecoder().decode(
+    let getOutput = try JSONDecoder().decode(DockerGetOutput.self, from: outputData!)
-      DockerGetOutput.self, from: outPipe.fileHandleForReading.readDataToEndOfFile()
-    )
     return (getOutput.Username, getOutput.Secret)
   }
 
@@ -59,6 +64,26 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 struct DockerConfig: Codable {
   var auths: Dictionary<String, DockerAuthConfig>? = Dictionary()
   var credHelpers: Dictionary<String, String>? = Dictionary()
+
+  func findCredHelper(host: String) throws -> String? {
+    // Tart supports wildcards in credHelpers
+    // Similar to what is requested from Docker: https://github.com/docker/cli/issues/2928
+
+    guard let credHelpers else {
+      return nil
+    }
+
+    for (hostPattern, helperProgram) in credHelpers {
+      if (hostPattern == host) {
+        return helperProgram
+      }
+      let compiledPattern = try? Regex(hostPattern)
+      if (try compiledPattern?.wholeMatch(in: host) != nil) {
+        return helperProgram
+      }
+    }
+    return nil
+  }
 }
 
 struct DockerAuthConfig: Codable {

Sources/tart/Credentials/EnvironmentCredentialsProvider.swift

@@ -2,6 +2,11 @@ import Foundation
 
 class EnvironmentCredentialsProvider: CredentialsProvider {
   func retrieve(host: String) throws -> (String, String)? {
+    if let tartRegistryHostname = ProcessInfo.processInfo.environment["TART_REGISTRY_HOSTNAME"],
+       tartRegistryHostname != host {
+      return nil
+    }
+
     let username = ProcessInfo.processInfo.environment["TART_REGISTRY_USERNAME"]
     let password = ProcessInfo.processInfo.environment["TART_REGISTRY_PASSWORD"]
     if let username = username, let password = password {

Sources/tart/Credentials/KeychainCredentialsProvider.swift

@@ -61,6 +61,24 @@ class KeychainCredentialsProvider: CredentialsProvider {
       throw CredentialsProviderError.Failed(message: "Keychain failed to find item: \(status.explanation())")
     }
   }
+
+  func remove(host: String) throws {
+    let query: [String: Any] = [kSecClass as String: kSecClassInternetPassword,
+                                kSecAttrServer as String: host,
+                                kSecAttrLabel as String: "Tart Credentials",
+    ]
+
+    let status = SecItemDelete(query as CFDictionary)
+
+    switch status {
+    case errSecSuccess:
+      return
+    case errSecItemNotFound:
+      return
+    default:
+      throw CredentialsProviderError.Failed(message: "Failed to remove Keychain item(s): \(status.explanation())")
+    }
+  }
 }
 
 extension OSStatus {

Sources/tart/Credentials/StdinCredentials.swift

@@ -13,7 +13,7 @@ class StdinCredentials {
     return (user, password)
   }
 
-  private static func readStdinCredential(name: String, prompt: String, maxCharacters: Int = 255, isSensitive: Bool) throws -> String {
+  private static func readStdinCredential(name: String, prompt: String, maxCharacters: Int = 1024, isSensitive: Bool) throws -> String {
     var buf = [CChar](repeating: 0, count: maxCharacters + 1 /* sentinel */ + 1 /* NUL */)
     guard let rawCredential = readpassphrase(prompt, &buf, buf.count, isSensitive ? RPP_ECHO_OFF : RPP_ECHO_ON) else {
       throw StdinCredentialsError.CredentialRequired(which: name)

Sources/tart/DeviceInfo/DeviceInfo.swift

@@ -0,0 +1,37 @@
+import Foundation
+import Sysctl
+
+class DeviceInfo {
+  private static var osMemoized: String? = nil
+  private static var modelMemoized: String? = nil
+
+  static var os: String {
+    if let os = osMemoized {
+      return os
+    }
+
+    osMemoized = getOS()
+
+    return osMemoized!
+  }
+
+  static var model: String {
+    if let model = modelMemoized {
+      return model
+    }
+
+    modelMemoized = getModel()
+
+    return modelMemoized!
+  }
+
+  private static func getOS() -> String {
+    let osVersion = ProcessInfo.processInfo.operatingSystemVersion
+
+    return "macOS \(osVersion.majorVersion).\(osVersion.minorVersion).\(osVersion.patchVersion)"
+  }
+
+  private static func getModel() -> String {
+    return SystemControl().hardware.model
+  }
+}

Sources/tart/Fetcher.swift

@@ -35,12 +35,12 @@ class Fetcher {
     //
     // This keeps a working reference to that file, yet we don't
     // have to deal with the cleanup any more.
-    let fh = try FileHandle(forReadingFrom: fileURL)
+    let mappedFile = try Data(contentsOf: fileURL, options: [.alwaysMapped])
     try FileManager.default.removeItem(at: fileURL)
 
     Task {
-      while let data = try fh.read(upToCount: 64 * 1024 * 1024) {
+      for chunk in (0 ..< mappedFile.count).chunks(ofCount: 64 * 1024 * 1024) {
-        await dataCh.send(data)
+        await dataCh.send(mappedFile.subdata(in: chunk))
       }
 
       dataCh.finish()

Sources/tart/Formatter/Format.swift

@@ -26,10 +26,16 @@ enum Format: String, ExpressibleByArgument, CaseIterable {
       }
       let table = TextTable<T> { (item: T) in
         let mirroredObject = Mirror(reflecting: item)
-        return mirroredObject.children.enumerated().map { (_, element) in
+        return mirroredObject.children.enumerated()
-          let fieldName = element.label!
+          .filter {(_, element) in
-          return Column(title: fieldName, value: element.value)
+            // Deprecate the "Running" field: only make it available
-        }
+            // from JSON for backwards-compatibility
+            element.label! != "Running"
+          }
+          .map { (_, element) in
+            let fieldName = element.label!
+            return Column(title: fieldName, value: element.value)
+          }
       }
       return table.string(for: data, style: Style.plain)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
     case .json:

Sources/tart/MACAddressResolver/Lease.swift

@@ -1,13 +1,17 @@
+import Foundation
 import Network
+import SwiftRadix
 
 struct Lease {
   var mac: MACAddress
   var ip: IPv4Address
+  var expiresAt: Date
 
   init?(fromRawLease: [String : String]) {
     // Retrieve the required fields
     guard let hwAddress = fromRawLease["hw_address"] else { return nil }
     guard let ipAddress = fromRawLease["ip_address"] else { return nil }
+    guard let lease = fromRawLease["lease"] else { return nil }
 
     // Parse MAC address
     let hwAddressSplits = hwAddress.split(separator: ",")
@@ -26,7 +30,13 @@ struct Lease {
       return nil
     }
 
+    // Parse expiration timestamp
+    guard let leaseTimestamp = lease.hex?.value else {
+      return nil
+    }
+
     self.ip = ip
     self.mac = mac
+    self.expiresAt = Date(timeIntervalSince1970: TimeInterval(leaseTimestamp))
   }
 }

Sources/tart/MACAddressResolver/Leases.swift

@@ -37,13 +37,18 @@ class Leases {
   }
 
   init(_ fromString: String) throws {
-    var leases: [MACAddress : Lease] = Dictionary()
+    let leases = try Self.retrieveRawLeases(fromString).compactMap({ rawLease in
+      Lease(fromRawLease: rawLease)
+    }).filter({ lease in
+      lease.expiresAt.isInFuture
+    }).map({ lease in
+      (lease.mac, lease)
+    })
 
-    for lease in try Self.retrieveRawLeases(fromString).compactMap({ Lease(fromRawLease: $0) }) {
+    self.leases = Dictionary(leases) { (left, right) in
-      leases[lease.mac] = lease
+      // When duplicate lease is found, prefer a newer lease over the older one
+      (left.expiresAt > right.expiresAt) ? left : right
     }
-
-    self.leases = leases
   }
 
   /// Parse leases from the host cache similarly to the PLCache_read() function found in Apple's Open Source releases.
@@ -107,7 +112,7 @@ class Leases {
     return rawLeases
   }
 
-  func ResolveMACAddress(macAddress: MACAddress) throws -> IPv4Address? {
+  func ResolveMACAddress(macAddress: MACAddress) -> IPv4Address? {
     leases[macAddress]?.ip
   }
 }

Sources/tart/Network/Network.swift

@@ -1,7 +1,7 @@
 import Virtualization
 
 protocol Network {
-  func attachment() -> VZNetworkDeviceAttachment
+  func attachments() -> [VZNetworkDeviceAttachment]
   func run(_ sema: DispatchSemaphore) throws
   func stop() async throws
 }

Sources/tart/Network/NetworkBridged.swift

@@ -2,14 +2,14 @@ import Foundation
 import Virtualization
 
 class NetworkBridged: Network {
-  let interface: VZBridgedNetworkInterface
+  let interfaces: [VZBridgedNetworkInterface]
 
-  init(interface: VZBridgedNetworkInterface) {
+  init(interfaces: [VZBridgedNetworkInterface]) {
-    self.interface = interface
+    self.interfaces = interfaces
   }
 
-  func attachment() -> VZNetworkDeviceAttachment {
+  func attachments() -> [VZNetworkDeviceAttachment] {
-    VZBridgedNetworkDeviceAttachment(interface: interface)
+    interfaces.map { VZBridgedNetworkDeviceAttachment(interface: $0) }
   }
 
   func run(_ sema: DispatchSemaphore) throws {

Sources/tart/Network/NetworkShared.swift

@@ -2,8 +2,8 @@ import Foundation
 import Virtualization
 
 class NetworkShared: Network {
-  func attachment() -> VZNetworkDeviceAttachment {
+  func attachments() -> [VZNetworkDeviceAttachment] {
-    VZNATNetworkDeviceAttachment()
+    [VZNATNetworkDeviceAttachment()]
   }
 
   func run(_ sema: DispatchSemaphore) throws {

Sources/tart/Network/Softnet.swift

@@ -15,7 +15,7 @@ class Softnet: Network {
 
   let vmFD: Int32
 
-  init(vmMACAddress: String) throws {
+  init(vmMACAddress: String, extraArguments: [String] = []) throws {
     let fds = UnsafeMutablePointer<Int32>.allocate(capacity: MemoryLayout<Int>.stride * 2)
 
     let ret = socketpair(AF_UNIX, SOCK_DGRAM, 0, fds)
@@ -30,7 +30,7 @@ class Softnet: Network {
     try setSocketBuffers(softnetFD, 1 * 1024 * 1024);
 
     process.executableURL = try Self.softnetExecutableURL()
-    process.arguments = ["--vm-fd", String(STDIN_FILENO), "--vm-mac-address", vmMACAddress]
+    process.arguments = ["--vm-fd", String(STDIN_FILENO), "--vm-mac-address", vmMACAddress] + extraArguments
     process.standardInput = FileHandle(fileDescriptor: softnetFD, closeOnDealloc: false)
   }
 
@@ -92,9 +92,9 @@ class Softnet: Network {
     }
   }
 
-  func attachment() -> VZNetworkDeviceAttachment {
+  func attachments() -> [VZNetworkDeviceAttachment] {
     let fh = FileHandle.init(fileDescriptor: vmFD)
-    return VZFileHandleNetworkDeviceAttachment(fileHandle: fh)
+    return [VZFileHandleNetworkDeviceAttachment(fileHandle: fh)]
   }
 
   static func configureSUIDBitIfNeeded() throws {

Sources/tart/OCI/AuthenticationKeeper.swift

@@ -0,0 +1,25 @@
+import Foundation
+
+actor AuthenticationKeeper {
+  var authentication: Authentication? = nil
+
+  func set(_ authentication: Authentication) {
+    self.authentication = authentication
+  }
+
+  func header() -> (String, String)? {
+    if let authentication = authentication {
+      // Do not suggest any headers if the
+      // authentication token has expired
+      if !authentication.isValid() {
+        return nil
+      }
+
+      return authentication.header()
+    }
+
+    // Do not suggest any headers if the
+    // authentication token is not set
+    return nil
+  }
+}

Sources/tart/OCI/Digest.swift

@@ -1,6 +1,11 @@
 import Foundation
 import CryptoKit
 
+enum DigestError: Error {
+  case InvalidOffset
+  case InvalidSize
+}
+
 class Digest {
   var hash: SHA256 = SHA256()
 
@@ -15,6 +20,37 @@ class Digest {
   static func hash(_ data: Data) -> String {
     SHA256.hash(data: data).hexdigest()
   }
+
+  static func hash(_ url: URL) throws -> String {
+    hash(try Data(contentsOf: url))
+  }
+
+  static func hash(_ url: URL, offset: UInt64, size: UInt64) throws -> String {
+    // Sanity check
+    let fhSanity = try FileHandle(forReadingFrom: url)
+    try fhSanity.seekToEnd()
+    let fileSize = try fhSanity.offset()
+    try fhSanity.close()
+
+    if offset > fileSize {
+      throw DigestError.InvalidOffset
+    }
+
+    if (offset + size) > fileSize {
+      throw DigestError.InvalidSize
+    }
+
+    // Read a chunk of size ``size`` at offset ``offset``
+    // and calculate it's digest
+    let fh = try FileHandle(forReadingFrom: url)
+    defer { try! fh.close() }
+
+    try fh.seek(toOffset: offset)
+
+    let data = try fh.read(upToCount: Int(size))!
+
+    return hash(data)
+  }
 }
 
 extension SHA256.Digest {

Sources/tart/OCI/Layerizer/Disk.swift

@@ -0,0 +1,6 @@
+import Foundation
+
+protocol Disk {
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer]
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws
+}

Sources/tart/OCI/Layerizer/DiskV1.swift

@@ -0,0 +1,75 @@
+import Foundation
+import Compression
+
+class DiskV1: Disk {
+  private static let bufferSizeBytes = 4 * 1024 * 1024
+  private static let layerLimitBytes = 500 * 1000 * 1000
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [OCIManifestLayer] = []
+
+    // Open the disk file
+    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+    var mappedDiskReadOffset = 0
+
+    // Compress the disk file as a single stream
+    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
+      // Determine the size of the next chunk
+      let bytesRead = min(length, mappedDisk.count - mappedDiskReadOffset)
+
+      // Read the next uncompressed chunk
+      let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
+
+      // Advance the offset
+      mappedDiskReadOffset += bytesRead
+
+      // Provide the uncompressed chunk to the compressing filter
+      return data
+    }
+
+    // Cut the compressed stream into layers, each equal exactly ``Self.layerLimitBytes`` bytes,
+    // except for the last one, which may be smaller
+    while let compressedData = try compressingFilter.readData(ofLength: Self.layerLimitBytes) {
+      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+
+      pushedLayers.append(OCIManifestLayer(
+        mediaType: diskV1MediaType,
+        size: compressedData.count,
+        digest: layerDigest
+      ))
+
+      // Update progress using an absolute value
+      progress.completedUnitCount = Int64(mappedDiskReadOffset)
+    }
+
+    return pushedLayers
+  }
+
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+      throw OCIError.FailedToCreateVmFile
+    }
+
+    // Open the disk file
+    let disk = try FileHandle(forWritingTo: diskURL)
+    defer { try! disk.close() }
+
+    // Decompress the layers onto the disk in a single stream
+    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
+      if let data = data {
+        disk.write(data)
+      }
+    }
+
+    for diskLayer in diskLayers {
+      try await registry.pullBlob(diskLayer.digest) { data in
+        try filter.write(data)
+
+        // Update the progress
+        progress.completedUnitCount += Int64(data.count)
+      }
+    }
+
+    try filter.finalize()
+  }
+}

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -0,0 +1,188 @@
+import Foundation
+import Compression
+
+class DiskV2: Disk {
+  private static let bufferSizeBytes = 4 * 1024 * 1024
+  private static let layerLimitBytes = 500 * 1000 * 1000
+  private static let holeGranularityBytes = 64 * 1024
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [OCIManifestLayer] = []
+
+    // Open the disk file
+    var mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+
+    // Compress the disk file as multiple individually decompressible streams,
+    // each equal ``Self.layerLimitBytes`` bytes or slightly larger due to the
+    // internal compressor's buffer
+    var offset: UInt64 = 0
+
+    while let (compressedData, uncompressedSize, uncompressedDigest) = try compressNextLayerOfLimitBytesOrMore(mappedDisk: mappedDisk, offset: offset) {
+      offset += uncompressedSize
+
+      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+
+      pushedLayers.append(OCIManifestLayer(
+        mediaType: diskV2MediaType,
+        size: compressedData.count,
+        digest: layerDigest,
+        uncompressedSize: uncompressedSize,
+        uncompressedContentDigest: uncompressedDigest
+      ))
+
+      // Update progress using a relative value
+      progress.completedUnitCount += Int64(uncompressedSize)
+    }
+
+    return pushedLayers
+  }
+
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+    // Support resumable pulls
+    let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
+
+    if !pullResumed && !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+      throw OCIError.FailedToCreateVmFile
+    }
+
+    // Calculate the uncompressed disk size
+    var uncompressedDiskSize: UInt64 = 0
+
+    for layer in diskLayers {
+      guard let uncompressedLayerSize = layer.uncompressedSize() else {
+        throw OCIError.LayerIsMissingUncompressedSizeAnnotation
+      }
+
+      uncompressedDiskSize += uncompressedLayerSize
+    }
+
+    // Truncate the target disk file so that it will be able
+    // to accomodate the uncompressed disk size
+    let disk = try FileHandle(forWritingTo: diskURL)
+    try disk.truncate(atOffset: uncompressedDiskSize)
+    try disk.close()
+
+    // Concurrently fetch and decompress layers
+    try await withThrowingTaskGroup(of: Void.self) { group in
+      var globalDiskWritingOffset: UInt64 = 0
+
+      for (index, diskLayer) in diskLayers.enumerated() {
+        // Respect the concurrency limit
+        if index >= concurrency {
+          try await group.next()
+        }
+
+        // Retrieve layer annotations
+        guard let uncompressedLayerSize = diskLayer.uncompressedSize() else {
+          throw OCIError.LayerIsMissingUncompressedSizeAnnotation
+        }
+        guard let uncompressedLayerContentDigest = diskLayer.uncompressedContentDigest() else {
+          throw OCIError.LayerIsMissingUncompressedDigestAnnotation
+        }
+
+        // Capture the current disk writing offset
+        let diskWritingOffset = globalDiskWritingOffset
+
+        // Launch a fetching and decompression task
+        group.addTask {
+          // No need to fetch and decompress anything if we've already done so
+          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+            // Update the progress
+            progress.completedUnitCount += Int64(diskLayer.size)
+
+            return
+          }
+
+          // Open the disk file
+          let disk = try FileHandle(forWritingTo: diskURL)
+
+          // A zero chunk for faster than byte-by-byte comparisons
+          //
+          // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
+          // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
+          //
+          // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
+          // +--------------------------------------+---------------------------------------------------+
+          // | Operation                            | time(1) result                                    |
+          // +--------------------------------------+---------------------------------------------------+
+          // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
+          // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
+          // +--------------------------------------+---------------------------------------------------+
+          let zeroChunk = Data(count: holeGranularityBytes)
+          var diskWritingOffset = diskWritingOffset
+
+          // Pull and decompress a single layer into the specific offset on disk
+          let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
+            guard let data = data else {
+              return
+            }
+
+            for chunk in data.chunks(ofCount: holeGranularityBytes) {
+              // Only write chunks that are not zero
+              if chunk != zeroChunk {
+                try disk.seek(toOffset: diskWritingOffset)
+                disk.write(chunk)
+              }
+
+              diskWritingOffset += UInt64(chunk.count)
+            }
+          }
+
+          try await registry.pullBlob(diskLayer.digest) { data in
+            try filter.write(data)
+
+            // Update the progress
+            progress.completedUnitCount += Int64(data.count)
+          }
+
+          try filter.finalize()
+
+          try disk.close()
+        }
+
+        globalDiskWritingOffset += uncompressedLayerSize
+      }
+    }
+  }
+
+  private static func compressNextLayerOfLimitBytesOrMore(mappedDisk: Data, offset: UInt64) throws -> (Data, UInt64, String)? {
+    var compressedData = Data()
+    var bytesRead: UInt64 = 0
+    let digest = Digest()
+
+    // Create a compressing filter that we will terminate upon
+    // reaching ``Self.layerLimitBytes`` of compressed data
+    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: bufferSizeBytes) { (length: Int) -> Data? in
+      if compressedData.count >= Self.layerLimitBytes {
+        return nil
+      }
+
+      let readFromByte = Int(offset + bytesRead)
+
+      let numBytesToRead = min(mappedDisk.count - readFromByte, bufferSizeBytes)
+      if numBytesToRead == 0 {
+        return nil
+      }
+
+      let uncompressedChunk = mappedDisk.subdata(in: readFromByte ..< (readFromByte + numBytesToRead))
+
+      bytesRead += UInt64(uncompressedChunk.count)
+      digest.update(uncompressedChunk)
+
+      return uncompressedChunk
+    }
+
+    // Retrieve compressed data chunks, but normally no more than ``Self.layerLimitBytes`` bytes
+    while let compressedChunk = try compressingFilter.readData(ofLength: Self.bufferSizeBytes) {
+      compressedData.append(compressedChunk)
+    }
+
+    // Nothing was read this time from the disk,
+    // signal that to the consumer
+    if bytesRead == 0 {
+      return nil
+    }
+
+    return (compressedData, bytesRead, digest.finalize())
+  }
+}

Sources/tart/OCI/Manifest.swift

@@ -1,10 +1,22 @@
 import Foundation
 
+// OCI manifest and OCI config media types
 let ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json"
 let ociConfigMediaType = "application/vnd.oci.image.config.v1+json"
 
-// Annotations
+// Layer media types
+let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
+let diskV1MediaType = "application/vnd.cirruslabs.tart.disk.v1"
+let diskV2MediaType = "application/vnd.cirruslabs.tart.disk.v2"
+let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
+
+// Manifest annotations
 let uncompressedDiskSizeAnnotation = "org.cirruslabs.tart.uncompressed-disk-size"
+let uploadTimeAnnotation = "org.cirruslabs.tart.upload-time"
+
+// Layer annotations
+let uncompressedSizeAnnotation = "org.cirruslabs.tart.uncompressed-size"
+let uncompressedContentDigestAnnotation = "org.cirruslabs.tart.uncompressed-content-digest"
 
 struct OCIManifest: Codable, Equatable {
   var schemaVersion: Int = 2
@@ -13,15 +25,21 @@ struct OCIManifest: Codable, Equatable {
   var layers: [OCIManifestLayer] = Array()
   var annotations: Dictionary<String, String>?
 
-  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil) {
+  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil, uploadDate: Date? = nil) {
     self.config = config
     self.layers = layers
 
+    var annotations: [String: String] = [:]
+
     if let uncompressedDiskSize = uncompressedDiskSize {
-      annotations = [
+      annotations[uncompressedDiskSizeAnnotation] = String(uncompressedDiskSize)
-        uncompressedDiskSizeAnnotation: String(uncompressedDiskSize)
-      ]
     }
+
+    if let uploadDate = uploadDate {
+      annotations[uploadTimeAnnotation] = uploadDate.toISO()
+    }
+
+    self.annotations = annotations
   }
 
   init(fromJSON: Data) throws {
@@ -64,6 +82,37 @@ struct OCIManifestLayer: Codable, Equatable {
   var mediaType: String
   var size: Int
   var digest: String
+  var annotations: Dictionary<String, String>?
+
+  init(mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil, uncompressedContentDigest: String? = nil) {
+    self.mediaType = mediaType
+    self.size = size
+    self.digest = digest
+
+    var annotations: [String: String] = [:]
+
+    if let uncompressedSize = uncompressedSize {
+      annotations[uncompressedSizeAnnotation] = String(uncompressedSize)
+    }
+
+    if let uncompressedContentDigest = uncompressedContentDigest {
+      annotations[uncompressedContentDigestAnnotation] = uncompressedContentDigest
+    }
+
+    self.annotations = annotations
+  }
+
+  func uncompressedSize() -> UInt64? {
+    guard let value = annotations?[uncompressedSizeAnnotation] else {
+      return nil
+    }
+
+    return UInt64(value)
+  }
+
+  func uncompressedContentDigest() -> String? {
+    annotations?[uncompressedContentDigestAnnotation]
+  }
 }
 
 struct Descriptor: Equatable {

Sources/tart/OCI/Registry.swift

@@ -45,7 +45,8 @@ struct TokenResponse: Decodable, Authentication {
   let defaultIssuedAt = Date()
   let defaultExpiresIn = 60
 
-  var token: String
+  var token: String?
+  var accessToken: String?
   var expiresIn: Int?
   var issuedAt: Date?
 
@@ -65,7 +66,13 @@ struct TokenResponse: Decodable, Authentication {
       return dateFormatter.date(from: dateString) ?? Date()
     }
 
-    return try decoder.decode(TokenResponse.self, from: fromData)
+    let response = try decoder.decode(TokenResponse.self, from: fromData)
+
+    guard response.token != nil || response.accessToken != nil else {
+      throw DecodingError.keyNotFound(CodingKeys.token, .init(codingPath: [], debugDescription: "Missing token or access_token. One must be present."))
+    }
+
+    return response
   }
 
   var tokenExpiresAt: Date {
@@ -83,7 +90,7 @@ struct TokenResponse: Decodable, Authentication {
   }
 
   func header() -> (String, String) {
-    ("Authorization", "Bearer \(token)")
+    return ("Authorization", "Bearer \(token ?? accessToken ?? "")")
   }
 
   func isValid() -> Bool {
@@ -92,11 +99,20 @@ struct TokenResponse: Decodable, Authentication {
 }
 
 class Registry {
-  let baseURL: URL
+  private let baseURL: URL
   let namespace: String
   let credentialsProviders: [CredentialsProvider]
+  let authenticationKeeper = AuthenticationKeeper()
 
-  var currentAuthToken: Authentication? = nil
+  var host: String? {
+    guard let host = baseURL.host else { return nil }
+
+    if let port = baseURL.port {
+      return "\(host):\(port)"
+    }
+
+    return host
+  }
 
   init(urlComponents: URLComponents,
        namespace: String,
@@ -225,7 +241,7 @@ class Registry {
     return digest
   }
 
-  public func pullBlob(_ digest: String, handler: (Data) throws -> Void) async throws {
+  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
     let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
     if response.statusCode != HTTPCode.Ok.rawValue {
       let body = try await channel.asData().asText()
@@ -236,7 +252,7 @@ class Registry {
     for try await part in channel {
       try Task.checkCancellation()
 
-      try handler(Data(part))
+      try await handler(part)
     }
   }
 
@@ -288,11 +304,6 @@ class Registry {
       request.httpBody = body
     }
 
-    // Invalidate token if it has expired
-    if currentAuthToken?.isValid() == false {
-      currentAuthToken = nil
-    }
-
     var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
 
     if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
@@ -314,7 +325,7 @@ class Registry {
 
     if wwwAuthenticate.scheme.lowercased() == "basic" {
       if let (user, password) = try lookupCredentials() {
-        currentAuthToken = BasicAuthentication(user: user, password: password)
+        await authenticationKeeper.set(BasicAuthentication(user: user, password: password))
       }
 
       return
@@ -361,7 +372,7 @@ class Registry {
         + "while retrieving an authentication token", details: data.asText())
     }
 
-    currentAuthToken = try TokenResponse.parse(fromData: data)
+    await authenticationKeeper.set(try TokenResponse.parse(fromData: data))
   }
 
   private func lookupCredentials() throws -> (String, String)? {
@@ -382,11 +393,13 @@ class Registry {
   private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
     var request = request
 
-    if let token = currentAuthToken {
+    if let (name, value) = await authenticationKeeper.header() {
-      let (name, value) = token.header()
       request.addValue(value, forHTTPHeaderField: name)
     }
 
+    request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",
+                     forHTTPHeaderField: "User-Agent")
+
     return try await Fetcher.fetch(request, viaFile: viaFile)
   }
 }

Sources/tart/PIDLock.swift

@@ -8,6 +8,11 @@ class PIDLock {
   init(lockURL: URL) throws {
     url = lockURL
     fd = open(lockURL.path, O_RDWR)
+    if fd == -1 {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.PIDLockFailed("failed to open lock file \(url): \(details)")
+    }
   }
 
   deinit {

Sources/tart/Platform/Darwin.swift

@@ -1,97 +1,136 @@
 import Virtualization
 
-struct Darwin: Platform {
+struct UnsupportedHostOSError: Error, CustomStringConvertible {
-  var ecid: VZMacMachineIdentifier
+  var description: String {
-  var hardwareModel: VZMacHardwareModel
+    "error: host macOS version is outdated to run this virtual machine"
-
-  init(ecid: VZMacMachineIdentifier, hardwareModel: VZMacHardwareModel) {
-    self.ecid = ecid
-    self.hardwareModel = hardwareModel
   }
+}
 
-  init(from decoder: Decoder) throws {
+#if arch(arm64)
-    let container = try decoder.container(keyedBy: CodingKeys.self)
 
-    let encodedECID = try container.decode(String.self, forKey: .ecid)
+  struct Darwin: PlatformSuspendable {
-    guard let data = Data.init(base64Encoded: encodedECID) else {
+    var ecid: VZMacMachineIdentifier
-      throw DecodingError.dataCorruptedError(forKey: .ecid,
+    var hardwareModel: VZMacHardwareModel
-                                             in: container,
+
-                                             debugDescription: "failed to initialize Data using the provided value")
+    init(ecid: VZMacMachineIdentifier, hardwareModel: VZMacHardwareModel) {
+      self.ecid = ecid
+      self.hardwareModel = hardwareModel
     }
-    guard let ecid = VZMacMachineIdentifier.init(dataRepresentation: data) else {
+
-      throw DecodingError.dataCorruptedError(forKey: .ecid,
+    init(from decoder: Decoder) throws {
-                                             in: container,
+      let container = try decoder.container(keyedBy: CodingKeys.self)
-                                             debugDescription: "failed to initialize VZMacMachineIdentifier using the provided value")
+
+      let encodedECID = try container.decode(String.self, forKey: .ecid)
+      guard let data = Data.init(base64Encoded: encodedECID) else {
+        throw DecodingError.dataCorruptedError(forKey: .ecid,
+                                               in: container,
+                                               debugDescription: "failed to initialize Data using the provided value")
+      }
+      guard let ecid = VZMacMachineIdentifier.init(dataRepresentation: data) else {
+        throw DecodingError.dataCorruptedError(forKey: .ecid,
+                                               in: container,
+                                               debugDescription: "failed to initialize VZMacMachineIdentifier using the provided value")
+      }
+      self.ecid = ecid
+
+      let encodedHardwareModel = try container.decode(String.self, forKey: .hardwareModel)
+      guard let data = Data.init(base64Encoded: encodedHardwareModel) else {
+        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+      }
+      guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
+        throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+      }
+      self.hardwareModel = hardwareModel
     }
-    self.ecid = ecid
 
-    let encodedHardwareModel = try container.decode(String.self, forKey: .hardwareModel)
+    func encode(to encoder: Encoder) throws {
-    guard let data = Data.init(base64Encoded: encodedHardwareModel) else {
+      var container = encoder.container(keyedBy: CodingKeys.self)
-      throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+
+      try container.encode(ecid.dataRepresentation.base64EncodedString(), forKey: .ecid)
+      try container.encode(hardwareModel.dataRepresentation.base64EncodedString(), forKey: .hardwareModel)
     }
-    guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
+
-      throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
+    func os() -> OS {
+      .darwin
     }
-    self.hardwareModel = hardwareModel
-  }
 
-  func encode(to encoder: Encoder) throws {
+    func bootLoader(nvramURL: URL) throws -> VZBootLoader {
-    var container = encoder.container(keyedBy: CodingKeys.self)
+      VZMacOSBootLoader()
+    }
 
-    try container.encode(ecid.dataRepresentation.base64EncodedString(), forKey: .ecid)
+    func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
-    try container.encode(hardwareModel.dataRepresentation.base64EncodedString(), forKey: .hardwareModel)
+      let result = VZMacPlatformConfiguration()
-  }
 
-  func os() -> OS {
+      result.machineIdentifier = ecid
-    .darwin
+      result.auxiliaryStorage = VZMacAuxiliaryStorage(url: nvramURL)
-  }
 
-  func bootLoader(nvramURL: URL) throws -> VZBootLoader {
+      if !hardwareModel.isSupported {
-    VZMacOSBootLoader()
+        // At the moment support of M1 chip is not yet dropped in any macOS version
-  }
+        // This mean that host software is not supporting this hardware model and should be updated
+        throw UnsupportedHostOSError()
+      }
 
-  func platform(nvramURL: URL) -> VZPlatformConfiguration {
+      result.hardwareModel = hardwareModel
-    let result = VZMacPlatformConfiguration()
 
-    result.machineIdentifier = ecid
+      return result
-    result.auxiliaryStorage = VZMacAuxiliaryStorage(contentsOf: nvramURL)
+    }
-    result.hardwareModel = hardwareModel
 
-    return result
+    func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
-  }
+      let result = VZMacGraphicsDeviceConfiguration()
 
-  func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
+      if let hostMainScreen = NSScreen.main {
-    let result = VZMacGraphicsDeviceConfiguration()
+        let vmScreenSize = NSSize(width: vmConfig.display.width, height: vmConfig.display.height)
+        result.displays = [
+          VZMacGraphicsDisplayConfiguration(for: hostMainScreen, sizeInPoints: vmScreenSize)
+        ]
+
+        return result
+      }
 
-    if let hostMainScreen = NSScreen.main {
-      let vmScreenSize = NSSize(width: vmConfig.display.width, height: vmConfig.display.height)
       result.displays = [
-        VZMacGraphicsDisplayConfiguration(for: hostMainScreen, sizeInPoints: vmScreenSize)
+        VZMacGraphicsDisplayConfiguration(
+          widthInPixels: vmConfig.display.width,
+          heightInPixels: vmConfig.display.height,
+          // A reasonable guess according to Apple's documentation[1]
+          // [1]: https://developer.apple.com/documentation/coregraphics/1456599-cgdisplayscreensize
+          pixelsPerInch: 72
+        )
       ]
 
       return result
     }
 
-    result.displays = [
+    func keyboards() -> [VZKeyboardConfiguration] {
-      VZMacGraphicsDisplayConfiguration(
+      if #available(macOS 14, *) {
-        widthInPixels: vmConfig.display.width,
+        // Mac keyboard is only supported by guests starting with macOS Ventura
-        heightInPixels: vmConfig.display.height,
+        return [VZUSBKeyboardConfiguration(), VZMacKeyboardConfiguration()]
-        // A reasonable guess according to Apple's documentation[1]
+      } else {
-        // [1]: https://developer.apple.com/documentation/coregraphics/1456599-cgdisplayscreensize
+        return [VZUSBKeyboardConfiguration()]
-        pixelsPerInch: 72
+      }
-      )
+    }
-    ]
 
-    return result
+    func keyboardsSuspendable() -> [VZKeyboardConfiguration] {
-  }
+      if #available(macOS 14, *) {
+        return [VZMacKeyboardConfiguration()]
+      } else {
+        // fallback to the regular configuration
+        return keyboards()
+      }
+    }
 
-  func pointingDevices() -> [VZPointingDeviceConfiguration] {
+    func pointingDevices() -> [VZPointingDeviceConfiguration] {
-    if #available(macOS 13, *) {
+      // Trackpad is only supported by guests starting with macOS Ventura
-      // Trackpad is only supported starting with macOS Ventura
+      [VZUSBScreenCoordinatePointingDeviceConfiguration(), VZMacTrackpadConfiguration()]
-      // macOS Monterey will continue using a USB device == .darwin 
+    }
-      return [VZMacTrackpadConfiguration(), VZUSBScreenCoordinatePointingDeviceConfiguration()]
+
-    } else {
+    func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration] {
-      return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+      if #available(macOS 14, *) {
+        return [VZMacTrackpadConfiguration()]
+      } else {
+        // fallback to the regular configuration
+        return pointingDevices()
+      }
     }
   }
-}
+
+#endif

Sources/tart/Platform/Linux.swift

@@ -14,7 +14,7 @@ struct Linux: Platform {
     return result
   }
 
-  func platform(nvramURL: URL) -> VZPlatformConfiguration {
+  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
     VZGenericPlatformConfiguration()
   }
 
@@ -31,6 +31,10 @@ struct Linux: Platform {
     return result
   }
 
+  func keyboards() -> [VZKeyboardConfiguration] {
+    [VZUSBKeyboardConfiguration()]
+  }
+
   func pointingDevices() -> [VZPointingDeviceConfiguration] {
     [VZUSBScreenCoordinatePointingDeviceConfiguration()]
   }

Sources/tart/Platform/Platform.swift

@@ -3,7 +3,13 @@ import Virtualization
 protocol Platform: Codable {
   func os() -> OS
   func bootLoader(nvramURL: URL) throws -> VZBootLoader
-  func platform(nvramURL: URL) -> VZPlatformConfiguration
+  func platform(nvramURL: URL) throws -> VZPlatformConfiguration
   func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
+  func keyboards() -> [VZKeyboardConfiguration]
   func pointingDevices() -> [VZPointingDeviceConfiguration]
 }
+
+protocol PlatformSuspendable: Platform {
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration]
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration]
+}

Sources/tart/Prunable.swift

@@ -8,5 +8,8 @@ protocol Prunable {
   var url: URL { get }
   func delete() throws
   func accessDate() throws -> Date
+  // size on disk as seen in Finder including empty blocks
   func sizeBytes() throws -> Int
+  // actual size on disk without empty blocks
+  func allocatedSizeBytes() throws -> Int
 }

Sources/tart/Root.swift

@@ -16,6 +16,7 @@ struct Root: AsyncParsableCommand {
       Get.self,
       List.self,
       Login.self,
+      Logout.self,
       IP.self,
       Pull.self,
       Push.self,
@@ -25,6 +26,7 @@ struct Root: AsyncParsableCommand {
       Rename.self,
       Stop.self,
       Delete.self,
+      FQN.self,
     ])
 
   public static func main() async throws {
@@ -57,6 +59,11 @@ struct Root: AsyncParsableCommand {
       }
     }
 
+    // Add commands that are only available on specific macOS versions
+    if #available(macOS 14, *) {
+      configuration.subcommands.append(Suspend.self)
+    }
+
     // Ensure the default SIGINT handled is disabled,
     // otherwise there's a race between two handlers
     signal(SIGINT, SIG_IGN);
@@ -76,10 +83,12 @@ struct Root: AsyncParsableCommand {
       var command = try parseAsRoot()
 
       // Run garbage-collection before each command (shouldn't take too long)
-      do {
+      if type(of: command) != type(of: Pull()) && type(of: command) != type(of: Clone()){
-        try Config().gc()
+        do {
-      } catch {
+          try Config().gc()
-        fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+        } catch {
+          fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+        }
       }
 
       if var asyncCommand = command as? AsyncParsableCommand {
@@ -94,7 +103,7 @@ struct Root: AsyncParsableCommand {
 
       // Handle a non-ArgumentParser's exception that requires a specific exit code to be set
       if let errorWithExitCode = error as? HasExitCode {
-        print(error)
+        fputs("\(error)\n", stderr)
 
         Foundation.exit(errorWithExitCode.exitCode)
       }

Sources/tart/URL+Prunable.swift

@@ -9,7 +9,11 @@ extension URL: Prunable {
     try FileManager.default.removeItem(at: self)
   }
 
-  func sizeBytes() throws -> Int {
+  func allocatedSizeBytes() throws -> Int {
     try resourceValues(forKeys: [.totalFileAllocatedSizeKey]).totalFileAllocatedSize!
   }
+
+  func sizeBytes() throws -> Int {
+    try resourceValues(forKeys: [.totalFileSizeKey]).totalFileSize!
+  }
 }

Sources/tart/VM+Recovery.swift

@@ -5,34 +5,26 @@ import Dynamic
 // Kudos to @saagarjha's VirtualApple for finding about _VZVirtualMachineStartOptions
 
 extension VZVirtualMachine {
-  @available(macOS 12, *)
+  @MainActor @available(macOS 12, *)
   func start(_ recovery: Bool) async throws {
     if !recovery {
       // just use the regular API
-      return try await withCheckedThrowingContinuation { continuation in
+      return try await self.start()
-        DispatchQueue.main.async {
-          self.start(completionHandler: { result in
-            continuation.resume(with: result)
-          })
-        }
-      }
     }
 
     // use some private stuff only for recovery
     return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
-      DispatchQueue.main.async {
+      let handler: @convention(block) (_ result: Any?) -> Void = { result in
-        let handler: @convention(block) (_ result: Any?) -> Void = { result in
+        if let error = result as? Error {
-          if let error = result as? Error {
+          continuation.resume(throwing: error)
-            continuation.resume(throwing: error)
+        } else {
-          } else {
+          continuation.resume(returning: ())
-            continuation.resume(returning: ())
-          }
         }
-        // dynamic magic
-        let options = Dynamic._VZVirtualMachineStartOptions()
-        options.bootMacOSRecovery = recovery
-        Dynamic(self)._start(withOptions: options, completionHandler: handler)
       }
+      // dynamic magic
+      let options = Dynamic._VZVirtualMachineStartOptions()
+      options.bootMacOSRecovery = recovery
+      Dynamic(self)._start(withOptions: options, completionHandler: handler)
     }
   }
 }

Sources/tart/VM.swift

@@ -26,6 +26,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   // Virtualization.Framework's virtual machine
   @Published var virtualMachine: VZVirtualMachine
 
+  // Virtualization.Framework's virtual machine configuration
+  var configuration: VZVirtualMachineConfiguration
+
   // Semaphore used to communicate with the VZVirtualMachineDelegate
   var sema = DispatchSemaphore(value: 0)
 
@@ -39,9 +42,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
   init(vmDir: VMDirectory,
        network: Network = NetworkShared(),
-       additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
+       additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
        directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
-       serialPorts: [VZSerialPortConfiguration] = []
+       serialPorts: [VZSerialPortConfiguration] = [],
+       suspendable: Bool = false,
+       audio: Bool = true
   ) throws {
     name = vmDir.name
     config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -52,11 +57,13 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
     // Initialize the virtual machine and its configuration
     self.network = network
-    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
+    configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
-                                                    nvramURL: vmDir.nvramURL, vmConfig: config,
+                                                nvramURL: vmDir.nvramURL, vmConfig: config,
-                                                    network: network, additionalDiskAttachments: additionalDiskAttachments,
+                                                network: network, additionalStorageDevices: additionalStorageDevices,
-                                                    directorySharingDevices: directorySharingDevices,
+                                                directorySharingDevices: directorySharingDevices,
-                                                    serialPorts: serialPorts
+                                                serialPorts: serialPorts,
+                                                suspendable: suspendable,
+                                                audio: audio
     )
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
@@ -66,9 +73,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
   static func retrieveIPSW(remoteURL: URL) async throws -> URL {
     // Check if we already have this IPSW in cache
-    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+    var headRequest = URLRequest(url: remoteURL)
+    headRequest.httpMethod = "HEAD"
+    let (_, headResponse) = try await Fetcher.fetch(headRequest, viaFile: false)
 
-    if let hash = response.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
+    if let hash = headResponse.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
       let ipswLocation = try IPSWCache().locationFor(fileName: "sha256:\(hash).ipsw")
 
       if FileManager.default.fileExists(atPath: ipswLocation.path) {
@@ -82,6 +91,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Download the IPSW
     defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")
 
+    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+
     let progress = Progress(totalUnitCount: response.expectedContentLength)
     ProgressObserver(progress).log(defaultLogger)
 
@@ -107,18 +118,6 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     return try FileManager.default.replaceItemAt(finalLocation, withItemAt: temporaryLocation)!
   }
 
-  static func latestIPSWURL() async throws -> URL {
-    defaultLogger.appendNewLine("Looking up the latest supported IPSW...")
-
-    let image = try await withCheckedThrowingContinuation { continuation in
-      VZMacOSRestoreImage.fetchLatestSupported() { result in
-        continuation.resume(with: result)
-      }
-    }
-
-    return image.url
-  }
-
   var inFinalState: Bool {
     get {
       virtualMachine.state == VZVirtualMachine.State.stopped ||
@@ -128,82 +127,84 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     }
   }
 
-  init(
+  #if arch(arm64)
-    vmDir: VMDirectory,
+    init(
-    ipswURL: URL,
+      vmDir: VMDirectory,
-    diskSizeGB: UInt16,
+      ipswURL: URL,
-    network: Network = NetworkShared(),
+      diskSizeGB: UInt16,
-    additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
+      network: Network = NetworkShared(),
-    directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
+      additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
-    serialPorts: [VZSerialPortConfiguration] = []
+      directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
-  ) async throws {
+      serialPorts: [VZSerialPortConfiguration] = []
-    var ipswURL = ipswURL
+    ) async throws {
+      var ipswURL = ipswURL
 
-    if !ipswURL.isFileURL {
+      if !ipswURL.isFileURL {
-      ipswURL = try await VM.retrieveIPSW(remoteURL: ipswURL)
+        ipswURL = try await VM.retrieveIPSW(remoteURL: ipswURL)
-    }
-
-    // We create a temporary TART_HOME directory in tests, which has its "cache" folder symlinked
-    // to the users Tart cache directory (~/.tart/cache). However, the Virtualization.Framework
-    // cannot deal with paths that contain symlinks, so expand them here first.
-    ipswURL.resolveSymlinksInPath()
-
-    // Load the restore image and try to get the requirements
-    // that match both the image and our platform
-    let image = try await withCheckedThrowingContinuation { continuation in
-      VZMacOSRestoreImage.load(from: ipswURL) { result in
-        continuation.resume(with: result)
       }
-    }
 
-    guard let requirements = image.mostFeaturefulSupportedConfiguration else {
+      // We create a temporary TART_HOME directory in tests, which has its "cache" folder symlinked
-      throw UnsupportedRestoreImageError()
+      // to the users Tart cache directory (~/.tart/cache). However, the Virtualization.Framework
-    }
+      // cannot deal with paths that contain symlinks, so expand them here first.
+      ipswURL.resolveSymlinksInPath()
 
-    // Create NVRAM
+      // Load the restore image and try to get the requirements
-    _ = try VZMacAuxiliaryStorage(creatingStorageAt: vmDir.nvramURL, hardwareModel: requirements.hardwareModel)
+      // that match both the image and our platform
-
+      let image = try await withCheckedThrowingContinuation { continuation in
-    // Create disk
+        VZMacOSRestoreImage.load(from: ipswURL) { result in
-    try vmDir.resizeDisk(diskSizeGB)
-
-    name = vmDir.name
-    // Create config
-    config = VMConfig(
-      platform: Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: requirements.hardwareModel),
-      cpuCountMin: requirements.minimumSupportedCPUCount,
-      memorySizeMin: requirements.minimumSupportedMemorySize
-    )
-    // allocate at least 4 CPUs because otherwise VMs are frequently freezing
-    try config.setCPU(cpuCount: max(4, requirements.minimumSupportedCPUCount))
-    try config.save(toURL: vmDir.configURL)
-
-    // Initialize the virtual machine and its configuration
-    self.network = network
-    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
-                                                    vmConfig: config, network: network,
-                                                    additionalDiskAttachments: additionalDiskAttachments,
-                                                    directorySharingDevices: directorySharingDevices,
-                                                    serialPorts: serialPorts
-    )
-    virtualMachine = VZVirtualMachine(configuration: configuration)
-
-    super.init()
-    virtualMachine.delegate = self
-
-    // Run automated installation
-    try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
-      DispatchQueue.main.async { [ipswURL] in
-        let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: ipswURL)
-
-        defaultLogger.appendNewLine("Installing OS...")
-        ProgressObserver(installer.progress).log(defaultLogger)
-
-        installer.install { result in
           continuation.resume(with: result)
         }
       }
+
+      guard let requirements = image.mostFeaturefulSupportedConfiguration else {
+        throw UnsupportedRestoreImageError()
+      }
+
+      // Create NVRAM
+      _ = try VZMacAuxiliaryStorage(creatingStorageAt: vmDir.nvramURL, hardwareModel: requirements.hardwareModel)
+
+      // Create disk
+      try vmDir.resizeDisk(diskSizeGB)
+
+      name = vmDir.name
+      // Create config
+      config = VMConfig(
+        platform: Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: requirements.hardwareModel),
+        cpuCountMin: requirements.minimumSupportedCPUCount,
+        memorySizeMin: requirements.minimumSupportedMemorySize
+      )
+      // allocate at least 4 CPUs because otherwise VMs are frequently freezing
+      try config.setCPU(cpuCount: max(4, requirements.minimumSupportedCPUCount))
+      try config.save(toURL: vmDir.configURL)
+
+      // Initialize the virtual machine and its configuration
+      self.network = network
+      configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
+                                                  vmConfig: config, network: network,
+                                                  additionalStorageDevices: additionalStorageDevices,
+                                                  directorySharingDevices: directorySharingDevices,
+                                                  serialPorts: serialPorts
+      )
+      virtualMachine = VZVirtualMachine(configuration: configuration)
+
+      super.init()
+      virtualMachine.delegate = self
+
+      // Run automated installation
+      try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
+        DispatchQueue.main.async { [ipswURL] in
+          let installer = VZMacOSInstaller(virtualMachine: self.virtualMachine, restoringFromImageAt: ipswURL)
+
+          defaultLogger.appendNewLine("Installing OS...")
+          ProgressObserver(installer.progress).log(defaultLogger)
+
+          installer.install { result in
+            continuation.resume(with: result)
+          }
+        }
+      }
     }
-  }
+  #endif
 
   @available(macOS 13, *)
   static func linux(vmDir: VMDirectory, diskSizeGB: UInt16) async throws -> VM {
@@ -220,27 +221,18 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     return try VM(vmDir: vmDir)
   }
 
-  func run(_ recovery: Bool) async throws {
+  func start(recovery: Bool, resume shouldResume: Bool) async throws {
     try network.run(sema)
 
-    let startTask = DispatchQueue.main.sync {
+    if shouldResume {
-      Task {
+      try await resume()
-        if #available(macOS 13, *) {
+    } else {
-          // new API introduced in Ventura
+      try await start(recovery)
-          let startOptions = VZMacOSVirtualMachineStartOptions()
-          startOptions.startUpFromMacOSRecovery = recovery
-          try await virtualMachine.start(options: startOptions)
-        } else {
-          // use method that also available on Monterey
-          try await virtualMachine.start(recovery)
-        }
-      }
     }
+  }
 
-    try await withTaskCancellationHandler(operation: {
+  func run() async throws {
-      // Await on VZVirtualMachine.start() result
+    await withTaskCancellationHandler(operation: {
-      _ = try await startTask.value
-
       // Wait for the VM to finish running
       // or for the exit condition
       sema.wait()
@@ -249,24 +241,43 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     })
 
     if Task.isCancelled {
-      DispatchQueue.main.sync {
+      try await stop()
-        Task {
-          try await self.virtualMachine.stop()
-        }
-      }
     }
 
     try await network.stop()
   }
 
+  @MainActor
+  private func start(_ recovery: Bool) async throws {
+    #if arch(arm64)
+      let startOptions = VZMacOSVirtualMachineStartOptions()
+      startOptions.startUpFromMacOSRecovery = recovery
+      try await virtualMachine.start(options: startOptions)
+    #else
+      try await virtualMachine.start()
+    #endif
+  }
+
+  @MainActor
+  private func resume() async throws {
+    try await virtualMachine.resume()
+  }
+
+  @MainActor
+  private func stop() async throws {
+    try await self.virtualMachine.stop()
+  }
+
   static func craftConfiguration(
     diskURL: URL,
     nvramURL: URL,
     vmConfig: VMConfig,
     network: Network = NetworkShared(),
-    additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment],
+    additionalStorageDevices: [VZStorageDeviceConfiguration],
     directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
-    serialPorts: [VZSerialPortConfiguration]
+    serialPorts: [VZSerialPortConfiguration],
+    suspendable: Bool = false,
+    audio: Bool = true
   ) throws -> VZVirtualMachineConfiguration {
     let configuration = VZVirtualMachineConfiguration()
 
@@ -278,37 +289,59 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     configuration.memorySize = vmConfig.memorySize
 
     // Platform
-    configuration.platform = vmConfig.platform.platform(nvramURL: nvramURL)
+    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL)
 
     // Display
     configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]
 
     // Audio
-    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+    if audio && !suspendable {
-    let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
+      let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
-    inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
+      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-    let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
-    outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
-    soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
+      outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
-    configuration.audioDevices = [soundDeviceConfiguration]
+      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
+      configuration.audioDevices = [soundDeviceConfiguration]
+    }
 
     // Keyboard and mouse
-    configuration.keyboards = [VZUSBKeyboardConfiguration()]
+    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
-    configuration.pointingDevices = vmConfig.platform.pointingDevices()
+      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
+      configuration.pointingDevices = platformSuspendable.pointingDevicesSuspendable()
+    } else {
+      configuration.keyboards = vmConfig.platform.keyboards()
+      configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    }
 
     // Networking
-    let vio = VZVirtioNetworkDeviceConfiguration()
+    configuration.networkDevices = network.attachments().map {
-    vio.attachment = network.attachment()
+      let vio = VZVirtioNetworkDeviceConfiguration()
-    vio.macAddress = vmConfig.macAddress
+      vio.attachment = $0
-    configuration.networkDevices = [vio]
+      vio.macAddress = vmConfig.macAddress
+      return vio
+    }
 
     // Storage
-    var attachments = [try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)]
+    let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
-    attachments.append(contentsOf: additionalDiskAttachments)
+      // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
-    configuration.storageDevices = attachments.map { VZVirtioBlockDeviceConfiguration(attachment: $0) }
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: .full) :
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
+
+    var device: VZStorageDeviceConfiguration
+    if #available(macOS 14, *), vmConfig.os == .linux {
+      device = VZNVMExpressControllerDeviceConfiguration(attachment: attachment)
+    } else {
+      device = VZVirtioBlockDeviceConfiguration(attachment: attachment)
+    }
+    var devices: [VZStorageDeviceConfiguration] = [device]
+    devices.append(contentsOf: additionalStorageDevices)
+    configuration.storageDevices = devices
 
     // Entropy
-    configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()]
+    if !suspendable {
+      configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()]
+    }
 
     // Directory sharing devices
     configuration.directorySharingDevices = directorySharingDevices
@@ -316,6 +349,20 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Serial Port
     configuration.serialPorts = serialPorts
 
+    // Version console device
+    //
+    // A dummy console device useful for implementing
+    // host feature checks in the guest agent software.
+    if !suspendable {
+      let consolePort = VZVirtioConsolePortConfiguration()
+      consolePort.name = "tart-version-\(CI.version)"
+
+      let consoleDevice = VZVirtioConsoleDeviceConfiguration()
+      consoleDevice.ports[0] = consolePort
+
+      configuration.consoleDevices.append(consoleDevice)
+    }
+
     try configuration.validate()
 
     return configuration

Sources/tart/VMConfig.swift

@@ -95,13 +95,16 @@ struct VMConfig: Codable {
     arch = try container.decodeIfPresent(Architecture.self, forKey: .arch) ?? .arm64
     switch os {
     case .darwin:
-      platform = try Darwin(from: decoder)
+      #if arch(arm64)
+        platform = try Darwin(from: decoder)
+      #else
+        throw DecodingError.dataCorruptedError(
+          forKey: .os,
+          in: container,
+          debugDescription: "Darwin VMs are only supported on Apple Silicon hosts")
+      #endif
     case .linux:
-      if #available(macOS 13, *) {
+      platform = try Linux(from: decoder)
-        platform = try Linux(from: decoder)
-      } else {
-        throw UnsupportedOSError("Linux VMs", "are")
-      }
     }
     cpuCountMin = try container.decode(Int.self, forKey: .cpuCountMin)
     cpuCount = try container.decode(Int.self, forKey: .cpuCount)
@@ -136,20 +139,30 @@ struct VMConfig: Codable {
   }
 
   mutating func setCPU(cpuCount: Int) throws {
-    if cpuCount < cpuCountMin {
+    if os == .darwin && cpuCount < cpuCountMin {
       throw LessThanMinimalResourcesError("VM should have \(cpuCountMin) CPU cores"
         + " at minimum (requested \(cpuCount))")
     }
 
+    if cpuCount < VZVirtualMachineConfiguration.minimumAllowedCPUCount {
+      throw LessThanMinimalResourcesError("VM should have \(VZVirtualMachineConfiguration.minimumAllowedCPUCount) CPU cores"
+        + " at minimum (requested \(cpuCount))")
+    }
+
     self.cpuCount = cpuCount
   }
 
   mutating func setMemory(memorySize: UInt64) throws {
-    if memorySize < memorySizeMin {
+    if os == .darwin && memorySize < memorySizeMin {
       throw LessThanMinimalResourcesError("VM should have \(memorySizeMin) bytes"
         + " of memory at minimum (requested \(memorySize))")
     }
 
+    if memorySize < VZVirtualMachineConfiguration.minimumAllowedMemorySize {
+      throw LessThanMinimalResourcesError("VM should have \(VZVirtualMachineConfiguration.minimumAllowedMemorySize) bytes"
+        + " of memory at minimum (requested \(memorySize))")
+    }
+
     self.memorySize = memorySize
   }
 }

Sources/tart/VMDirectory+Archive.swift

@@ -81,7 +81,8 @@ extension VMDirectory {
       try? decodeStream.close()
     }
 
-    guard let extractStream = ArchiveStream.extractStream(extractingTo: FilePath(baseURL.path)) else {
+    guard let extractStream = ArchiveStream.extractStream(extractingTo: FilePath(baseURL.path),
+                                                          flags: [.ignoreOperationNotPermitted]) else {
       let details = Errno(rawValue: CInt(errno))
 
       throw RuntimeError.ImportFailed("ArchiveStream.extractStream() failed: \(details)")

Sources/tart/VMDirectory+OCI.swift

@@ -1,33 +1,30 @@
 import Foundation
-import Compression
 import Sentry
 
 enum OCIError: Error {
   case ShouldBeExactlyOneLayer
   case ShouldBeAtLeastOneLayer
   case FailedToCreateVmFile
+  case LayerIsMissingUncompressedSizeAnnotation
+  case LayerIsMissingUncompressedDigestAnnotation
 }
 
 extension VMDirectory {
   private static let bufferSizeBytes = 64 * 1024 * 1024
   private static let layerLimitBytes = 500 * 1000 * 1000
 
-  private static let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
+  func pullFromRegistry(registry: Registry, reference: String, concurrency: UInt) async throws {
-  private static let diskMediaType = "application/vnd.cirruslabs.tart.disk.v1"
-  private static let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
-
-  func pullFromRegistry(registry: Registry, reference: String) async throws {
     defaultLogger.appendNewLine("pulling manifest...")
 
     let (manifest, _) = try await registry.pullManifest(reference: reference)
 
-    return try await pullFromRegistry(registry: registry, manifest: manifest)
+    return try await pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
   }
 
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt) async throws {
     // Pull VM's config file layer and re-serialize it into a config file
     let configLayers = manifest.layers.filter {
-      $0.mediaType == Self.configMediaType
+      $0.mediaType == configMediaType
     }
     if configLayers.count != 1 {
       throw OCIError.ShouldBeExactlyOneLayer
@@ -41,50 +38,36 @@ extension VMDirectory {
     }
     try configFile.close()
 
-    // Pull VM's disk layers and decompress them sequentially into a disk file
+    // Pull VM's disk layers and decompress them into a disk file
-    let diskLayers = manifest.layers.filter {
+    let diskImplType: Disk.Type
-      $0.mediaType == Self.diskMediaType
+    let layers: [OCIManifestLayer]
-    }
+
-    if diskLayers.isEmpty {
+    if manifest.layers.contains(where: { $0.mediaType == diskV1MediaType }) {
+      diskImplType = DiskV1.self
+      layers = manifest.layers.filter { $0.mediaType == diskV1MediaType }
+    } else if manifest.layers.contains(where: { $0.mediaType == diskV2MediaType }) {
+      diskImplType = DiskV2.self
+      layers = manifest.layers.filter { $0.mediaType == diskV2MediaType }
+    } else {
       throw OCIError.ShouldBeAtLeastOneLayer
     }
-    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
-      throw OCIError.FailedToCreateVmFile
-    }
-    let disk = try FileHandle(forWritingTo: diskURL)
-    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
-      if let data = data {
-        disk.write(data)
-      }
-    }
 
-    // Progress
+    let diskCompressedSize = layers.map { Int64($0.size) }.reduce(0, +)
-    let diskCompressedSize: Int64 = Int64(diskLayers.map {
+    SentrySDK.span?.setMeasurement(name: "compressed_disk_size", value: diskCompressedSize as NSNumber, unit: MeasurementUnitInformation.byte)
-      $0.size
+
-    }
-    .reduce(0) {
-      $0 + $1
-    })
     let prettyDiskSize = String(format: "%.1f", Double(diskCompressedSize) / 1_000_000_000.0)
     defaultLogger.appendNewLine("pulling disk (\(prettyDiskSize) GB compressed)...")
+
     let progress = Progress(totalUnitCount: diskCompressedSize)
     ProgressObserver(progress).log(defaultLogger)
 
-    for diskLayer in diskLayers {
+    try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL, concurrency: concurrency, progress: progress)
-      try await registry.pullBlob(diskLayer.digest) { data in
-        try filter.write(data)
-        progress.completedUnitCount += Int64(data.count)
-      }
-    }
-    try filter.finalize()
-    try disk.close()
-    SentrySDK.span?.setMeasurement(name: "compressed_disk_size", value: diskCompressedSize as NSNumber, unit: MeasurementUnitInformation.byte);
 
     // Pull VM's NVRAM file layer and store it in an NVRAM file
     defaultLogger.appendNewLine("pulling NVRAM...")
 
     let nvramLayers = manifest.layers.filter {
-      $0.mediaType == Self.nvramMediaType
+      $0.mediaType == nvramMediaType
     }
     if nvramLayers.count != 1 {
       throw OCIError.ShouldBeExactlyOneLayer
@@ -99,7 +82,7 @@ extension VMDirectory {
     try nvram.close()
   }
 
-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String) async throws -> RemoteName {
     var layers = Array<OCIManifestLayer>()
 
     // Read VM's config and push it as blob
@@ -107,32 +90,22 @@ extension VMDirectory {
     let configJSON = try JSONEncoder().encode(config)
     defaultLogger.appendNewLine("pushing config...")
     let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb)
-    layers.append(OCIManifestLayer(mediaType: Self.configMediaType, size: configJSON.count, digest: configDigest))
+    layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest))
 
-    // Progress
+    // Compress the disk file as multiple chunks and push them as disk layers
     let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64
 
     defaultLogger.appendNewLine("pushing disk... this will take a while...")
     let progress = Progress(totalUnitCount: diskSize)
     ProgressObserver(progress).log(defaultLogger)
 
-    // Read VM's compressed disk as chunks
+    switch diskFormat {
-    // and sequentially upload them as blobs
+    case "v1":
-    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
-    let mappedDiskSize = mappedDisk.count
+    case "v2":
-    var mappedDiskReadOffset = 0
+      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
-    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
+    default:
-      let bytesRead = min(length, mappedDiskSize - mappedDiskReadOffset)
+      throw RuntimeError.OCIUnsupportedDiskFormat(diskFormat)
-      let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
-      mappedDiskReadOffset += bytesRead
-
-      progress.completedUnitCount = Int64(mappedDiskReadOffset)
-
-      return data
-    }
-    while let compressedLayerData = try compressingFilter.readData(ofLength: Self.layerLimitBytes) {
-      let layerDigest = try await registry.pushBlob(fromData: compressedLayerData, chunkSizeMb: chunkSizeMb)
-      layers.append(OCIManifestLayer(mediaType: Self.diskMediaType, size: compressedLayerData.count, digest: layerDigest))
     }
 
     // Read VM's NVRAM and push it as blob
@@ -140,7 +113,7 @@ extension VMDirectory {
 
     let nvram = try FileHandle(forReadingFrom: nvramURL).readToEnd()!
     let nvramDigest = try await registry.pushBlob(fromData: nvram, chunkSizeMb: chunkSizeMb)
-    layers.append(OCIManifestLayer(mediaType: Self.nvramMediaType, size: nvram.count, digest: nvramDigest))
+    layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest))
 
     // Craft a stub OCI config for Docker Hub compatibility
     let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os).toJSON()
@@ -148,7 +121,8 @@ extension VMDirectory {
     let manifest = OCIManifest(
       config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
       layers: layers,
-      uncompressedDiskSize: UInt64(mappedDiskReadOffset)
+      uncompressedDiskSize: UInt64(diskSize),
+      uploadDate: Date()
     )
 
     // Manifest
@@ -159,7 +133,7 @@ extension VMDirectory {
     }
 
     let pushedReference = Reference(digest: try manifest.digest())
-    return RemoteName(host: registry.baseURL.host!, namespace: registry.namespace, reference: pushedReference)
+    return RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference)
   }
 }
 

Sources/tart/VMDirectory.swift

@@ -1,5 +1,6 @@
 import Foundation
 import Virtualization
+import CryptoKit
 
 struct VMDirectory: Prunable {
   var baseURL: URL
@@ -13,6 +14,9 @@ struct VMDirectory: Prunable {
   var nvramURL: URL {
     baseURL.appendingPathComponent("nvram.bin")
   }
+  var stateURL: URL {
+    baseURL.appendingPathComponent("state.vzvmsave")
+  }
 
   var explicitlyPulledMark: URL {
     baseURL.appendingPathComponent(".explicitly-pulled")
@@ -26,8 +30,31 @@ struct VMDirectory: Prunable {
     baseURL
   }
 
+  func lock() throws -> PIDLock {
+    try PIDLock(lockURL: configURL)
+  }
+
   func running() throws -> Bool {
-    try PIDLock(lockURL: configURL).pid() != 0
+    // The most common reason why PIDLock() instantiation fails is a race with "tart delete" (ENOENT),
+    // which is fine to report as "not running".
+    //
+    // The other reasons are unlikely and the cost of getting a false positive is way less than
+    // the cost of crashing with an exception when calling "tart list" on a busy machine, for example.
+    guard let lock = try? lock() else {
+      return false
+    }
+
+    return try lock.pid() != 0
+  }
+
+  func state() throws -> String {
+    if try running() {
+      return "running"
+    } else if FileManager.default.fileExists(atPath: stateURL.path) {
+      return "suspended"
+    } else {
+      return "stopped"
+    }
   }
 
   static func temporary() throws -> VMDirectory {
@@ -37,6 +64,17 @@ struct VMDirectory: Prunable {
     return VMDirectory(baseURL: tmpDir)
   }
 
+  //Create tmp directory with hashing
+  static func temporaryDeterministic(key: String) throws -> VMDirectory {
+    let keyData = Data(key.utf8)
+    let hash = Insecure.MD5.hash(data: keyData)
+    // Convert hash to string
+    let hashString = hash.compactMap { String(format: "%02x", $0) }.joined()
+    let tmpDir = try Config().tartTmpDir.appendingPathComponent(hashString)
+    try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true)
+    return VMDirectory(baseURL: tmpDir)
+  }
+
   var initialized: Bool {
     FileManager.default.fileExists(atPath: configURL.path) &&
       FileManager.default.fileExists(atPath: diskURL.path) &&
@@ -55,9 +93,9 @@ struct VMDirectory: Prunable {
     try? FileManager.default.removeItem(at: nvramURL)
   }
 
-  func validate() throws {
+  func validate(userFriendlyName: String) throws {
     if !FileManager.default.fileExists(atPath: baseURL.path) {
-      throw RuntimeError.VMDoesNotExist(name: baseURL.lastPathComponent)
+      throw RuntimeError.VMDoesNotExist(name: userFriendlyName)
     }
 
     if !initialized {
@@ -70,6 +108,7 @@ struct VMDirectory: Prunable {
     try FileManager.default.copyItem(at: configURL, to: to.configURL)
     try FileManager.default.copyItem(at: nvramURL, to: to.nvramURL)
     try FileManager.default.copyItem(at: diskURL, to: to.diskURL)
+    try? FileManager.default.copyItem(at: stateURL, to: to.stateURL)
 
     // Re-generate MAC address
     if generateMAC {
@@ -85,6 +124,8 @@ struct VMDirectory: Prunable {
     var vmConfig = try VMConfig(fromURL: configURL)
 
     vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
+    // cleanup state if any
+    try? FileManager.default.removeItem(at: stateURL)
 
     try vmConfig.save(toURL: configURL)
   }
@@ -93,20 +134,45 @@ struct VMDirectory: Prunable {
     if !FileManager.default.fileExists(atPath: diskURL.path) {
       FileManager.default.createFile(atPath: diskURL.path, contents: nil, attributes: nil)
     }
+
     let diskFileHandle = try FileHandle.init(forWritingTo: diskURL)
-    // macOS considers kilo being 1000 and not 1024
+    let currentDiskFileLength = try diskFileHandle.seekToEnd()
-    try diskFileHandle.truncate(atOffset: UInt64(sizeGB) * 1000 * 1000 * 1000)
+    let desiredDiskFileLength = UInt64(sizeGB) * 1000 * 1000 * 1000
+    if desiredDiskFileLength < currentDiskFileLength {
+      let currentLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(currentDiskFileLength))
+      let desiredLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(desiredDiskFileLength))
+      throw RuntimeError.InvalidDiskSize("new disk size of \(desiredLengthHuman) should be larger " +
+        "than the current disk size of \(currentLengthHuman)")
+    } else if desiredDiskFileLength > currentDiskFileLength {
+      try diskFileHandle.truncate(atOffset: desiredDiskFileLength)
+    }
     try diskFileHandle.close()
   }
 
   func delete() throws {
+    let lock = try lock()
+
+    if try !lock.trylock() {
+      throw RuntimeError.VMIsRunning(name)
+    }
+
     try FileManager.default.removeItem(at: baseURL)
+
+    try lock.unlock()
   }
 
   func accessDate() throws -> Date {
     try baseURL.accessDate()
   }
 
+  func allocatedSizeBytes() throws -> Int {
+    try configURL.allocatedSizeBytes() + diskURL.allocatedSizeBytes() + nvramURL.allocatedSizeBytes()
+  }
+
+  func allocatedSizeGB() throws -> Int {
+    try allocatedSizeBytes() / 1000 / 1000 / 1000
+  }
+
   func sizeBytes() throws -> Int {
     try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes()
   }

Sources/tart/VMStorageHelper.swift

@@ -34,9 +34,15 @@ class VMStorageHelper {
   }
 }
 
+extension NSError {
+  func isFileNotFound() -> Bool {
+    return self.code == NSFileNoSuchFileError || self.code == NSFileReadNoSuchFileError
+  }
+}
+
 extension Error {
   func isFileNotFound() -> Bool {
-    (self as NSError).code == NSFileReadNoSuchFileError
+    (self as NSError).isFileNotFound() || (self as NSError).underlyingErrors.contains(where: { $0.isFileNotFound() })
   }
 }
 
@@ -44,10 +50,12 @@ enum RuntimeError : Error {
   case VMConfigurationError(_ message: String)
   case VMDoesNotExist(name: String)
   case VMMissingFiles(_ message: String)
-  case VMNotRunning(_ message: String)
+  case VMIsRunning(_ name: String)
+  case VMNotRunning(_ name: String)
   case VMAlreadyRunning(_ message: String)
   case NoIPAddressFound(_ message: String)
   case DiskAlreadyInUse(_ message: String)
+  case InvalidDiskSize(_ message: String)
   case FailedToUpdateAccessDate(_ message: String)
   case PIDLockFailed(_ message: String)
   case FailedToParseRemoteName(_ message: String)
@@ -57,6 +65,9 @@ enum RuntimeError : Error {
   case ExportFailed(_ message: String)
   case ImportFailed(_ message: String)
   case SoftnetFailed(_ message: String)
+  case OCIStorageError(_ message: String)
+  case OCIUnsupportedDiskFormat(_ format: String)
+  case SuspendFailed(_ message: String)
 }
 
 protocol HasExitCode {
@@ -72,14 +83,18 @@ extension RuntimeError : CustomStringConvertible {
       return "the specified VM \"\(name)\" does not exist"
     case .VMMissingFiles(let message):
       return message
-    case .VMNotRunning(let message):
+    case .VMIsRunning(let name):
-      return message
+      return "VM \"\(name)\" is running"
+    case .VMNotRunning(let name):
+      return "VM \"\(name)\" is not running"
     case .VMAlreadyRunning(let message):
       return message
     case .NoIPAddressFound(let message):
       return message
     case .DiskAlreadyInUse(let message):
       return message
+    case .InvalidDiskSize(let message):
+      return message
     case .FailedToUpdateAccessDate(let message):
       return message
     case .PIDLockFailed(let message):
@@ -98,6 +113,12 @@ extension RuntimeError : CustomStringConvertible {
       return "VM import failed: \(message)"
     case .SoftnetFailed(let message):
       return "Softnet failed: \(message)"
+    case .OCIStorageError(let message):
+      return "OCI storage error: \(message)"
+    case .OCIUnsupportedDiskFormat(let format):
+      return "OCI disk format \(format) is not supported by this version of Tart"
+    case .SuspendFailed(let message):
+      return "Failed to suspend the VM: \(message)"
     }
   }
 }
@@ -105,6 +126,8 @@ extension RuntimeError : CustomStringConvertible {
 extension RuntimeError : HasExitCode {
   var exitCode: Int32 {
     switch self {
+    case .VMDoesNotExist:
+      return 2
     case .VMNotRunning:
       return 2
     case .VMAlreadyRunning:

Sources/tart/VMStorageLocal.swift

@@ -1,6 +1,6 @@
 import Foundation
 
-class VMStorageLocal {
+class VMStorageLocal: PrunableStorage {
   let baseURL: URL = try! Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)
 
   private func vmURL(_ name: String) -> URL {
@@ -14,7 +14,9 @@ class VMStorageLocal {
   func open(_ name: String) throws -> VMDirectory {
     let vmDir = VMDirectory(baseURL: vmURL(name))
 
-    try vmDir.validate()
+    try vmDir.validate(userFriendlyName: name)
+
+    try vmDir.baseURL.updateAccessDate()
 
     return vmDir
   }
@@ -37,7 +39,7 @@ class VMStorageLocal {
   }
 
   func delete(_ name: String) throws {
-    try FileManager.default.removeItem(at: vmURL(name))
+    try VMDirectory(baseURL: vmURL(name)).delete()
   }
 
   func list() throws -> [(String, VMDirectory)] {
@@ -63,6 +65,10 @@ class VMStorageLocal {
     }
   }
 
+  func prunables() throws -> [Prunable] {
+    try list().map { (_, vmDir) in vmDir }
+  }
+
   func hasVMsWithMACAddress(macAddress: String) throws -> Bool {
     try list().contains { try $1.macAddress() == macAddress }
   }

Sources/tart/VMStorageOCI.swift

@@ -16,10 +16,20 @@ class VMStorageOCI: PrunableStorage {
     VMDirectory(baseURL: vmURL(name)).initialized
   }
 
+  func digest(_ name: RemoteName) throws -> String {
+    let digest = vmURL(name).resolvingSymlinksInPath().lastPathComponent
+
+    if !digest.starts(with: "sha256:") {
+      throw RuntimeError.OCIStorageError("\(name) is not a digest and doesn't point to a digest")
+    }
+
+    return digest
+  }
+
   func open(_ name: RemoteName) throws -> VMDirectory {
     let vmDir = VMDirectory(baseURL: vmURL(name))
 
-    try vmDir.validate()
+    try vmDir.validate(userFriendlyName: name.description)
 
     try vmDir.baseURL.updateAccessDate()
 
@@ -122,7 +132,11 @@ class VMStorageOCI: PrunableStorage {
     try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
   }
 
-  func pull(_ name: RemoteName, registry: Registry) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
+    SentrySDK.configureScope { scope in
+      scope.setContext(value: ["imageName": name], key: "OCI")
+    }
+
     defaultLogger.appendNewLine("pulling manifest...")
 
     let (manifest, manifestData) = try await registry.pullManifest(reference: name.reference.value)
@@ -130,6 +144,12 @@ class VMStorageOCI: PrunableStorage {
     let digestName = RemoteName(host: name.host, namespace: name.namespace,
                                 reference: Reference(digest: Digest.hash(manifestData)))
 
+    if exists(name) && exists(digestName) && linked(from: name, to: digestName) {
+      // optimistically check if we need to do anything at all before locking
+      defaultLogger.appendNewLine("\(digestName) image is already cached and linked!")
+      return
+    }
+
     // Ensure that host directory for given RemoteName exists in OCI storage
     let hostDirectoryURL = hostDirectoryURL(digestName)
     try FileManager.default.createDirectory(at: hostDirectoryURL, withIntermediateDirectories: true)
@@ -150,7 +170,7 @@ class VMStorageOCI: PrunableStorage {
 
     if !exists(digestName) {
       let transaction = SentrySDK.startTransaction(name: name.description, operation: "pull", bindToScope: true)
-      let tmpVMDir = try VMDirectory.temporary()
+      let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description)
 
       // Lock the temporary VM directory to prevent it's garbage collection
       let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
@@ -158,40 +178,17 @@ class VMStorageOCI: PrunableStorage {
 
       // Try to reclaim some cache space if we know the VM size in advance
       if let uncompressedDiskSize = manifest.uncompressedDiskSize() {
-        let requiredCapacityBytes = UInt64(uncompressedDiskSize + 128 * 1024 * 1024)
+        SentrySDK.configureScope { scope in
-
+          scope.setContext(value: ["imageUncompressedDiskSize": uncompressedDiskSize], key: "OCI")
-        let attrs = try Config().tartCacheDir.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey, .volumeAvailableCapacityKey])
-        let capacityImportant = attrs.volumeAvailableCapacityForImportantUsage!
-        let capacityAvailable = attrs.volumeAvailableCapacity!
-        let availableCapacityBytes = max(UInt64(capacityImportant), UInt64(capacityAvailable))
-
-        if capacityImportant == 0 || capacityAvailable == 0 {
-          SentrySDK.capture(message: "Zero capacity") { scope in
-            scope.setLevel(.warning)
-
-            scope.setContext(value: [
-              "volumeAvailableCapacityForImportantUsageKey": capacityImportant,
-              "volumeAvailableCapacityKey": capacityAvailable,
-            ], key: "Attributes")
-          }
         }
 
-        // There is a suspicious that occasionally capacity is returned as zero which can't be true.
+        let otherVMFilesSize: UInt64 = 128 * 1024 * 1024
-        // Let's validate to avoid unnecessary pruning.
-        if 0 < availableCapacityBytes && availableCapacityBytes < requiredCapacityBytes {
-          let transaction = SentrySDK.startTransaction(name: "Automatically Pruning Cache", operation: "prune", bindToScope: true)
-          transaction.setData(value: name, key: "name")
-          transaction.setData(value: uncompressedDiskSize, key: "uncompressedDiskSize")
-          transaction.setData(value: availableCapacityBytes, key: "availableCapacity")
-          transaction.setData(value: requiredCapacityBytes, key: "requiredCapacity")
-          defer { transaction.finish() }
 
-          try Prune.pruneReclaim(reclaimBytes: requiredCapacityBytes - availableCapacityBytes)
+        try Prune.reclaimIfNeeded(uncompressedDiskSize + otherVMFilesSize)
-        }
       }
 
       try await withTaskCancellationHandler(operation: {
-        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest)
+        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
         try move(digestName, from: tmpVMDir)
         transaction.finish()
       }, onCancel: {
@@ -212,10 +209,17 @@ class VMStorageOCI: PrunableStorage {
     }
   }
 
-  func link(from: RemoteName, to: RemoteName) throws {
+  func linked(from: RemoteName, to: RemoteName) -> Bool {
-    if FileManager.default.fileExists(atPath: vmURL(from).path) {
+    do {
-      try FileManager.default.removeItem(at: vmURL(from))
+      let resolvedFrom = try FileManager.default.destinationOfSymbolicLink(atPath: vmURL(from).path)
+      return resolvedFrom == vmURL(to).path
+    } catch {
+      return false
     }
+  }
+
+  func link(from: RemoteName, to: RemoteName) throws {
+    try? FileManager.default.removeItem(at: vmURL(from))
 
     try FileManager.default.createSymbolicLink(at: vmURL(from), withDestinationURL: vmURL(to))
 

Tests/TartTests/DirecotryShareTests.swift

@@ -0,0 +1,74 @@
+import XCTest
+@testable import tart
+
+import Virtualization
+
+final class DirectoryShareTests: XCTestCase {
+  func testNamedParsing() throws {
+    let share = try DirectoryShare(parseFrom: "build:/Users/admin/build")
+    XCTAssertEqual(share.name, "build")
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+  }
+
+  func testNamedReadOnlyParsing() throws {
+    let share = try DirectoryShare(parseFrom: "build:/Users/admin/build:ro")
+    XCTAssertEqual(share.name, "build")
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(share.readOnly)
+  }
+
+  func testOptionalNameParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+  }
+
+  func testOptionalNameReadOnlyParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build:ro")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(share.readOnly)
+  }
+
+  func testMountTagParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build:tag=foo-bar")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+    XCTAssertEqual(share.mountTag, "foo-bar")
+
+    let roShare = try DirectoryShare(parseFrom: "/Users/admin/build:ro,tag=foo-bar")
+    XCTAssertNil(roShare.name)
+    XCTAssertEqual(roShare.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(roShare.readOnly)
+    XCTAssertEqual(roShare.mountTag, "foo-bar")
+
+    let inverseRoShare = try DirectoryShare(parseFrom: "/Users/admin/build:tag=foo-bar,ro")
+    XCTAssertNil(inverseRoShare.name)
+    XCTAssertEqual(inverseRoShare.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(inverseRoShare.readOnly)
+    XCTAssertEqual(inverseRoShare.mountTag, "foo-bar")
+  }
+
+  func testURL() throws {
+    let archiveWithoutNameOrOptions = try DirectoryShare(parseFrom: "https://example.com/archive.tar.gz")
+    XCTAssertNil(archiveWithoutNameOrOptions.name)
+    XCTAssertEqual(archiveWithoutNameOrOptions.path, URL(string: "https://example.com/archive.tar.gz")!)
+    XCTAssertFalse(archiveWithoutNameOrOptions.readOnly)
+    XCTAssertEqual(archiveWithoutNameOrOptions.mountTag, VZVirtioFileSystemDeviceConfiguration.macOSGuestAutomountTag)
+
+    let archiveWithOptions = try DirectoryShare(parseFrom: "https://example.com/archive.tar.gz:ro,tag=sometag")
+    XCTAssertNil(archiveWithOptions.name)
+    XCTAssertEqual(archiveWithOptions.path, URL(string: "https://example.com/archive.tar.gz")!)
+    XCTAssertTrue(archiveWithOptions.readOnly)
+    XCTAssertEqual(archiveWithOptions.mountTag, "sometag")
+
+    let archiveWithNameAndOptions = try DirectoryShare(parseFrom: "somename:https://example.com/archive.tar.gz:ro,tag=sometag")
+    XCTAssertEqual(archiveWithNameAndOptions.name, "somename")
+    XCTAssertEqual(archiveWithNameAndOptions.path, URL(string: "https://example.com/archive.tar.gz")!)
+    XCTAssertTrue(archiveWithNameAndOptions.readOnly)
+    XCTAssertEqual(archiveWithNameAndOptions.mountTag, "sometag")
+  }
+}

Tests/TartTests/DockerConfigTests.swift

@@ -0,0 +1,16 @@
+import XCTest
+@testable import tart
+
+final class DockerConfigTests: XCTestCase {
+  func testHelpers() throws {
+    let config = DockerConfig(credHelpers: [
+      "(.*).dkr.ecr.(.*).amazonaws.com": "ecr-login",
+      "gcr.io": "gcloud"
+    ])
+
+    XCTAssertEqual(try config.findCredHelper(host: "gcr.io"), "gcloud")
+    XCTAssertEqual(try config.findCredHelper(host: "123.dkr.ecr.eu-west-1.amazonaws.com"), "ecr-login")
+    XCTAssertEqual(try config.findCredHelper(host: "456.dkr.ecr.us-east-1.amazonaws.com"), "ecr-login")
+    XCTAssertNil(try config.findCredHelper(host: "ghcr.io"))
+  }
+}

Tests/TartTests/LayerizerTests.swift

@@ -0,0 +1,90 @@
+import XCTest
+@testable import tart
+
+final class LayerizerTests: XCTestCase {
+  var registryRunner: RegistryRunner?
+
+  var registry: Registry {
+    registryRunner!.registry
+  }
+
+  override func setUp() async throws {
+    try await super.setUp()
+
+    do {
+      registryRunner = try await RegistryRunner()
+    } catch {
+      try XCTSkipIf(ProcessInfo.processInfo.environment["CI"] == nil)
+    }
+  }
+
+  override func tearDown() async throws {
+    try await super.tearDown()
+
+    registryRunner = nil
+  }
+
+  func testDiskV1() async throws {
+    // Original disk file to be pushed to the registry
+    let originalDiskFileURL = try fileWithRandomData(sizeBytes: 5 * 1024 * 1024 * 1024)
+    addTeardownBlock {
+      try FileManager.default.removeItem(at: originalDiskFileURL)
+    }
+
+    // Disk file to be pulled from the registry
+    // and compared against the original disk file
+    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+
+    print("pushing disk...")
+    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+
+    print("pulling disk...")
+    try await DiskV1.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
+
+    print("comparing disks...")
+    try XCTAssertEqual(Digest.hash(originalDiskFileURL), Digest.hash(pulledDiskFileURL))
+  }
+
+  func testDiskV2() async throws {
+    // Original disk file to be pushed to the registry
+    let originalDiskFileURL = try fileWithRandomData(sizeBytes: 5 * 1024 * 1024 * 1024)
+    addTeardownBlock {
+      try FileManager.default.removeItem(at: originalDiskFileURL)
+    }
+
+    // Disk file to be pulled from the registry
+    // and compared against the original disk file
+    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+
+    print("pushing disk...")
+    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+
+    print("pulling disk...")
+    try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
+
+    print("comparing disks...")
+    try XCTAssertEqual(Digest.hash(originalDiskFileURL), Digest.hash(pulledDiskFileURL))
+  }
+
+  private func fileWithRandomData(sizeBytes: Int) throws -> URL {
+    let devUrandom = try FileHandle(forReadingFrom: URL(filePath: "/dev/urandom"))
+
+    let temporaryFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+    FileManager.default.createFile(atPath: temporaryFileURL.path, contents: nil)
+    let temporaryFile = try FileHandle(forWritingTo: temporaryFileURL)
+
+    var remainingBytes = sizeBytes
+
+    while remainingBytes > 0 {
+      let randomData = try devUrandom.read(upToCount: min(64 * 1024 * 1024, remainingBytes))!
+      remainingBytes -= randomData.count
+      try temporaryFile.write(contentsOf: randomData)
+    }
+
+    try devUrandom.close()
+
+    try temporaryFile.close()
+
+    return temporaryFileURL
+  }
+}

Tests/TartTests/LeaseTest.swift

@@ -0,0 +1,18 @@
+import XCTest
+@testable import tart
+
+import Network
+import SwiftRadix
+
+final class LeaseTests: XCTestCase {
+  func testCorrectTimezone() throws {
+    let lease = Lease(fromRawLease: [
+      "hw_address": "1,11:22:33:44:55:66",
+      "ip_address": "1.2.3.4",
+      "lease": "0x6565da9e",
+    ])
+
+    XCTAssertNotNil(lease)
+    XCTAssertEqual(lease!.expiresAt.toISO(), "2023-11-28T12:18:38Z")
+  }
+}

Tests/TartTests/LeasesTests.swift

@@ -0,0 +1,61 @@
+import XCTest
+@testable import tart
+
+import Network
+import SwiftDate
+
+final class LeasesTests: XCTestCase {
+  func testNoExpired() throws {
+    let macAddress = MACAddress(fromString: "11:22:33:44:55:66")!
+
+    let leases = try Leases("""
+    {
+      name=whatever
+      ip_address=66.66.66.66
+      hw_address=1,\(macAddress)
+      identifier=1,\(macAddress)
+      lease=\(Int((Date() - 1.seconds).timeIntervalSince1970).hex)
+
+    }
+    {
+      name=whatever
+      ip_address=1.2.3.4
+      hw_address=1,\(macAddress)
+      identifier=1,\(macAddress)
+      lease=\(Int((Date() + 10.minutes).timeIntervalSince1970).hex)
+    }
+    {
+      name=whatever
+      ip_address=66.66.66.66
+      hw_address=1,\(macAddress)
+      identifier=1,\(macAddress)
+      lease=\(Int((Date() - 1.seconds).timeIntervalSince1970).hex)
+    }
+    """)
+
+    XCTAssertEqual(IPv4Address("1.2.3.4"), leases.ResolveMACAddress(macAddress: macAddress))
+  }
+
+  func testDuplicateYetNotExpiredLeases() throws {
+    let macAddress = MACAddress(fromString: "11:22:33:44:55:66")!
+
+    let leases = try Leases("""
+    {
+            name=debian
+            ip_address=192.168.64.1
+            hw_address=1,\(macAddress)
+            identifier=1,\(macAddress)
+            lease=\(Int((Date() + 10.minutes).timeIntervalSince1970).hex)
+    }
+    {
+            name=debian
+            ip_address=192.168.64.2
+            hw_address=1,\(macAddress)
+            identifier=1,\(macAddress)
+            lease=\(Int((Date() + 5.minutes).timeIntervalSince1970).hex)
+    }
+    """)
+
+    XCTAssertEqual(IPv4Address("192.168.64.1"), leases.ResolveMACAddress(macAddress: macAddress))
+  }
+}

Tests/TartTests/MACAddressResolverTests.swift

@@ -8,11 +8,12 @@ final class MACAddressResolverTests: XCTestCase {
     {
       ip_address=1.2.3.4
       hw_address=1,00:11:22:33:44:55
+      lease=0x7fffffff
     }
     """)
 
     XCTAssertEqual(IPv4Address("1.2.3.4"),
-                   try leases.ResolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
+                   leases.ResolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
   }
 
   func testMultipleEntries() throws {
@@ -20,16 +21,18 @@ final class MACAddressResolverTests: XCTestCase {
     {
       ip_address=1.2.3.4
       hw_address=1,00:11:22:33:44:55
+      lease=0x7fffffff
     }
     {
       ip_address=5.6.7.8
       hw_address=1,AA:BB:CC:DD:EE:FF
+      lease=0x7fffffff
     }
     """)
 
     XCTAssertEqual(IPv4Address("1.2.3.4"),
-                   try leases.ResolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
+                   leases.ResolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
     XCTAssertEqual(IPv4Address("5.6.7.8"),
-                   try leases.ResolveMACAddress(macAddress: MACAddress(fromString: "AA:BB:CC:DD:EE:FF")!))
+                   leases.ResolveMACAddress(macAddress: MACAddress(fromString: "AA:BB:CC:DD:EE:FF")!))
   }
 }

docs/.markdownlint.yml

@@ -0,0 +1,12 @@
+"default": true
+"MD002": false # First heading should be a top level heading
+"MD007": # Unordered list indentation
+  indent: 4
+"MD009": false # Trailing spaces
+"MD013": false # Line length
+"MD025": false # Multiple top level headings in the same document
+"MD026": false # Trailing punctuation in heading
+"MD033": false # Inline HTML
+"MD041": false # First line in file should be a top level heading
+"MD045": false # OK not to have a description for an image
+"MD046": false #  Code block style [Expected: fenced; Actual: indented]

docs/blog/.authors.yml

@@ -1,8 +1,9 @@
-edigaryev:
+authors:
-  name: Nikolay Edigaryev
+  edigaryev:
-  description: Creator
+    name: Nikolay Edigaryev
-  avatar: https://github.com/edigaryev.png
+    description: Creator
-fkorotkov:
+    avatar: https://github.com/edigaryev.png
-  name: Fedor Korotkov
+  fkorotkov:
-  description: Creator
+    name: Fedor Korotkov
-  avatar: https://github.com/fkorotkov.png
+    description: Creator
+    avatar: https://github.com/fkorotkov.png