tart set: make --disk-size change idempotent (#698 )

...when the size is not changed.
tart run: disable dynamic display reconfiguration for Linux (#697 )
2026-10-01 19:51:10 +02:00 · 2024-01-08 14:34:13 +00:00 · 2024-01-08 14:25:56 +00:00 · 2024-01-04 01:09:25 +00:00 · 2024-01-04 00:34:55 +04:00 · 2024-01-03 19:48:13 +04:00
124 changed files with 3056 additions and 593 deletions
@@ -5,7 +5,7 @@ set -e
 export VERSION="${CIRRUS_TAG:-0}"

 mkdir -p .ci/pkg/
-cp .build/arm64-apple-macosx/debug/tart .ci/pkg/tart
+cp .build/arm64-apple-macosx/release/tart .ci/pkg/tart
 cp Resources/embedded.provisionprofile .ci/pkg/embedded.provisionprofile
 pkgbuild --root .ci/pkg/ --identifier com.github.cirruslabs.tart --version $VERSION \
  --scripts .ci/pkg/scripts \
@@ -1,11 +1,16 @@
 use_compute_credits: true

+env: 
+  XCODE_TAG: 15
+
 task:
-  name: Test on Ventura
+  name: Test on Sonoma
  alias: test
  persistent_worker:
    labels:
      name: dev-mini
+    resources:
+      tart-vms: 1
  test_script:
    - swift test
  integration_test_script:
@@ -24,11 +29,19 @@ task:
    path: "integration-tests/pytest-junit.xml"
    format: junit

+task:
+  name: Markdown Lint
+  only_if: $CIRRUS_BRANCH != 'gh-pages' && changesInclude('**.md')
+  container:
+    image: node:latest
+  install_script: npm install -g markdownlint-cli
+  lint_script: markdownlint --config=docs/.markdownlint.yml docs/
+
 task:
  name: Lint
  alias: lint
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
  lint_script:
    - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
  always:
@@ -37,24 +50,24 @@ task:
      format: swiftformat

 task:
+  only_if: $CIRRUS_TAG == ''
  name: Build
  alias: build
-  only_if: $CIRRUS_TAG == ''
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
  build_script: swift build --product tart
  sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
  binary_artifacts:
    path: .build/debug/tart

 task:
-  name: Release (Dry Run)
  only_if: $CIRRUS_TAG == '' && ($CIRRUS_USER_PERMISSION == 'write' || $CIRRUS_USER_PERMISSION == 'admin')
+  name: Release (Dry Run)
  depends_on:
    - lint
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -70,8 +83,9 @@ task:
    - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k password101 build.keychain
    - xcrun notarytool store-credentials "notarytool" --apple-id "hello@cirruslabs.org" --team-id "9M2P8L4D89" --password $AC_PASSWORD
  install_script:
-    - brew install go goreleaser/tap/goreleaser-pro getsentry/tools/sentry-cli
+    - brew install go goreleaser/tap/goreleaser-pro
    - brew install mitchellh/gon/gon
+    - curl -sL https://sentry.io/get-cli/ | sh
  info_script:
    - security find-identity -v
    - xcodebuild -version
@@ -80,8 +94,6 @@ task:
  always:
    dist_artifacts:
      path: "dist/*"
-    pkg_artifacts:
-      path: ".ci/*.pkg"

 task:
  name: Release
@@ -91,7 +103,7 @@ task:
    - test
    - build
  macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
  env:
    MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
    AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -118,13 +130,15 @@ task:
    - swift -version
  release_script: goreleaser
  upload_sentry_debug_files_script:
-    - cd .build/arm64-apple-macosx/debug/
+    - cd .build/arm64-apple-macosx/release/
    # Generate and upload symbols
    - dsymutil tart
    - sentry-cli debug-files upload tart.dSYM/
+    - SENTRY_PROJECT=tart sentry-cli debug-files upload tart.dSYM/
    # Bundle and upload sources
    - sentry-cli debug-files bundle-sources tart.dSYM
    - sentry-cli debug-files upload tart.src.zip
+    - SENTRY_PROJECT=tart sentry-cli debug-files upload tart.src.zip
  create_sentry_release_script:
    - export SENTRY_RELEASE="tart@$CIRRUS_TAG"
    - sentry-cli releases new $SENTRY_RELEASE
@@ -3,11 +3,10 @@ project_name: tart
 before:
  hooks:
    - .ci/set-version.sh
-    - swift build -c debug --product tart
-    - xattr -cr .build/arm64-apple-macosx/debug/tart
+    - swift build -c release --product tart
    - gon gon.hcl
-    - codesign --verify --deep --strict .build/arm64-apple-macosx/debug/tart
-    - ./.ci/create-pkg.sh
+    - mkdir -p tart.app/Contents/MacOS
+    - cp .build/arm64-apple-macosx/release/tart tart.app/Contents/MacOS/

 builds:
  - builder: prebuilt
@@ -15,23 +14,20 @@ builds:
      - darwin
    goarch:
      - arm64
+    binary: tart.app/Contents/MacOS/tart
    prebuilt:
-      path: .build/arm64-apple-macosx/debug/tart
+      path: tart.app/Contents/MacOS/tart

 archives:
  - name_template: "{{ .ProjectName }}"
-    format: zip
    files:
      - src: Resources/embedded.provisionprofile
+        dst: tart.app/Contents
        strip_parent: true
-    hooks:
-      after:
-        - xcrun notarytool submit "dist/tart.zip" --keychain-profile "notarytool" --wait
+      - LICENSE

 release:
  prerelease: auto
-  extra_files:
-    - glob: ./.ci/Tart-{{ .Tag }}.pkg

 brews:
  - name: tart
@@ -46,12 +42,10 @@ brews:
    dependencies:
      - "cirruslabs/cli/softnet"
    install: |
-      mkdir_p libexec/"tart.app/Contents/MacOS/"
-      libexec.install "tart" => "tart.app/Contents/MacOS/tart"
-      libexec.install "embedded.provisionprofile" => "tart.app/Contents/embedded.provisionprofile"
+      libexec.install Dir["*"]
      bin.write_exec_script "#{libexec}/tart.app/Contents/MacOS/tart"
    custom_block: |
-      depends_on :macos => :monterey
+      depends_on :macos => :ventura
      
      on_macos do
        unless Hardware::CPU.arm?
@@ -1,17 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="sign debug" type="ShConfigurationType">
-    <option name="SCRIPT_TEXT" value="codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart" />
-    <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
-    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/scripts/sign.sh" />
-    <option name="SCRIPT_OPTIONS" value="" />
-    <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
-    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
-    <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
-    <option name="INTERPRETER_PATH" value="/bin/zsh" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="EXECUTE_IN_TERMINAL" value="true" />
-    <option name="EXECUTE_SCRIPT_FILE" value="false" />
-    <envs />
-    <method v="2" />
-  </configuration>
-</component>
@@ -1,8 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="tart create" type="SwiftPackageManagerRunConfiguration" factoryName="Swift Package Run" PROGRAM_PARAMS="create latest --from-ipsw=latest" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="tart" TARGET_NAME="tart" CONFIG_NAME="tart" RUN_TARGET_PROJECT_NAME="tart" RUN_TARGET_NAME="tart" WAS_MODIFIED="">
-    <method v="2">
-      <option name="SPM.BUILD_TASK_PROVIDER" enabled="true" />
-      <option name="RunConfigurationTask" enabled="true" run_configuration_name="sign debug" run_configuration_type="ShConfigurationType" />
-    </method>
-  </configuration>
-</component>
@@ -1,8 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="tart run" type="SwiftPackageManagerRunConfiguration" factoryName="Swift Package Run" PROGRAM_PARAMS="run latest" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="tart" TARGET_NAME="tart" CONFIG_NAME="tart" RUN_TARGET_PROJECT_NAME="tart" RUN_TARGET_NAME="tart" WAS_MODIFIED="">
-    <method v="2">
-      <option name="SPM.BUILD_TASK_PROVIDER" enabled="true" />
-      <option name="RunConfigurationTask" enabled="true" run_configuration_name="sign debug" run_configuration_type="ShConfigurationType" />
-    </method>
-  </configuration>
-</component>
@@ -0,0 +1,40 @@
+# Contributing to Tart
+
+Table of Contents
+-----------------
+
+- [How to Build](#how-to-build)
+- [How to Create an Issue/Enhancement](#how-to-create-an-issueenhancement)
+- [Style Guidelines](#style-guidelines)
+- [Pull Requests](#Pull-Requests)
+
+## How to Build
+
+1. Fork the repository to your own GitHub account
+2. Clone the forked repository to your local machine
+3. If using Xcode, use from Xcode 15 or newer
+4. Run ./scripts/run-signed.sh from the root of your repository
+
+```bash
+./scripts/run-signed.sh list
+```
+## How to Create an Issue/Enhancement
+
+1. Go to the [Issue page](https://github.com/cirruslabs/tart/issues) of the repository
+2. Click on the "New Issue" button
+3. Provide a descriptive title and detailed description of the issue or enhancement you're suggesting
+4. Submit the issue
+
+## Style Guidelines
+
+1. Code should follow camel case
+2. Code should follow [SwiftFormat](https://github.com/nicklockwood/SwiftFormat#swift-package-manager-plugin) guidelines. You can auto-format the code by running the following command:
+```bash
+swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore .
+```
+
+## Pull Requests
+
+1. Provide a detailed description of the changes you made in the pull request
+2. Wait for pull request to be reviewed 
+3. Make adjustments if necessary
@@ -23,8 +23,8 @@
      "kind" : "remoteSourceControl",
      "location" : "https://github.com/getsentry/sentry-cocoa",
      "state" : {
-        "revision" : "c3c19e29f775ee95b77aa3168f3e2fd6c20deba6",
-        "version" : "7.31.3"
+        "revision" : "d277532e1c8af813981ba01f591b15bbdd735615",
+        "version" : "8.8.0"
      }
    },
    {
@@ -81,6 +81,15 @@
        "version" : "1.0.2"
      }
    },
+    {
+      "identity" : "swift-sysctl",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/sersoft-gmbh/swift-sysctl.git",
+      "state" : {
+        "revision" : "71fd64ee84819bb19fbecfb36d5a4503726b6fb7",
+        "version" : "1.6.0"
+      }
+    },
    {
      "identity" : "swiftdate",
      "kind" : "remoteSourceControl",
@@ -99,6 +108,15 @@
        "version" : "0.50.6"
      }
    },
+    {
+      "identity" : "swiftradix",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/orchetect/SwiftRadix",
+      "state" : {
+        "revision" : "a52c37a4c213403f7377ae77b4c68451bcab8330",
+        "version" : "1.3.1"
+      }
+    },
    {
      "identity" : "texttable",
      "kind" : "remoteSourceControl",
@@ -4,7 +4,7 @@ import PackageDescription
 let package = Package(
  name: "Tart",
  platforms: [
-    .macOS(.v12)
+    .macOS(.v13)
  ],
  products: [
    .executable(name: "tart", targets: ["tart"])
@@ -18,8 +18,10 @@ let package = Package(
    .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
    .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.0.0")),
    .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.50.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "7.31.3"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.8.0"),
    .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
+    .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.0.0"),
+    .package(url: "https://github.com/orchetect/SwiftRadix", from: "1.3.0")
  ],
  targets: [
    .executableTarget(name: "tart", dependencies: [
@@ -32,6 +34,8 @@ let package = Package(
      .product(name: "Atomics", package: "swift-atomics"),
      .product(name: "Sentry", package: "sentry-cocoa"),
      .product(name: "TextTable", package: "TextTable"),
+      .product(name: "Sysctl", package: "swift-sysctl"),
+      .product(name: "SwiftRadix", package: "SwiftRadix"),
    ], exclude: [
      "OCI/Reference/Makefile",
      "OCI/Reference/Reference.g4",
@@ -6,9 +6,18 @@ Built by CI engineers for your automation needs. Here are some highlights of Tar
 * Tart uses Apple's own `Virtualization.Framework` for [near-native performance](https://browser.geekbench.com/v5/cpu/compare/20382844?baseline=20382722).
 * Push/Pull virtual machines from any OCI-compatible container registry.
 * Use Tart Packer Plugin to automate VM creation.
-* Built-in CI integration.
+* Easily integrates with any CI system.

-*Tart* is already adopted by several automation services:
+Tart powers [Cirrus Runners](https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral)
+service — a drop-in replacement for the standard GitHub-hosted runners, offering 2-3 times better performance for a fraction of the price.
+
+<p align="center">
+  <a href="https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/CirrusRunnersForGHA.png" height="65"/>
+  </a>
+</p>
+
+Tart is also adopted by several other automation services:

 <p align="center">
  <a href="https://cirrus-ci.org/guide/macOS/" target=_blank>
@@ -31,24 +40,45 @@ Many more companies are using Tart in their internal setups. Here are a few of t
  <a href="https://krisp.ai/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png" height="65"/>
  </a>
+  <a href="https://mullvad.net/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Mullvad.png" height="65"/>
+  </a>
+  <a href="https://shape.dk/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/shape.png" height="65"/>
+  </a>
  <a href="https://suran.com/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Suran.png" height="65"/>
  </a>
  <a href="https://symflower.com/" target=_blank>
    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Symflower.png" height="65"/>
  </a>
+  <a href="https://transloadit.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png" height="65"/>
+  </a>
+  <a href="https://uphold.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png" height="65"/>
+  </a>
+  <a href="https://www.pitsdatarecovery.net/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
+  </a>
 </p>

 **Note:** If your company or project is using Tart please consider [adding yourself to the list above](/Resources/Users/HowToAddYourself.md).

+<p align="center">
+  <a href="https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws?utm_source=github&utm_medium=referral" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/AWSMarkeplaceLogo.png" height="90"/>
+  </a>
+</p>
+
 ## Usage

-Try running a Tart VM on your Apple Silicon device running macOS 12.0 (Monterey) or later (will download a 25 GB image):
+Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura) or later (will download a 25 GB image):

 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
-tart run ventura-base
+tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+tart run sonoma-base
 ```

 Please check the [official documentation](https://tart.run) for more information and/or feel free to use [discussions](https://github.com/cirruslabs/tart/discussions)
@@ -4,5 +4,7 @@
 <dict>
 	<key>com.apple.security.virtualization</key>
 	<true/>
+	<key>com.apple.security.get-task-allow</key>
+	<true/>
 </dict>
 </plist>
@@ -3,7 +3,20 @@ import Foundation
 import SystemConfiguration

 struct Clone: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Clone a VM")
+  static var configuration = CommandConfiguration(
+    abstract: "Clone a VM",
+    discussion: """
+    Creates a local virtual machine by cloning either a remote or another local virtual machine.
+
+    Due to copy-on-write magic in Apple File System a cloned VM won't actually claim all the space right away.
+    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
+    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
+    will be modified.
+
+    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable. This might be helpful
+    for use cases when the original image is very big and a workload is known to only modify a fraction of the cloned disk.
+    """
+  )

  @Argument(help: "source VM name")
  var sourceName: String
@@ -14,10 +27,17 @@ struct Clone: AsyncParsableCommand {
  @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
  var insecure: Bool = false

+  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+  var concurrency: UInt = 4
+
  func validate() throws {
    if newName.contains("/") {
      throw ValidationError("<new-name> should be a local name")
    }
+
+    if concurrency < 1 {
+      throw ValidationError("network concurrency cannot be less than 1")
+    }
  }

  func run() async throws {
@@ -27,11 +47,10 @@ struct Clone: AsyncParsableCommand {
    if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
      // Pull the VM in case it's OCI-based and doesn't exist locally yet
      let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
    }

    let sourceVM = try VMStorageHelper.open(sourceName)
-
    let tmpVMDir = try VMDirectory.temporary()

    // Lock the temporary VM directory to prevent it's garbage collection
@@ -44,10 +63,17 @@ struct Clone: AsyncParsableCommand {
      try lock.lock()

      let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
+        && sourceVM.state() != "suspended"
      try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)
+
      try localStorage.move(newName, from: tmpVMDir)

      try lock.unlock()
+
+      // APFS is doing copy-on-write so the above cloning operation (just copying files on disk)
+      // is not actually claiming new space until the VM is started and it writes something to disk.
+      // So once we clone the VM let's try to claim a little bit of space for the VM to run.
+      try Prune.reclaimIfNeeded(UInt64(sourceVM.sizeBytes()), sourceVM)
    }, onCancel: {
      try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
    })
@@ -40,18 +40,14 @@ struct Create: AsyncParsableCommand {
        } else if fromIPSW.starts(with: "http://") || fromIPSW.starts(with: "https://") {
          ipswURL = URL(string: fromIPSW)!
        } else {
-          ipswURL = URL(fileURLWithPath: fromIPSW)
+          ipswURL = URL(fileURLWithPath: NSString(string: fromIPSW).expandingTildeInPath)
        }

        _ = try await VM(vmDir: tmpVMDir, ipswURL: ipswURL, diskSizeGB: diskSize)
      }

      if linux {
-        if #available(macOS 13, *) {
-          _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
-        } else {
-          throw UnsupportedOSError("Linux VMs", "are")
-        }
+        _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
      }

      try VMStorageLocal().move(name, from: tmpVMDir)
@@ -1,16 +1,44 @@
 import ArgumentParser
+import Foundation

 struct Export: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Export VM to a file")
+  static var configuration = CommandConfiguration(abstract: "Export VM to a compressed .tvm file")

  @Argument(help: "Source VM name.")
  var name: String

  @Argument(help: "Path to the destination file.")
-  var path: String
+  var path: String?

  func run() async throws {
+    let correctedPath: String
+
+    if let path = path {
+      correctedPath = path
+    } else {
+      correctedPath = "\(name).tvm"
+
+      if FileManager.default.fileExists(atPath: correctedPath) {
+        while true {
+          if userWantsOverwrite(correctedPath) {
+            break
+          } else {
+            return
+          }
+        }
+      }
+    }
+
    print("exporting...")
-    try VMStorageHelper.open(name).exportToArchive(path: path)
+
+    try VMStorageHelper.open(name).exportToArchive(path: correctedPath)
+  }
+
+  func userWantsOverwrite(_ filename: String) -> Bool {
+    print("file \(filename) already exists, are you sure you want to overwrite it? (yes, [no])? ", terminator: "")
+
+    let answer = readLine()!
+
+    return answer == "yes"
  }
 }
@@ -7,6 +7,7 @@ fileprivate struct VMInfo: Encodable {
  let Disk: Int
  let Display: String
  let Running: Bool
+  let State: String
 }

 struct Get: AsyncParsableCommand {
@@ -23,9 +24,9 @@ struct Get: AsyncParsableCommand {
    let vmConfig = try VMConfig(fromURL: vmDir.configURL)
    let diskSizeInGb = try vmDir.sizeGB()
    let memorySizeInMb = vmConfig.memorySize / 1024 / 1024
-    let running = try PIDLock(lockURL: vmDir.configURL).pid() > 0

-    let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: diskSizeInGb, Display: vmConfig.display.description, Running: running)
+    let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: diskSizeInGb,
+                      Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state())
    print(format.renderSingle(info))
  }
 }
@@ -4,6 +4,12 @@ import Network
 import SystemConfiguration
 import Sentry

+enum IPResolutionStrategy: String, ExpressibleByArgument, CaseIterable {
+  case dhcp, arp
+
+  private(set) static var allValueStrings: [String] = Format.allCases.map { "\($0)"}
+}
+
 struct IP: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Get VM's IP address")

@@ -13,46 +19,51 @@ struct IP: AsyncParsableCommand {
  @Option(help: "Number of seconds to wait for a potential VM booting")
  var wait: UInt16 = 0

+  @Option(help: ArgumentHelp("Strategy for resolving IP address: dhcp or arp",
+                             discussion: """
+                             By default, Tart is looking up and parsing DHCP lease file to determine the IP of the VM.\n
+                             This method is fast and the most reliable but only returns local IP adresses.\n
+                             Alternatively, Tart can call external `arp` executable and parse it's output.\n
+                             In case of enabled Bridged Networking this method will return VM's IP address on the network interface used for Bridged Networking.\n
+                             Note that `arp` strategy won't work for VMs using `--net-softnet`.
+                             """))
+  var resolver: IPResolutionStrategy = .dhcp
+
  func run() async throws {
    let vmDir = try VMStorageLocal().open(name)
    let vmConfig = try VMConfig.init(fromURL: vmDir.configURL)
    let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!

-    guard let ipViaDHCP = try await IP.resolveIP(vmMACAddress, secondsToWait: wait) else {
-      throw RuntimeError.NoIPAddressFound("no IP address found, is your VM running?")
-    }
+    guard let ip = try await IP.resolveIP(vmMACAddress, resolutionStrategy: resolver, secondsToWait: wait) else {
+      var message = "no IP address found"

-    let arpCache = try ARPCache()
-
-    if let ipViaARP = try arpCache.ResolveMACAddress(macAddress: vmMACAddress), ipViaARP != ipViaDHCP {
-      // Capture the warning into Sentry
-      SentrySDK.capture(message: "DHCP lease and ARP cache entries for a single MAC address differ") { scope in
-        scope.setLevel(.warning)
-
-        scope.setContext(value: [
-          "MAC address": vmMACAddress,
-          "IP via ARP": ipViaARP,
-          "IP via DHCP": ipViaDHCP,
-        ], key: "Address conflict details")
-
-        scope.add(Attachment(path: "/var/db/dhcpd_leases", filename: "dhcpd_leases.txt", contentType: "text/plain"))
-        scope.add(Attachment(data: arpCache.arpCommandOutput, filename: "arp-an-output.txt", contentType: "text/plain"))
+      if try !vmDir.running() {
+        message += ", is your VM running?"
      }

-      fputs("WARNING: DHCP lease and ARP cache entries for MAC address \(vmMACAddress) differ: "
-        + "got \(ipViaDHCP) and \(ipViaARP) respectively, consider reporting this case to"
-        + " https://github.com/cirruslabs/tart/issues/172\n", stderr)
+      if (vmConfig.os == .linux && resolver == .arp) {
+        message += " (not all Linux distributions are compatible with the ARP resolver)"
+      }
+
+      throw RuntimeError.NoIPAddressFound(message)
    }

-    print(ipViaDHCP)
+    print(ip)
  }

-  static public func resolveIP(_ vmMACAddress: MACAddress, secondsToWait: UInt16) async throws -> IPv4Address? {
+  static public func resolveIP(_ vmMACAddress: MACAddress, resolutionStrategy: IPResolutionStrategy = .dhcp, secondsToWait: UInt16 = 0) async throws -> IPv4Address? {
    let waitUntil = Calendar.current.date(byAdding: .second, value: Int(secondsToWait), to: Date.now)!

    repeat {
-      if let leases = try Leases(), let ip = try leases.resolveMACAddress(macAddress: vmMACAddress) {
-        return ip
+      switch resolutionStrategy {
+      case .arp:
+        if let ip = try ARPCache().ResolveMACAddress(macAddress: vmMACAddress) {
+          return ip
+        }
+      case .dhcp:
+        if let leases = try Leases(), let ip = try leases.ResolveMACAddress(macAddress: vmMACAddress) {
+          return ip
+        }
      }

      // wait a second
@@ -2,7 +2,7 @@ import ArgumentParser
 import Foundation

 struct Import: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Import VM from a file")
+  static var configuration = CommandConfiguration(abstract: "Import VM from a compressed .tvm file")

  @Argument(help: "Path to a file created with \"tart export\".")
  var path: String
@@ -6,6 +6,8 @@ fileprivate struct VMInfo: Encodable {
  let Source: String
  let Name: String
  let Size: Int
+  let Running: Bool
+  let State: String
 }

 struct List: AsyncParsableCommand {
@@ -32,17 +34,19 @@ struct List: AsyncParsableCommand {

  func run() async throws {
    var infos: [VMInfo] = []
+
    if source == nil || source == "local" {
      infos += sortedInfos(try VMStorageLocal().list().map { (name, vmDir) in
-        try VMInfo(Source: "local", Name: name, Size: vmDir.sizeGB())
+        try VMInfo(Source: "local", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running(), State: vmDir.state())
      })
    }

    if source == nil || source == "oci" {
      infos += sortedInfos(try VMStorageOCI().list().map { (name, vmDir, _) in
-        try VMInfo(Source: "oci", Name: name, Size: vmDir.sizeGB())
+        try VMInfo(Source: "oci", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running(), State: vmDir.state())
      })
    }
+
    if (quiet) {
      for info in infos {
        print(info.Name)
@@ -17,6 +17,9 @@ struct Login: AsyncParsableCommand {
  @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
  var insecure: Bool = false

+  @Flag(help: "skip validation of the registry's credentials before logging-in")
+  var noValidate: Bool = false
+
  func validate() throws {
    let usernameProvided = username != nil
    let passwordProvided = passwordStdin
@@ -35,6 +38,9 @@ struct Login: AsyncParsableCommand {

      let passwordData = FileHandle.standardInput.readDataToEndOfFile()
      password = String(decoding: passwordData, as: UTF8.self)
+
+      // Support "echo $PASSWORD | tart login --username $USERNAME --password-stdin $REGISTRY"
+      password.trimSuffix { c in c.isNewline }
    } else {
      (user, password) = try StdinCredentials.retrieve()
    }
@@ -42,12 +48,14 @@ struct Login: AsyncParsableCommand {
      host: (user, password)
    ])

-    do {
-      let registry = try Registry(host: host, namespace: "", insecure: insecure,
-                                  credentialsProviders: [credentialsProvider])
-      try await registry.ping()
-    } catch {
-      throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
+    if !noValidate {
+      do {
+        let registry = try Registry(host: host, namespace: "", insecure: insecure,
+                                    credentialsProviders: [credentialsProvider])
+        try await registry.ping()
+      } catch {
+        throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
+      }
    }

    try KeychainCredentialsProvider().store(host: host, user: user, password: password)
@@ -0,0 +1,14 @@
+import ArgumentParser
+import Dispatch
+import SwiftUI
+
+struct Logout: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(abstract: "Logout from a registry")
+
+  @Argument(help: "host")
+  var host: String
+
+  func run() async throws {
+    try KeychainCredentialsProvider().remove(host: host)
+  }
+}
@@ -5,23 +5,40 @@ import SwiftUI
 import SwiftDate

 struct Prune: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches")
+  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches or local VMs")

-  @Option(help: ArgumentHelp("Remove cache entries last accessed more than n days ago",
+  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."))
+  var entries: String = "caches"
+
+  @Option(help: ArgumentHelp("Remove entries that were last accessed more than n days ago",
                             discussion: "For example, --older-than=7 will remove entries that weren't accessed by Tart in the last 7 days.",
                             valueName: "n"))
  var olderThan: UInt?

-  @Option(help: ArgumentHelp("Remove least recently used cache entries that do not fit the specified cache size budget n, expressed in gigabytes",
-                             discussion: "For example, --cache-budget=50 will effectively shrink all caches to a total size of 50 gigabytes.",
-                             valueName: "n"))
+  @Option(help: .hidden)
  var cacheBudget: UInt?

+  @Option(help: ArgumentHelp("Remove the least recently used entries that do not fit the specified space size budget n, expressed in gigabytes",
+                             discussion: "For example, --space-budget=50 will effectively shrink all entries to a total size of 50 gigabytes.",
+                             valueName: "n"))
+  var spaceBudget: UInt?
+
  @Flag(help: .hidden)
  var gc: Bool = false

-  func validate() throws {
-    if olderThan == nil && cacheBudget == nil && !gc {
+  mutating func validate() throws {
+    // --cache-budget deprecation logic
+    if let cacheBudget = cacheBudget {
+      fputs("--cache-budget is deprecated, please use --space-budget\n", stderr)
+
+      if spaceBudget != nil {
+        throw ValidationError("--cache-budget is deprecated, please use --space-budget")
+      }
+
+      spaceBudget = cacheBudget
+    }
+
+    if olderThan == nil && spaceBudget == nil && !gc {
      throw ValidationError("at least one pruning criteria must be specified")
    }
  }
@@ -31,43 +48,53 @@ struct Prune: AsyncParsableCommand {
      try VMStorageOCI().gc()
    }

+    // Build a list of prunable storages that we're going to prune based on user's request
+    let prunableStorages: [PrunableStorage]
+
+    switch entries {
+    case "caches":
+      prunableStorages = [VMStorageOCI(), try IPSWCache()]
+    case "vms":
+      prunableStorages = [VMStorageLocal()]
+    default:
+      throw ValidationError("unsupported --entries value, please specify either \"caches\" or \"vms\"")
+    }
+
    // Clean up cache entries based on last accessed date
    if let olderThan = olderThan {
      let olderThanInterval = Int(exactly: olderThan)!.days.timeInterval
-      let olderThanDate = Date().addingTimeInterval(olderThanInterval)
+      let olderThanDate = Date() - olderThanInterval

-      try Prune.pruneOlderThan(olderThanDate: olderThanDate)
+      try Prune.pruneOlderThan(prunableStorages: prunableStorages, olderThanDate: olderThanDate)
    }

    // Clean up cache entries based on imposed cache size limit and entry's last accessed date
-    if let cacheBudget = cacheBudget {
-      try Prune.pruneCacheBudget(cacheBudgetBytes: UInt64(cacheBudget) * 1024 * 1024 * 1024)
+    if let spaceBudget = spaceBudget {
+      try Prune.pruneSpaceBudget(prunableStorages: prunableStorages, spaceBudgetBytes: UInt64(spaceBudget) * 1024 * 1024 * 1024)
    }
  }

-  static func pruneOlderThan(olderThanDate: Date) throws {
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
+  static func pruneOlderThan(prunableStorages: [PrunableStorage], olderThanDate: Date) throws {
    let prunables: [Prunable] = try prunableStorages.flatMap { try $0.prunables() }

    try prunables.filter { try $0.accessDate() <= olderThanDate }.forEach { try $0.delete() }
  }

-  static func pruneCacheBudget(cacheBudgetBytes: UInt64) throws {
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
+  static func pruneSpaceBudget(prunableStorages: [PrunableStorage], spaceBudgetBytes: UInt64) throws {
    let prunables: [Prunable] = try prunableStorages
      .flatMap { try $0.prunables() }
      .sorted { try $0.accessDate() > $1.accessDate() }

-    var cacheBudgetBytes = cacheBudgetBytes
+    var spaceBudgetBytes = spaceBudgetBytes
    var prunablesToDelete: [Prunable] = []

    for prunable in prunables {
      let prunableSizeBytes = UInt64(try prunable.sizeBytes())

-      if prunableSizeBytes <= cacheBudgetBytes {
+      if prunableSizeBytes <= spaceBudgetBytes {
        // Don't mark for deletion as
        // there's a budget available
-        cacheBudgetBytes -= prunableSizeBytes
+        spaceBudgetBytes -= prunableSizeBytes
      } else {
        // Mark for deletion
        prunablesToDelete.append(prunable)
@@ -77,7 +104,54 @@ struct Prune: AsyncParsableCommand {
    try prunablesToDelete.forEach { try $0.delete() }
  }

-  static func pruneReclaim(reclaimBytes: UInt64) throws {
+  static func reclaimIfNeeded(_ requiredBytes: UInt64, _ initiator: Prunable? = nil) throws {
+    if ProcessInfo.processInfo.environment.keys.contains("TART_NO_AUTO_PRUNE") {
+      return
+    }
+
+    SentrySDK.configureScope { scope in
+      scope.setContext(value: ["requiredBytes": requiredBytes], key: "Prune")
+    }
+
+    // Figure out how much disk space is available
+    let attrs = try Config().tartCacheDir.resourceValues(forKeys: [
+      .volumeAvailableCapacityKey,
+      .volumeAvailableCapacityForImportantUsageKey
+    ])
+    let volumeAvailableCapacityCalculated = max(
+      UInt64(attrs.volumeAvailableCapacity!),
+      UInt64(attrs.volumeAvailableCapacityForImportantUsage!)
+    )
+
+    SentrySDK.configureScope { scope in
+      scope.setContext(value: [
+        "volumeAvailableCapacity": attrs.volumeAvailableCapacity!,
+        "volumeAvailableCapacityForImportantUsage": attrs.volumeAvailableCapacityForImportantUsage!,
+        "volumeAvailableCapacityCalculated": volumeAvailableCapacityCalculated
+      ], key: "Prune")
+    }
+
+    if volumeAvailableCapacityCalculated <= 0 {
+      SentrySDK.capture(message: "Zero volume capacity reported") { scope in
+        scope.setLevel(.warning)
+      }
+
+      return
+    }
+
+    // Now that we know how much free space is left,
+    // check if we even need to reclaim anything
+    if requiredBytes < volumeAvailableCapacityCalculated {
+      return
+    }
+
+    try Prune.reclaimIfPossible(requiredBytes - volumeAvailableCapacityCalculated, initiator)
+  }
+
+  private static func reclaimIfPossible(_ reclaimBytes: UInt64, _ initiator: Prunable? = nil) throws {
+    let transaction = SentrySDK.startTransaction(name: "Pruning cache", operation: "prune", bindToScope: true)
+    defer { transaction.finish() }
+
    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
    let prunables: [Prunable] = try prunableStorages
      .flatMap { try $0.prunables() }
@@ -98,10 +172,16 @@ struct Prune: AsyncParsableCommand {
        break
      }

-      cacheReclaimedBytes += try prunable.sizeBytes()
-      try prunable.delete()
+      if prunable.url == initiator?.url.resolvingSymlinksInPath() {
+        // do not prune the initiator
+        continue
+      }

-      try SentrySDK.span?.setExtra(value: prunable.sizeBytes(), key: prunable.url.path);
+      try SentrySDK.span?.setData(value: prunable.sizeBytes(), key: prunable.url.path)
+
+      cacheReclaimedBytes += try prunable.sizeBytes()
+
+      try prunable.delete()
    }

    SentrySDK.span?.setMeasurement(name: "gc_disk_reclaimed", value: cacheReclaimedBytes as NSNumber, unit: MeasurementUnitInformation.byte);
@@ -3,7 +3,16 @@ import Dispatch
 import SwiftUI

 struct Pull: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Pull a VM from a registry")
+  static var configuration = CommandConfiguration(
+    abstract: "Pull a VM from a registry",
+    discussion: """
+    Pulls a virtual machine from a remote OCI-compatible registry. Supports authorization via Keychain (see "tart login --help"),
+    Docker credential helpers defined in ~/.docker/config.json or via TART_REGISTRY_USERNAME/TART_REGISTRY_PASSWORD environment variables.
+
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image to fit via "tart prune".
+    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
+    """
+  )

  @Argument(help: "remote VM name")
  var remoteName: String
@@ -11,6 +20,15 @@ struct Pull: AsyncParsableCommand {
  @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
  var insecure: Bool = false

+  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+  var concurrency: UInt = 4
+
+  func validate() throws {
+    if concurrency < 1 {
+      throw ValidationError("network concurrency cannot be less than 1")
+    }
+  }
+
  func run() async throws {
    // Be more liberal when accepting local image as argument,
    // see https://github.com/cirruslabs/tart/issues/36
@@ -25,6 +43,6 @@ struct Pull: AsyncParsableCommand {

    defaultLogger.appendNewLine("pulling \(remoteName)...")

-    try await VMStorageOCI().pull(remoteName, registry: registry)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
  }
 }
@@ -6,7 +6,7 @@ import Compression
 struct Push: AsyncParsableCommand {
  static var configuration = CommandConfiguration(abstract: "Push a VM to a registry")

-  @Argument(help: "local VM name")
+  @Argument(help: "local or remote VM name")
  var localName: String

  @Argument(help: "remote VM name(s)")
@@ -23,12 +23,16 @@ struct Push: AsyncParsableCommand {
                             """))
  var chunkSize: Int = 0

+  @Option(help: .hidden)
+  var diskFormat: String = "v2"
+
  @Flag(help: ArgumentHelp("cache pushed images locally",
                           discussion: "Increases disk usage, but saves time if you're going to pull the pushed images later."))
  var populateCache: Bool = false

  func run() async throws {
-    let localVMDir = try VMStorageLocal().open(localName)
+    let ociStorage = VMStorageOCI()
+    let localVMDir = try VMStorageHelper.open(localName)

    // Parse remote names supplied by the user
    let remoteNames = try remoteNames.map{
@@ -53,23 +57,56 @@ struct Push: AsyncParsableCommand {
      defaultLogger.appendNewLine("pushing \(localName) to "
        + "\(registryIdentifier.host)/\(registryIdentifier.namespace)\(remoteNamesForRegistry.referenceNames())...")

-      let pushedRemoteName = try await localVMDir.pushToRegistry(
-        registry: registry,
-        references: remoteNamesForRegistry.map{ $0.reference.value },
-        chunkSizeMb: chunkSize
-      )
+      let references = remoteNamesForRegistry.map{ $0.reference.value }

-      // Populate the local cache (if requested)
+      let pushedRemoteName: RemoteName
+      // If we're pushing a local OCI VM, check if points to an already existing registry manifest
+      // and if so, only upload manifests (without config, disk and NVRAM) to the user-specified references
+      if let remoteName = try? RemoteName(localName) {
+        pushedRemoteName = try await lightweightPushToRegistry(
+          registry: registry, 
+          remoteName: remoteName,
+          references: references
+        )
+      } else {
+        pushedRemoteName = try await localVMDir.pushToRegistry(
+          registry: registry,
+          references: references,
+          chunkSizeMb: chunkSize,
+          diskFormat: diskFormat
+        )
+        // Populate the local cache (if requested)
+        if populateCache {
+          let expectedPushedVMDir = try ociStorage.create(pushedRemoteName)
+          try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false)
+        }
+      }
+
+      // link the rest remote names
      if populateCache {
-        let ociStorage = VMStorageOCI()
-        let expectedPushedVMDir = try ociStorage.create(pushedRemoteName)
-        try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false)
        for remoteName in remoteNamesForRegistry {
          try ociStorage.link(from: remoteName, to: pushedRemoteName)
        }
      }
    }
  }
+
+  func lightweightPushToRegistry(registry: Registry, remoteName: RemoteName, references: [String]) async throws -> RemoteName {
+    // Is the local OCI VM already present in the registry?
+    let digest = try VMStorageOCI().digest(remoteName)
+
+    let (remoteManifest, _) = try await registry.pullManifest(reference: digest)
+
+    // Overwrite registry's references with the retrieved manifest
+    for reference in references {
+      defaultLogger.appendNewLine("pushing manifest for \(reference)...")
+
+      _ = try await registry.pushManifest(reference: reference, manifest: remoteManifest)
+    }
+
+    return RemoteName(host: registry.host!, namespace: registry.namespace,
+                      reference: Reference(digest: digest))
+  }
 }

 extension Collection where Element == RemoteName {
@@ -1,4 +1,6 @@
 import ArgumentParser
+import Cocoa
+import Darwin
 import Dispatch
 import SwiftUI
 import Virtualization
@@ -17,13 +19,24 @@ struct Run: AsyncParsableCommand {

  @Flag(help: ArgumentHelp(
    "Don't open a UI window.",
-    discussion: "Useful for integrating Tart VMs into other tools.\nUse `tart ip` in order to get an IP for SSHing or VNCing into the VM.")) 
+    discussion: "Useful for integrating Tart VMs into other tools.\nUse `tart ip` in order to get an IP for SSHing or VNCing into the VM."))
  var noGraphics: Bool = false

+  @Flag(help: ArgumentHelp(
+    "Open serial console in /dev/ttySXX",
+    discussion: "Useful for debugging Linux Kernel."))
+  var serial: Bool = false
+
+  @Option(help: ArgumentHelp(
+    "Attach an externally created serial console",
+    discussion: "Alternative to `--serial` flag for programmatic integrations."
+  ))
+  var serialPath: String?
+
  @Flag(help: "Force open a UI window, even when VNC is enabled.")
  var graphics: Bool = false

-  @Flag(help: "Boot into recovery mode") 
+  @Flag(help: "Boot into recovery mode")
  var recovery: Bool = false

  @Flag(help: ArgumentHelp(
@@ -39,10 +52,17 @@ struct Run: AsyncParsableCommand {
  var vncExperimental: Bool = false

  @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\")
+  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\" --disk=\"/dev/disk0\")
  """, discussion: """
+  Can be either a disk image file or a block device like a local SSD on AWS EC2 Mac instances.
+
  Learn how to create a disk image using Disk Utility here:
  https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+
+  To work with block devices 'tart' binary must be executed as root which affects locating Tart VMs.
+  To workaround this issue pass TART_HOME explicitly:
+
+  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
  """, valueName: "path[:ro]"))
  var disk: [String] = []

@@ -61,13 +81,17 @@ struct Run: AsyncParsableCommand {
  var rosettaTag: String?

  @Option(help: ArgumentHelp("""
-  Additional directory shares with an optional read-only specifier\n(e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\")
+  Additional directory shares with an optional read-only specifier\n(e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")
  """, discussion: """
  Requires host to be macOS 13.0 (Ventura) or newer.
-  All shared directories are automatically mounted to "/Volumes/My Shared Files" directory on macOS,
+  A shared directory is automatically mounted to "/Volumes/My Shared Files" directory on macOS,
  while on Linux you have to do it manually: "mount -t virtiofs com.apple.virtio-fs.automount /mount/point".
  For macOS guests, they must be running macOS 13.0 (Ventura) or newer.
-  """, valueName: "name:path[:ro]"))
+
+  In case of passing multiple directories it is required to prefix them with names e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\"
+  These names will be used as directory names under the mounting point inside guests. For the example above it will be
+  "/Volumes/My Shared Files/build" and "/Volumes/My Shared Files/sources" respectively.
+  """, valueName: "[name:]path[:ro]"))
  var dir: [String] = []

  @Option(help: ArgumentHelp("""
@@ -75,28 +99,74 @@ struct Run: AsyncParsableCommand {
  """, discussion: """
  Specify "list" as an interface name (--net-bridged=list) to list the available bridged interfaces.
  """, valueName: "interface name"))
-  var netBridged: String?
+  var netBridged: [String] = []

  @Flag(help: ArgumentHelp("Use software networking instead of the default shared (NAT) networking",
                           discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
  var netSoftnet: Bool = false

-  func validate() throws {
+  @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
+  var suspendable: Bool = false
+
+  @Flag(help: ArgumentHelp("Whether system hot keys should be sent to the guest instead of the host",
+                           discussion: "If enabled then system hot keys like Cmd+Tab will be sent to the guest instead of the host."))
+  var captureSystemKeys: Bool = false
+
+  mutating func validate() throws {
    if vnc && vncExperimental {
      throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
    }
-    if netBridged != nil && netSoftnet {
+
+    if netBridged.count > 0 && netSoftnet {
      throw ValidationError("--net-bridged and --net-softnet are mutually exclusive")
    }

    if graphics && noGraphics {
      throw ValidationError("--graphics and --no-graphics are mutually exclusive")
    }
+
+    if (noGraphics || vnc || vncExperimental) && captureSystemKeys {
+      throw ValidationError("--captures-system-keys can only be used with the default VM view")
+    }
+
+    let localStorage = VMStorageLocal()
+    let vmDir = try localStorage.open(name)
+    if try vmDir.state() == "suspended" {
+      suspendable = true
+    }
+
+    if suspendable {
+      if dir.count > 0 {
+        throw ValidationError("Suspending VMs with shared directories is not supported")
+      }
+    }
+
+    for disk in disk {
+      if disk.hasSuffix("-amd64.iso") {
+        throw ValidationError("Seems you have a disk targeting x86 architecture (hence amd64 in the name). Please use an 'arm64' version of the disk.")
+      }
+    }
  }

  @MainActor
  func run() async throws {
-    let vmDir = try VMStorageLocal().open(name)
+    let localStorage = VMStorageLocal()
+    let vmDir = try localStorage.open(name)
+
+    let storageLock = try FileLock(lockURL: Config().tartHomeDir)
+    if try vmDir.state() == "suspended" {
+      try storageLock.lock() // lock before checking
+      let needToGenerateNewMac = try localStorage.list().contains {
+        // check if there is a running VM with the same MAC but different name
+        try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
+      }
+
+      if needToGenerateNewMac {
+        print("There is already a running VM with the same MAC address!")
+        print("Resetting VM to assign a new MAC address...")
+        try vmDir.regenerateMACAddress()
+      }
+    }

    if netSoftnet && isInteractiveSession() {
      try Softnet.configureSUIDBitIfNeeded()
@@ -104,25 +174,31 @@ struct Run: AsyncParsableCommand {

    let additionalDiskAttachments = try additionalDiskAttachments()

-    // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-    // see https://github.com/cirruslabs/tart/issues/323 for more details.
-    for additionalDiskAttachment in additionalDiskAttachments {
-      // Read-only attachments do not seem to acquire the lock
-      if additionalDiskAttachment.isReadOnly {
-        continue
+    var serialPorts: [VZSerialPortConfiguration] = []
+    if serial {
+      let tty_fd = createPTY()
+      if (tty_fd < 0) {
+        throw RuntimeError.VMConfigurationError("Failed to create PTY")
      }
-
-      if try !FileLock(lockURL: additionalDiskAttachment.url).trylock() {
-        throw RuntimeError.DiskAlreadyInUse("disk \(additionalDiskAttachment.url.path) seems to be already in use, "
-          + "unmount it first in Finder")
+      let tty_read = FileHandle.init(fileDescriptor: tty_fd)
+      let tty_write = FileHandle.init(fileDescriptor: tty_fd)
+      serialPorts.append(createSerialPortConfiguration(tty_read, tty_write))
+    } else if serialPath != nil {
+      let tty_read = FileHandle.init(forReadingAtPath: serialPath!)
+      let tty_write = FileHandle.init(forWritingAtPath: serialPath!)
+      if (tty_read == nil || tty_write == nil) {
+        throw RuntimeError.VMConfigurationError("Failed to open PTY")
      }
+      serialPorts.append(createSerialPortConfiguration(tty_read!, tty_write!))
    }

    vm = try VM(
      vmDir: vmDir,
      network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalDiskAttachments: additionalDiskAttachments,
-      directorySharingDevices: directoryShares() + rosettaDirectoryShare()
+      additionalStorageDevices: additionalDiskAttachments,
+      directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
+      serialPorts: serialPorts,
+      suspendable: suspendable
    )

    let vncImpl: VNC? = try {
@@ -148,13 +224,30 @@ struct Run: AsyncParsableCommand {
    // configuration file, otherwise we will loose the lock.
    //
    // [1]: https://man.openbsd.org/fcntl
-    let lock = try PIDLock(lockURL: vmDir.configURL)
+    let lock = try vmDir.lock()
    if try !lock.trylock() {
      throw RuntimeError.VMAlreadyRunning("VM \"\(name)\" is already running!")
    }

+    // now VM state will return "running" so we can unlock
+    try storageLock.unlock()
+
    let task = Task {
      do {
+        var resume = false
+
+        if #available(macOS 14, *) {
+          if FileManager.default.fileExists(atPath: vmDir.stateURL.path) {
+            print("restoring VM state from a snapshot...")
+            try await vm!.virtualMachine.restoreMachineStateFrom(url: vmDir.stateURL)
+            try FileManager.default.removeItem(at: vmDir.stateURL)
+            resume = true
+            print("resuming VM...")
+          }
+        }
+
+        try await vm!.start(recovery: recovery, resume: resume)
+
        if let vncImpl = vncImpl {
          let vncURL = try await vncImpl.waitForURL()

@@ -166,7 +259,7 @@ struct Run: AsyncParsableCommand {
          }
        }

-        try await vm!.run(recovery)
+        try await vm!.run()

        if let vncImpl = vncImpl {
          try vncImpl.stop()
@@ -178,26 +271,73 @@ struct Run: AsyncParsableCommand {
        SentrySDK.capture(error: error)
        SentrySDK.flush(timeout: 2.seconds.timeInterval)

-        print(error)
+        fputs("\(error)\n", stderr)

        Foundation.exit(1)
      }
    }

+    // "tart stop" support
    let sigintSrc = DispatchSource.makeSignalSource(signal: SIGINT)
    sigintSrc.setEventHandler {
      task.cancel()
    }
    sigintSrc.activate()

+    // "tart suspend" / UI window closing support
+    signal(SIGUSR1, SIG_IGN)
+    let sigusr1Src = DispatchSource.makeSignalSource(signal: SIGUSR1)
+    sigusr1Src.setEventHandler {
+      Task {
+        do {
+          if #available(macOS 14, *) {
+            try vm!.configuration.validateSaveRestoreSupport()
+
+            print("pausing VM to take a snapshot...")
+            try await vm!.virtualMachine.pause()
+
+            print("creating a snapshot...")
+            try await vm!.virtualMachine.saveMachineStateTo(url: vmDir.stateURL)
+
+            print("snapshot created successfully! shutting down the VM...")
+
+            task.cancel()
+          } else {
+            print(RuntimeError.SuspendFailed("this functionality is only supported on macOS 14 (Sonoma) or newer"))
+
+            Foundation.exit(1)
+          }
+        } catch (let e) {
+          print(RuntimeError.SuspendFailed(e.localizedDescription))
+
+          Foundation.exit(1)
+        }
+      }
+    }
+    sigusr1Src.activate()
+
    let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
    if noGraphics || useVNCWithoutGraphics {
-      dispatchMain()
+      // enter the main even loop, without bringing up any UI,
+      // and just wait for the VM to exit.
+      let nsApp = NSApplication.shared
+      nsApp.setActivationPolicy(.prohibited)
+      nsApp.run()
    } else {
-      runUI()
+      runUI(suspendable, captureSystemKeys)
    }
  }

+  private func createSerialPortConfiguration(_ tty_read: FileHandle, _ tty_write: FileHandle) -> VZVirtioConsoleDeviceSerialPortConfiguration {
+    let serialPortConfiguration = VZVirtioConsoleDeviceSerialPortConfiguration()
+    let serialPortAttachment = VZFileHandleSerialPortAttachment(
+      fileHandleForReading: tty_read,
+      fileHandleForWriting: tty_write)
+
+    serialPortConfiguration.attachment = serialPortAttachment
+    return serialPortConfiguration
+  }
+
  func isInteractiveSession() -> Bool {
    isatty(STDOUT_FILENO) == 1
  }
@@ -209,23 +349,19 @@ struct Run: AsyncParsableCommand {
      return try Softnet(vmMACAddress: config.macAddress.string)
    }

-    if let netBridged = netBridged {
-      let matchingInterfaces = VZBridgedNetworkInterface.networkInterfaces.filter { interface in
-        interface.identifier == netBridged || interface.localizedDisplayName == netBridged
+    if netBridged.count > 0 {
+      func findBridgedInterface(_ name: String) throws -> VZBridgedNetworkInterface {
+        let interface = VZBridgedNetworkInterface.networkInterfaces.first { interface in
+          interface.identifier == name || interface.localizedDisplayName == name
+        }
+        if (interface == nil) {
+          throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
+            + "available interfaces: \(bridgeInterfaces())")
+        }
+        return interface!
      }

-      if matchingInterfaces.isEmpty {
-        let available = bridgeInterfaces().joined(separator: ", ")
-        throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
-          + "available interfaces: \(available)")
-      }
-
-      if matchingInterfaces.count > 1 {
-        throw ValidationError("more than one bridge interface matched \"\(netBridged)\", "
-          + "consider refining the search criteria")
-      }
-
-      return NetworkBridged(interface: matchingInterfaces.first!)
+      return NetworkBridged(interfaces: try netBridged.map { try findBridgedInterface($0) })
    }

    return nil
@@ -243,22 +379,43 @@ struct Run: AsyncParsableCommand {
    }
  }

-  func additionalDiskAttachments() throws -> [VZDiskImageStorageDeviceAttachment] {
-    var result: [VZDiskImageStorageDeviceAttachment] = []
+  func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
+    var result: [VZStorageDeviceConfiguration] = []
    let readOnlySuffix = ":ro"
    let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }

    for rawDisk in expandedDiskPaths {
-      if rawDisk.hasSuffix(readOnlySuffix) {
-        result.append(try VZDiskImageStorageDeviceAttachment(
-          url: URL(fileURLWithPath: String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count))),
-          readOnly: true
-        ))
+      let diskReadOnly = rawDisk.hasSuffix(readOnlySuffix)
+      let diskPath = diskReadOnly ? String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count)) : rawDisk
+      let diskURL = URL(fileURLWithPath: diskPath)
+
+      // check if `diskPath` is a block device or a directory
+      if pathHasMode(diskPath, mode: S_IFBLK) || pathHasMode(diskPath, mode: S_IFDIR) {
+        print("Using block device\n")
+        guard #available(macOS 14, *) else {
+          throw UnsupportedOSError("attaching block devices", "are")
+        }
+        let fileHandle = FileHandle(forUpdatingAtPath: diskPath)
+        guard fileHandle != nil else {
+          if ProcessInfo.processInfo.userName != "root" {
+            throw RuntimeError.VMConfigurationError("need to run as root to work with block devices")
+          }
+          throw RuntimeError.VMConfigurationError("block device \(diskURL.url.path) seems to be already in use, unmount it first via 'diskutil unmount'")
+        }
+        let attachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: fileHandle!, readOnly: diskReadOnly, synchronizationMode: .full)
+        result.append(VZVirtioBlockDeviceConfiguration(attachment: attachment))
      } else {
-        result.append(try VZDiskImageStorageDeviceAttachment(
-          url: URL(fileURLWithPath: rawDisk),
-          readOnly: false
-        ))
+        // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
+        // see https://github.com/cirruslabs/tart/issues/323 for more details.
+        if try !diskReadOnly && !FileLock(lockURL: diskURL).trylock() {
+          throw RuntimeError.DiskAlreadyInUse("disk \(diskURL.url.path) seems to be already in use, unmount it first in Finder")
+        }
+
+        let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
+          url: diskURL,
+          readOnly: diskReadOnly
+        )
+        result.append(VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
      }
    }

@@ -274,46 +431,31 @@ struct Run: AsyncParsableCommand {
      throw UnsupportedOSError("directory sharing", "is")
    }

-    struct DirectoryShare {
-      let name: String
-      let path: URL
-      let readOnly: Bool
-    }
-
    var directoryShares: [DirectoryShare] = []

+    var allNamedShares = true
    for rawDir in dir {
-      let splits = rawDir.split(maxSplits: 2) { $0 == ":" }
-
-      if splits.count < 2 {
-        throw ValidationError("invalid --dir syntax: should at least include name and path, colon-separated")
+      let directoryShare = try DirectoryShare(parseFrom: rawDir)
+      if (directoryShare.name == nil) {
+        allNamedShares = false
      }
-
-      var readOnly: Bool = false
-
-      if splits.count == 3 {
-        if splits[2] == "ro" {
-          readOnly = true
-        } else {
-          throw ValidationError("invalid --dir syntax: optional read-only specifier can only be \"ro\"")
-        }
-      }
-
-      let (name, path) = (String(splits[0]), String(splits[1]))
-
-      directoryShares.append(DirectoryShare(
-        name: name,
-        path: URL(fileURLWithPath: NSString(string: path).expandingTildeInPath),
-        readOnly: readOnly)
-      )
+      directoryShares.append(directoryShare)
    }

-    var directories: [String : VZSharedDirectory] = Dictionary()
-    directoryShares.forEach { directories[$0.name] = VZSharedDirectory(url: $0.path, readOnly: $0.readOnly) }

    let automountTag = VZVirtioFileSystemDeviceConfiguration.macOSGuestAutomountTag
    let sharingDevice = VZVirtioFileSystemDeviceConfiguration(tag: automountTag)
-    sharingDevice.share = VZMultipleDirectoryShare(directories: directories)
+    if allNamedShares {
+      var directories: [String : VZSharedDirectory] = Dictionary()
+      try directoryShares.forEach { directories[$0.name!] = try $0.createConfiguration() }
+      sharingDevice.share = VZMultipleDirectoryShare(directories: directories)
+    } else if dir.count > 1 {
+      throw ValidationError("invalid --dir syntax: for multiple directory shares each one of them should be named")
+    } else if dir.count == 1 {
+      let directoryShare = directoryShares.first!
+      let singleDirectoryShare = VZSingleDirectoryShare(directory: try directoryShare.createConfiguration())
+      sharingDevice.share = singleDirectoryShare
+    }

    return [sharingDevice]
  }
@@ -343,7 +485,7 @@ struct Run: AsyncParsableCommand {
    return [device]
  }

-  private func runUI() {
+  private func runUI(_ suspendable: Bool, _ captureSystemKeys: Bool) {
    let nsApp = NSApplication.shared
    nsApp.setActivationPolicy(.regular)
    nsApp.activate(ignoringOtherApps: true)
@@ -351,15 +493,33 @@ struct Run: AsyncParsableCommand {
    nsApp.applicationIconImage = NSImage(data: AppIconData)

    struct MainApp: App {
+      static var disappearSignal: Int32 = SIGINT
+      static var capturesSystemKeys: Bool = false
+
+      @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
+
      var body: some Scene {
        WindowGroup(vm!.name) {
          Group {
-            VMView(vm: vm!).onAppear {
+            VMView(vm: vm!, capturesSystemKeys: MainApp.capturesSystemKeys).onAppear {
              NSWindow.allowsAutomaticWindowTabbing = false
            }.onDisappear {
-              NSApplication.shared.terminate(self)
+              let ret = kill(getpid(), MainApp.disappearSignal)
+              if ret != 0 {
+                // Fallback to the old termination method that doesn't
+                // propagate the cancellation to Task's in case graceful
+                // termination via kill(2) is not successful
+                NSApplication.shared.terminate(self)
+              }
            }
-          }.frame(width: CGFloat(vm!.config.display.width), height: CGFloat(vm!.config.display.height))
+          }.frame(
+            minWidth: CGFloat(vm!.config.display.width),
+            idealWidth: CGFloat(vm!.config.display.width),
+            maxWidth: .infinity,
+            minHeight: CGFloat(vm!.config.display.height),
+            idealHeight: CGFloat(vm!.config.display.height),
+            maxHeight: .infinity
+          )
        }.commands {
          // Remove some standard menu options
          CommandGroup(replacing: .help, addition: {})
@@ -369,23 +529,69 @@ struct Run: AsyncParsableCommand {
          CommandGroup(replacing: .undoRedo, addition: {})
          CommandGroup(replacing: .windowSize, addition: {})
          // Replace some standard menu options
-          CommandGroup(replacing: .appInfo) { AboutTart() }
+          CommandGroup(replacing: .appInfo) { AboutTart(config: vm!.config) }
+          CommandMenu("Control") {
+            Button("Start") {
+              Task { try await vm!.virtualMachine.start() }
+            }
+            Button("Stop") {
+              Task { try await vm!.virtualMachine.stop() }
+            }
+            Button("Request Stop") {
+              Task { try vm!.virtualMachine.requestStop() }
+            }
+            if #available(macOS 14, *) {
+              Button("Suspend") {
+                kill(getpid(), SIGUSR1)
+              }
+            }
+          }
        }
      }
    }

+    MainApp.disappearSignal = suspendable ? SIGUSR1 : SIGINT
+    MainApp.capturesSystemKeys = captureSystemKeys
    MainApp.main()
  }
 }

+// The only way to fully remove Edit menu item.
+class MinimalMenuAppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
+  let indexOfEditMenu = 2
+
+  func applicationDidFinishLaunching(_ : Notification) {
+    NSApplication.shared.mainMenu?.removeItem(at: indexOfEditMenu)
+  }
+}
+
 struct AboutTart: View {
+  var credits: NSAttributedString
+
+  init(config: VMConfig) {
+    let mutableAttrStr = NSMutableAttributedString()
+    let style = NSMutableParagraphStyle()
+    style.alignment = NSTextAlignment.center
+    let attrCenter: [NSAttributedString.Key : Any] = [
+      .paragraphStyle: style,
+    ]
+    mutableAttrStr.append(NSAttributedString(string: "CPU: \(config.cpuCount) cores\n", attributes: attrCenter))
+    mutableAttrStr.append(NSAttributedString(string: "Memory: \(config.memorySize / 1024 / 1024) MB\n", attributes: attrCenter))
+    mutableAttrStr.append(NSAttributedString(string: "Display: \(config.display.description)\n", attributes: attrCenter))
+    mutableAttrStr.append(NSAttributedString(string: "https://github.com/cirruslabs/tart", attributes: [
+      .paragraphStyle: style,
+      .link : "https://github.com/cirruslabs/tart"
+    ]))
+    credits = mutableAttrStr
+  }
+
  var body: some View {
    Button("About Tart") {
      NSApplication.shared.orderFrontStandardAboutPanel(options: [
        NSApplication.AboutPanelOptionKey.applicationIcon: NSApplication.shared.applicationIconImage as Any,
        NSApplication.AboutPanelOptionKey.applicationName: "Tart",
        NSApplication.AboutPanelOptionKey.applicationVersion: CI.version,
-        NSApplication.AboutPanelOptionKey.credits: try! NSAttributedString(markdown: "https://github.com/cirruslabs/tart"),
+        NSApplication.AboutPanelOptionKey.credits: credits,
      ])
    }
  }
@@ -395,10 +601,22 @@ struct VMView: NSViewRepresentable {
  typealias NSViewType = VZVirtualMachineView

  @ObservedObject var vm: VM
+  var capturesSystemKeys: Bool

  func makeNSView(context: Context) -> NSViewType {
    let machineView = VZVirtualMachineView()
-    machineView.capturesSystemKeys = true
+
+    machineView.capturesSystemKeys = capturesSystemKeys
+
+    // Enable automatic display reconfiguration
+    // for guests that support it
+    //
+    // This is disabled for Linux because of poor HiDPI
+    // support, which manifests in fonts being too small
+    if #available(macOS 14.0, *), vm.config.os != .linux {
+      machineView.automaticallyReconfiguresDisplay = true
+    }
+
    return machineView
  }

@@ -406,3 +624,120 @@ struct VMView: NSViewRepresentable {
    nsView.virtualMachine = vm.virtualMachine
  }
 }
+
+struct DirectoryShare {
+  let name: String?
+  let path: URL
+  let readOnly: Bool
+
+  init(parseFrom: String) throws {
+    let readOnlySuffix = ":ro"
+    readOnly = parseFrom.hasSuffix(readOnlySuffix)
+    let maybeNameAndURL = readOnly ? String(parseFrom.dropLast(readOnlySuffix.count)) : parseFrom
+
+    if maybeNameAndURL.starts(with: "https://") || maybeNameAndURL.starts(with: "http://") {
+      // just a URL
+      name = nil
+      path = URL(string: maybeNameAndURL)!
+      return
+    }
+
+    let splits = maybeNameAndURL.split(separator: ":", maxSplits: 1)
+
+    if splits.count == 2 {
+      name = String(splits[0])
+      path = String(splits[1]).toRemoteOrLocalURL()
+    } else {
+      name = nil
+      path = String(splits[0]).toRemoteOrLocalURL()
+    }
+  }
+
+  func createConfiguration() throws -> VZSharedDirectory {
+    if (path.isFileURL) {
+      return VZSharedDirectory(url: path, readOnly: readOnly)
+    }
+
+    let urlCache = URLCache(memoryCapacity: 0, diskCapacity: 1 * 1024 * 1024 * 1024)
+
+    let archiveRequest = URLRequest(url: path, cachePolicy: .returnCacheDataElseLoad)
+    var response: CachedURLResponse? = urlCache.cachedResponse(for: archiveRequest)
+    if (response == nil || response?.data.isEmpty == true) {
+      print("Downloading \(path)...")
+      // download and unarchive remote directories if needed here
+      // use old school API to prevent deadlocks since we are running via MainActor
+      let downloadSemaphore = DispatchSemaphore(value: 0)
+      Task {
+        do {
+          let (archiveData, archiveResponse) = try await URLSession.shared.data(for: archiveRequest)
+          if archiveData.isEmpty {
+            print("Remote archive is empty!")
+          } else {
+            urlCache.storeCachedResponse(CachedURLResponse(response: archiveResponse, data: archiveData, storagePolicy: .allowed), for: archiveRequest)
+            print("Cached for future invocations!")
+          }
+        } catch {
+          print("Download failed: \(error)")
+        }
+        downloadSemaphore.signal()
+      }
+      downloadSemaphore.wait()
+      response = urlCache.cachedResponse(for: archiveRequest)
+    } else {
+      print("Using cached archive for \(path)...")
+    }
+
+    if (response == nil) {
+      throw ValidationError("Failed to fetch a remote archive!")
+    }
+
+    let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".volume")
+    try FileManager.default.createDirectory(atPath: temporaryLocation.path, withIntermediateDirectories: true)
+    let lock = try FileLock(lockURL: temporaryLocation)
+    try lock.lock()
+
+    guard let executableURL = resolveBinaryPath("tar") else {
+      throw ValidationError("tar not found in PATH")
+    }
+
+    let process = Process.init()
+    process.executableURL = executableURL
+    process.currentDirectoryURL = temporaryLocation
+    process.arguments = ["-xz"]
+
+    let inPipe = Pipe()
+    process.standardInput = inPipe
+    process.launch()
+
+    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.close()
+    process.waitUntilExit()
+
+    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
+      throw ValidationError("Unarchiving failed!")
+    }
+
+    print("Unarchived into a temporary directory!")
+
+    return VZSharedDirectory(url: temporaryLocation, readOnly: readOnly)
+  }
+}
+
+extension String {
+  func toRemoteOrLocalURL() -> URL {
+    if (starts(with: "https://") || starts(with: "https://")) {
+      URL(string: self)!
+    } else {
+      URL(fileURLWithPath: NSString(string: self).expandingTildeInPath)
+    }
+  }
+}
+
+func pathHasMode(_ path: String, mode: mode_t) -> Bool {
+  var st = stat()
+  let statRes = stat(path, &st)
+  guard statRes != -1 else {
+    return false
+  }
+  return (Int32(st.st_mode) & Int32(mode)) == Int32(mode)
+}
@@ -16,7 +16,18 @@ struct Set: AsyncParsableCommand {
  @Option(help: "VM display resolution in a format of <width>x<height>. For example, 1200x800")
  var display: VMDisplayConfig?

-  @Option(help: .hidden)
+  @Option(help: ArgumentHelp("Resize the VMs disk to the specified size in GB (note that the disk size can only be increased to avoid losing data)",
+                             discussion: """
+                             Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images
+                             have the \"cloud-initramfs-growroot\" package installed that runs on boot) and on the rest of the
+                             distributions by running the \"growpart\" or \"resize2fs\" commands.
+
+                             For macOS, however, things are a bit more complicated: you need to remove the recovery partition
+                             first and then run various \"diskutil\" commands, see Tart's packer plugin source code for more
+                             details[1].
+
+                             [1]: https://github.com/cirruslabs/packer-plugin-tart/blob/main/builder/tart/step_disk_resize.go
+                             """))
  var diskSize: UInt16?

  func run() async throws {
@@ -14,12 +14,27 @@ struct Stop: AsyncParsableCommand {

  func run() async throws {
    let vmDir = try VMStorageLocal().open(name)
-    let lock = try PIDLock(lockURL: vmDir.configURL)
+    switch try vmDir.state() {
+    case "suspended":
+      try stopSuspended(vmDir)
+    case "running":
+      try await stopRunning(vmDir)
+    default:
+      return
+    }
+  }
+
+  func stopSuspended(_ vmDir: VMDirectory) throws {
+    try? FileManager.default.removeItem(at: vmDir.stateURL)
+  }
+
+  func stopRunning(_ vmDir: VMDirectory) async throws {
+    let lock = try vmDir.lock()

    // Find the VM's PID
    var pid = try lock.pid()
    if pid == 0 {
-      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
+      throw RuntimeError.VMNotRunning(name)
    }

    // Try to gracefully terminate the VM
@@ -0,0 +1,28 @@
+import ArgumentParser
+import Foundation
+import System
+import SwiftDate
+
+struct Suspend: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(commandName: "suspend", abstract: "Suspend a VM")
+
+  @Argument(help: "VM name")
+  var name: String
+
+  func run() async throws {
+    let vmDir = try VMStorageLocal().open(name)
+    let lock = try vmDir.lock()
+
+    // Find the VM's PID
+    var pid = try lock.pid()
+    if pid == 0 {
+      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
+    }
+
+    // Tell the "tart run" process to suspend the VM
+    let ret = kill(pid, SIGUSR1)
+    if ret != 0 {
+      throw RuntimeError.SuspendFailed("failed to send SIGUSR1 signal to the \"tart run\" process running VM \"\(name)\"")
+    }
+  }
+}
@@ -11,7 +11,7 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
    if let credentialsFromAuth = config.auths?[host]?.decodeCredentials() {
      return credentialsFromAuth
    }
-    if let helperProgram = config.credHelpers?[host] {
+    if let helperProgram = try config.findCredHelper(host: host) {
      return try executeHelper(binaryName: "docker-credential-\(helperProgram)", host: host)
    }

@@ -41,13 +41,18 @@ class DockerConfigCredentialsProvider: CredentialsProvider {

    process.waitUntilExit()

+    let outputData = try outPipe.fileHandleForReading.readToEnd()
    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
+      if let outputData = outputData {
+        print(String(decoding: outputData, as: UTF8.self))
+      }
      throw CredentialsProviderError.Failed(message: "Docker helper failed!")
    }
+    if outputData == nil || outputData?.count == 0 {
+      throw CredentialsProviderError.Failed(message: "Docker helper output is empty!")
+    }

-    let getOutput = try JSONDecoder().decode(
-      DockerGetOutput.self, from: outPipe.fileHandleForReading.readDataToEndOfFile()
-    )
+    let getOutput = try JSONDecoder().decode(DockerGetOutput.self, from: outputData!)
    return (getOutput.Username, getOutput.Secret)
  }

@@ -59,6 +64,26 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 struct DockerConfig: Codable {
  var auths: Dictionary<String, DockerAuthConfig>? = Dictionary()
  var credHelpers: Dictionary<String, String>? = Dictionary()
+
+  func findCredHelper(host: String) throws -> String? {
+    // Tart supports wildcards in credHelpers
+    // Similar to what is requested from Docker: https://github.com/docker/cli/issues/2928
+
+    guard let credHelpers else {
+      return nil
+    }
+
+    for (hostPattern, helperProgram) in credHelpers {
+      if (hostPattern == host) {
+        return helperProgram
+      }
+      let compiledPattern = try? Regex(hostPattern)
+      if (try compiledPattern?.wholeMatch(in: host) != nil) {
+        return helperProgram
+      }
+    }
+    return nil
+  }
 }

 struct DockerAuthConfig: Codable {
@@ -2,6 +2,11 @@ import Foundation

 class EnvironmentCredentialsProvider: CredentialsProvider {
  func retrieve(host: String) throws -> (String, String)? {
+    if let tartRegistryHostname = ProcessInfo.processInfo.environment["TART_REGISTRY_HOSTNAME"],
+       tartRegistryHostname != host {
+      return nil
+    }
+
    let username = ProcessInfo.processInfo.environment["TART_REGISTRY_USERNAME"]
    let password = ProcessInfo.processInfo.environment["TART_REGISTRY_PASSWORD"]
    if let username = username, let password = password {
@@ -61,6 +61,24 @@ class KeychainCredentialsProvider: CredentialsProvider {
      throw CredentialsProviderError.Failed(message: "Keychain failed to find item: \(status.explanation())")
    }
  }
+
+  func remove(host: String) throws {
+    let query: [String: Any] = [kSecClass as String: kSecClassInternetPassword,
+                                kSecAttrServer as String: host,
+                                kSecAttrLabel as String: "Tart Credentials",
+    ]
+
+    let status = SecItemDelete(query as CFDictionary)
+
+    switch status {
+    case errSecSuccess:
+      return
+    case errSecItemNotFound:
+      return
+    default:
+      throw CredentialsProviderError.Failed(message: "Failed to remove Keychain item(s): \(status.explanation())")
+    }
+  }
 }

 extension OSStatus {
@@ -13,7 +13,7 @@ class StdinCredentials {
    return (user, password)
  }

-  private static func readStdinCredential(name: String, prompt: String, maxCharacters: Int = 255, isSensitive: Bool) throws -> String {
+  private static func readStdinCredential(name: String, prompt: String, maxCharacters: Int = 1024, isSensitive: Bool) throws -> String {
    var buf = [CChar](repeating: 0, count: maxCharacters + 1 /* sentinel */ + 1 /* NUL */)
    guard let rawCredential = readpassphrase(prompt, &buf, buf.count, isSensitive ? RPP_ECHO_OFF : RPP_ECHO_ON) else {
      throw StdinCredentialsError.CredentialRequired(which: name)
@@ -0,0 +1,37 @@
+import Foundation
+import Sysctl
+
+class DeviceInfo {
+  private static var osMemoized: String? = nil
+  private static var modelMemoized: String? = nil
+
+  static var os: String {
+    if let os = osMemoized {
+      return os
+    }
+
+    osMemoized = getOS()
+
+    return osMemoized!
+  }
+
+  static var model: String {
+    if let model = modelMemoized {
+      return model
+    }
+
+    modelMemoized = getModel()
+
+    return modelMemoized!
+  }
+
+  private static func getOS() -> String {
+    let osVersion = ProcessInfo.processInfo.operatingSystemVersion
+
+    return "macOS \(osVersion.majorVersion).\(osVersion.minorVersion).\(osVersion.patchVersion)"
+  }
+
+  private static func getModel() -> String {
+    return SystemControl().hardware.model
+  }
+}
@@ -35,12 +35,12 @@ class Fetcher {
    //
    // This keeps a working reference to that file, yet we don't
    // have to deal with the cleanup any more.
-    let fh = try FileHandle(forReadingFrom: fileURL)
+    let mappedFile = try Data(contentsOf: fileURL, options: [.alwaysMapped])
    try FileManager.default.removeItem(at: fileURL)

    Task {
-      while let data = try fh.read(upToCount: 64 * 1024 * 1024) {
-        await dataCh.send(data)
+      for chunk in (0 ..< mappedFile.count).chunks(ofCount: 64 * 1024 * 1024) {
+        await dataCh.send(mappedFile.subdata(in: chunk))
      }

      dataCh.finish()
@@ -26,10 +26,16 @@ enum Format: String, ExpressibleByArgument, CaseIterable {
      }
      let table = TextTable<T> { (item: T) in
        let mirroredObject = Mirror(reflecting: item)
-        return mirroredObject.children.enumerated().map { (_, element) in
-          let fieldName = element.label!
-          return Column(title: fieldName, value: element.value)
-        }
+        return mirroredObject.children.enumerated()
+          .filter {(_, element) in
+            // Deprecate the "Running" field: only make it available
+            // from JSON for backwards-compatibility
+            element.label! != "Running"
+          }
+          .map { (_, element) in
+            let fieldName = element.label!
+            return Column(title: fieldName, value: element.value)
+          }
      }
      return table.string(for: data, style: Style.plain)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
    case .json:
@@ -67,7 +67,7 @@ struct ARPCache {
    self.arpCommandOutput = arpCommandOutput
  }

-  func ResolveMACAddress(macAddress: MACAddress, bridgeOnly: Bool = true) throws -> IPv4Address? {
+  func ResolveMACAddress(macAddress: MACAddress) throws -> IPv4Address? {
    let lines = String(decoding: arpCommandOutput, as: UTF8.self)
      .trimmingCharacters(in: .whitespacesAndNewlines)
      .components(separatedBy: "\n")
@@ -95,11 +95,6 @@ struct ARPCache {
        throw ARPCommandYieldedInvalidOutputError(explanation: "failed to parse MAC address \(rawMAC)")
      }

-      let interface = try match.getCaptureGroup(name: "interface", for: line)
-      if bridgeOnly && !interface.starts(with: "bridge") {
-        continue
-      }
-
      if macAddress == mac {
        return ip
      }
@@ -1,13 +1,17 @@
+import Foundation
 import Network
+import SwiftRadix

 struct Lease {
  var mac: MACAddress
  var ip: IPv4Address
+  var expiresAt: Date

  init?(fromRawLease: [String : String]) {
    // Retrieve the required fields
    guard let hwAddress = fromRawLease["hw_address"] else { return nil }
    guard let ipAddress = fromRawLease["ip_address"] else { return nil }
+    guard let lease = fromRawLease["lease"] else { return nil }

    // Parse MAC address
    let hwAddressSplits = hwAddress.split(separator: ",")
@@ -26,7 +30,13 @@ struct Lease {
      return nil
    }

+    // Parse expiration timestamp
+    guard let leaseTimestamp = lease.hex?.value else {
+      return nil
+    }
+
    self.ip = ip
    self.mac = mac
+    self.expiresAt = Date(timeIntervalSince1970: TimeInterval(leaseTimestamp))
  }
 }
@@ -37,13 +37,15 @@ class Leases {
  }

  init(_ fromString: String) throws {
-    var leases: [MACAddress : Lease] = Dictionary()
+    let leases = try Self.retrieveRawLeases(fromString).compactMap({ rawLease in
+      Lease(fromRawLease: rawLease)
+    }).filter({ lease in
+      lease.expiresAt.isInFuture
+    }).map({ lease in
+      (lease.mac, lease)
+    })

-    for lease in try Self.retrieveRawLeases(fromString).compactMap({ Lease(fromRawLease: $0) }) {
-      leases[lease.mac] = lease
-    }
-
-    self.leases = leases
+    self.leases = Dictionary(uniqueKeysWithValues: leases)
  }

  /// Parse leases from the host cache similarly to the PLCache_read() function found in Apple's Open Source releases.
@@ -107,7 +109,7 @@ class Leases {
    return rawLeases
  }

-  func resolveMACAddress(macAddress: MACAddress) throws -> IPv4Address? {
+  func ResolveMACAddress(macAddress: MACAddress) -> IPv4Address? {
    leases[macAddress]?.ip
  }
 }
@@ -1,7 +1,7 @@
 import Virtualization

 protocol Network {
-  func attachment() -> VZNetworkDeviceAttachment
+  func attachments() -> [VZNetworkDeviceAttachment]
  func run(_ sema: DispatchSemaphore) throws
  func stop() async throws
 }
@@ -2,14 +2,14 @@ import Foundation
 import Virtualization

 class NetworkBridged: Network {
-  let interface: VZBridgedNetworkInterface
+  let interfaces: [VZBridgedNetworkInterface]

-  init(interface: VZBridgedNetworkInterface) {
-    self.interface = interface
+  init(interfaces: [VZBridgedNetworkInterface]) {
+    self.interfaces = interfaces
  }

-  func attachment() -> VZNetworkDeviceAttachment {
-    VZBridgedNetworkDeviceAttachment(interface: interface)
+  func attachments() -> [VZNetworkDeviceAttachment] {
+    interfaces.map { VZBridgedNetworkDeviceAttachment(interface: $0) }
  }

  func run(_ sema: DispatchSemaphore) throws {
@@ -2,8 +2,8 @@ import Foundation
 import Virtualization

 class NetworkShared: Network {
-  func attachment() -> VZNetworkDeviceAttachment {
-    VZNATNetworkDeviceAttachment()
+  func attachments() -> [VZNetworkDeviceAttachment] {
+    [VZNATNetworkDeviceAttachment()]
  }

  func run(_ sema: DispatchSemaphore) throws {
@@ -92,9 +92,9 @@ class Softnet: Network {
    }
  }

-  func attachment() -> VZNetworkDeviceAttachment {
+  func attachments() -> [VZNetworkDeviceAttachment] {
    let fh = FileHandle.init(fileDescriptor: vmFD)
-    return VZFileHandleNetworkDeviceAttachment(fileHandle: fh)
+    return [VZFileHandleNetworkDeviceAttachment(fileHandle: fh)]
  }

  static func configureSUIDBitIfNeeded() throws {
@@ -0,0 +1,25 @@
+import Foundation
+
+actor AuthenticationKeeper {
+  var authentication: Authentication? = nil
+
+  func set(_ authentication: Authentication) {
+    self.authentication = authentication
+  }
+
+  func header() -> (String, String)? {
+    if let authentication = authentication {
+      // Do not suggest any headers if the
+      // authentication token has expired
+      if !authentication.isValid() {
+        return nil
+      }
+
+      return authentication.header()
+    }
+
+    // Do not suggest any headers if the
+    // authentication token is not set
+    return nil
+  }
+}
@@ -1,6 +1,11 @@
 import Foundation
 import CryptoKit

+enum DigestError: Error {
+  case InvalidOffset
+  case InvalidSize
+}
+
 class Digest {
  var hash: SHA256 = SHA256()

@@ -15,6 +20,37 @@ class Digest {
  static func hash(_ data: Data) -> String {
    SHA256.hash(data: data).hexdigest()
  }
+
+  static func hash(_ url: URL) throws -> String {
+    hash(try Data(contentsOf: url))
+  }
+
+  static func hash(_ url: URL, offset: UInt64, size: UInt64) throws -> String {
+    // Sanity check
+    let fhSanity = try FileHandle(forReadingFrom: url)
+    try fhSanity.seekToEnd()
+    let fileSize = try fhSanity.offset()
+    try fhSanity.close()
+
+    if offset > fileSize {
+      throw DigestError.InvalidOffset
+    }
+
+    if (offset + size) > fileSize {
+      throw DigestError.InvalidSize
+    }
+
+    // Read a chunk of size ``size`` at offset ``offset``
+    // and calculate it's digest
+    let fh = try FileHandle(forReadingFrom: url)
+    defer { try! fh.close() }
+
+    try fh.seek(toOffset: offset)
+
+    let data = try fh.read(upToCount: Int(size))!
+
+    return hash(data)
+  }
 }

 extension SHA256.Digest {
@@ -0,0 +1,6 @@
+import Foundation
+
+protocol Disk {
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer]
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws
+}
@@ -0,0 +1,75 @@
+import Foundation
+import Compression
+
+class DiskV1: Disk {
+  private static let bufferSizeBytes = 4 * 1024 * 1024
+  private static let layerLimitBytes = 500 * 1000 * 1000
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [OCIManifestLayer] = []
+
+    // Open the disk file
+    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+    var mappedDiskReadOffset = 0
+
+    // Compress the disk file as a single stream
+    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
+      // Determine the size of the next chunk
+      let bytesRead = min(length, mappedDisk.count - mappedDiskReadOffset)
+
+      // Read the next uncompressed chunk
+      let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
+
+      // Advance the offset
+      mappedDiskReadOffset += bytesRead
+
+      // Provide the uncompressed chunk to the compressing filter
+      return data
+    }
+
+    // Cut the compressed stream into layers, each equal exactly ``Self.layerLimitBytes`` bytes,
+    // except for the last one, which may be smaller
+    while let compressedData = try compressingFilter.readData(ofLength: Self.layerLimitBytes) {
+      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+
+      pushedLayers.append(OCIManifestLayer(
+        mediaType: diskV1MediaType,
+        size: compressedData.count,
+        digest: layerDigest
+      ))
+
+      // Update progress using an absolute value
+      progress.completedUnitCount = Int64(mappedDiskReadOffset)
+    }
+
+    return pushedLayers
+  }
+
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+      throw OCIError.FailedToCreateVmFile
+    }
+
+    // Open the disk file
+    let disk = try FileHandle(forWritingTo: diskURL)
+    defer { try! disk.close() }
+
+    // Decompress the layers onto the disk in a single stream
+    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
+      if let data = data {
+        disk.write(data)
+      }
+    }
+
+    for diskLayer in diskLayers {
+      try await registry.pullBlob(diskLayer.digest) { data in
+        try filter.write(data)
+
+        // Update the progress
+        progress.completedUnitCount += Int64(data.count)
+      }
+    }
+
+    try filter.finalize()
+  }
+}
@@ -0,0 +1,188 @@
+import Foundation
+import Compression
+
+class DiskV2: Disk {
+  private static let bufferSizeBytes = 4 * 1024 * 1024
+  private static let layerLimitBytes = 500 * 1000 * 1000
+  private static let holeGranularityBytes = 64 * 1024
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [OCIManifestLayer] = []
+
+    // Open the disk file
+    var mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+
+    // Compress the disk file as multiple individually decompressible streams,
+    // each equal ``Self.layerLimitBytes`` bytes or slightly larger due to the
+    // internal compressor's buffer
+    var offset: UInt64 = 0
+
+    while let (compressedData, uncompressedSize, uncompressedDigest) = try compressNextLayerOfLimitBytesOrMore(mappedDisk: mappedDisk, offset: offset) {
+      offset += uncompressedSize
+
+      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+
+      pushedLayers.append(OCIManifestLayer(
+        mediaType: diskV2MediaType,
+        size: compressedData.count,
+        digest: layerDigest,
+        uncompressedSize: uncompressedSize,
+        uncompressedContentDigest: uncompressedDigest
+      ))
+
+      // Update progress using a relative value
+      progress.completedUnitCount += Int64(uncompressedSize)
+    }
+
+    return pushedLayers
+  }
+
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+    // Support resumable pulls
+    let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
+
+    if !pullResumed && !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+      throw OCIError.FailedToCreateVmFile
+    }
+
+    // Calculate the uncompressed disk size
+    var uncompressedDiskSize: UInt64 = 0
+
+    for layer in diskLayers {
+      guard let uncompressedLayerSize = layer.uncompressedSize() else {
+        throw OCIError.LayerIsMissingUncompressedSizeAnnotation
+      }
+
+      uncompressedDiskSize += uncompressedLayerSize
+    }
+
+    // Truncate the target disk file so that it will be able
+    // to accomodate the uncompressed disk size
+    let disk = try FileHandle(forWritingTo: diskURL)
+    try disk.truncate(atOffset: uncompressedDiskSize)
+    try disk.close()
+
+    // Concurrently fetch and decompress layers
+    try await withThrowingTaskGroup(of: Void.self) { group in
+      var globalDiskWritingOffset: UInt64 = 0
+
+      for (index, diskLayer) in diskLayers.enumerated() {
+        // Respect the concurrency limit
+        if index >= concurrency {
+          try await group.next()
+        }
+
+        // Retrieve layer annotations
+        guard let uncompressedLayerSize = diskLayer.uncompressedSize() else {
+          throw OCIError.LayerIsMissingUncompressedSizeAnnotation
+        }
+        guard let uncompressedLayerContentDigest = diskLayer.uncompressedContentDigest() else {
+          throw OCIError.LayerIsMissingUncompressedDigestAnnotation
+        }
+
+        // Capture the current disk writing offset
+        let diskWritingOffset = globalDiskWritingOffset
+
+        // Launch a fetching and decompression task
+        group.addTask {
+          // No need to fetch and decompress anything if we've already done so
+          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+            // Update the progress
+            progress.completedUnitCount += Int64(diskLayer.size)
+
+            return
+          }
+
+          // Open the disk file
+          let disk = try FileHandle(forWritingTo: diskURL)
+
+          // A zero chunk for faster than byte-by-byte comparisons
+          //
+          // Assumes that the other Data(...) is equal in size, but it's fine to get a false-negative
+          // on the last block since it costs only 64 KiB of excess data per 500 MB layer.
+          //
+          // Some simple benchmarks ("sync && sudo purge" command was used to negate the disk caching effects):
+          // +--------------------------------------+---------------------------------------------------+
+          // | Operation                            | time(1) result                                    |
+          // +--------------------------------------+---------------------------------------------------+
+          // | Data(...) == zeroChunk               | 2.16s user 11.71s system 73% cpu 18.928 total     |
+          // | Data(...).contains(where: {$0 != 0}) | 603.68s user 12.97s system 99% cpu 10:22.85 total |
+          // +--------------------------------------+---------------------------------------------------+
+          let zeroChunk = Data(count: holeGranularityBytes)
+          var diskWritingOffset = diskWritingOffset
+
+          // Pull and decompress a single layer into the specific offset on disk
+          let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
+            guard let data = data else {
+              return
+            }
+
+            for chunk in data.chunks(ofCount: holeGranularityBytes) {
+              // Only write chunks that are not zero
+              if chunk != zeroChunk {
+                try disk.seek(toOffset: diskWritingOffset)
+                disk.write(chunk)
+              }
+
+              diskWritingOffset += UInt64(chunk.count)
+            }
+          }
+
+          try await registry.pullBlob(diskLayer.digest) { data in
+            try filter.write(data)
+
+            // Update the progress
+            progress.completedUnitCount += Int64(data.count)
+          }
+
+          try filter.finalize()
+
+          try disk.close()
+        }
+
+        globalDiskWritingOffset += uncompressedLayerSize
+      }
+    }
+  }
+
+  private static func compressNextLayerOfLimitBytesOrMore(mappedDisk: Data, offset: UInt64) throws -> (Data, UInt64, String)? {
+    var compressedData = Data()
+    var bytesRead: UInt64 = 0
+    let digest = Digest()
+
+    // Create a compressing filter that we will terminate upon
+    // reaching ``Self.layerLimitBytes`` of compressed data
+    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: bufferSizeBytes) { (length: Int) -> Data? in
+      if compressedData.count >= Self.layerLimitBytes {
+        return nil
+      }
+
+      let readFromByte = Int(offset + bytesRead)
+
+      let numBytesToRead = min(mappedDisk.count - readFromByte, bufferSizeBytes)
+      if numBytesToRead == 0 {
+        return nil
+      }
+
+      let uncompressedChunk = mappedDisk.subdata(in: readFromByte ..< (readFromByte + numBytesToRead))
+
+      bytesRead += UInt64(uncompressedChunk.count)
+      digest.update(uncompressedChunk)
+
+      return uncompressedChunk
+    }
+
+    // Retrieve compressed data chunks, but normally no more than ``Self.layerLimitBytes`` bytes
+    while let compressedChunk = try compressingFilter.readData(ofLength: Self.bufferSizeBytes) {
+      compressedData.append(compressedChunk)
+    }
+
+    // Nothing was read this time from the disk,
+    // signal that to the consumer
+    if bytesRead == 0 {
+      return nil
+    }
+
+    return (compressedData, bytesRead, digest.finalize())
+  }
+}
@@ -1,10 +1,22 @@
 import Foundation

+// OCI manifest and OCI config media types
 let ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json"
 let ociConfigMediaType = "application/vnd.oci.image.config.v1+json"

-// Annotations
+// Layer media types
+let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
+let diskV1MediaType = "application/vnd.cirruslabs.tart.disk.v1"
+let diskV2MediaType = "application/vnd.cirruslabs.tart.disk.v2"
+let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
+
+// Manifest annotations
 let uncompressedDiskSizeAnnotation = "org.cirruslabs.tart.uncompressed-disk-size"
+let uploadTimeAnnotation = "org.cirruslabs.tart.upload-time"
+
+// Layer annotations
+let uncompressedSizeAnnotation = "org.cirruslabs.tart.uncompressed-size"
+let uncompressedContentDigestAnnotation = "org.cirruslabs.tart.uncompressed-content-digest"

 struct OCIManifest: Codable, Equatable {
  var schemaVersion: Int = 2
@@ -13,15 +25,21 @@ struct OCIManifest: Codable, Equatable {
  var layers: [OCIManifestLayer] = Array()
  var annotations: Dictionary<String, String>?

-  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil) {
+  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil, uploadDate: Date? = nil) {
    self.config = config
    self.layers = layers

+    var annotations: [String: String] = [:]
+
    if let uncompressedDiskSize = uncompressedDiskSize {
-      annotations = [
-        uncompressedDiskSizeAnnotation: String(uncompressedDiskSize)
-      ]
+      annotations[uncompressedDiskSizeAnnotation] = String(uncompressedDiskSize)
    }
+
+    if let uploadDate = uploadDate {
+      annotations[uploadTimeAnnotation] = uploadDate.toISO()
+    }
+
+    self.annotations = annotations
  }

  init(fromJSON: Data) throws {
@@ -64,6 +82,37 @@ struct OCIManifestLayer: Codable, Equatable {
  var mediaType: String
  var size: Int
  var digest: String
+  var annotations: Dictionary<String, String>?
+
+  init(mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil, uncompressedContentDigest: String? = nil) {
+    self.mediaType = mediaType
+    self.size = size
+    self.digest = digest
+
+    var annotations: [String: String] = [:]
+
+    if let uncompressedSize = uncompressedSize {
+      annotations[uncompressedSizeAnnotation] = String(uncompressedSize)
+    }
+
+    if let uncompressedContentDigest = uncompressedContentDigest {
+      annotations[uncompressedContentDigestAnnotation] = uncompressedContentDigest
+    }
+
+    self.annotations = annotations
+  }
+
+  func uncompressedSize() -> UInt64? {
+    guard let value = annotations?[uncompressedSizeAnnotation] else {
+      return nil
+    }
+
+    return UInt64(value)
+  }
+
+  func uncompressedContentDigest() -> String? {
+    annotations?[uncompressedContentDigestAnnotation]
+  }
 }

 struct Descriptor: Equatable {
@@ -45,7 +45,8 @@ struct TokenResponse: Decodable, Authentication {
  let defaultIssuedAt = Date()
  let defaultExpiresIn = 60

-  var token: String
+  var token: String?
+  var accessToken: String?
  var expiresIn: Int?
  var issuedAt: Date?

@@ -65,7 +66,13 @@ struct TokenResponse: Decodable, Authentication {
      return dateFormatter.date(from: dateString) ?? Date()
    }

-    return try decoder.decode(TokenResponse.self, from: fromData)
+    let response = try decoder.decode(TokenResponse.self, from: fromData)
+
+    guard response.token != nil || response.accessToken != nil else {
+      throw DecodingError.keyNotFound(CodingKeys.token, .init(codingPath: [], debugDescription: "Missing token or access_token. One must be present."))
+    }
+
+    return response
  }

  var tokenExpiresAt: Date {
@@ -83,7 +90,7 @@ struct TokenResponse: Decodable, Authentication {
  }

  func header() -> (String, String) {
-    ("Authorization", "Bearer \(token)")
+    return ("Authorization", "Bearer \(token ?? accessToken ?? "")")
  }

  func isValid() -> Bool {
@@ -92,11 +99,20 @@ struct TokenResponse: Decodable, Authentication {
 }

 class Registry {
-  let baseURL: URL
+  private let baseURL: URL
  let namespace: String
  let credentialsProviders: [CredentialsProvider]
+  let authenticationKeeper = AuthenticationKeeper()

-  var currentAuthToken: Authentication? = nil
+  var host: String? {
+    guard let host = baseURL.host else { return nil }
+
+    if let port = baseURL.port {
+      return "\(host):\(port)"
+    }
+
+    return host
+  }

  init(urlComponents: URLComponents,
       namespace: String,
@@ -225,7 +241,7 @@ class Registry {
    return digest
  }

-  public func pullBlob(_ digest: String, handler: (Data) throws -> Void) async throws {
+  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
    let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
    if response.statusCode != HTTPCode.Ok.rawValue {
      let body = try await channel.asData().asText()
@@ -236,7 +252,7 @@ class Registry {
    for try await part in channel {
      try Task.checkCancellation()

-      try handler(Data(part))
+      try await handler(part)
    }
  }

@@ -288,11 +304,6 @@ class Registry {
      request.httpBody = body
    }

-    // Invalidate token if it has expired
-    if currentAuthToken?.isValid() == false {
-      currentAuthToken = nil
-    }
-
    var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)

    if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
@@ -314,7 +325,7 @@ class Registry {

    if wwwAuthenticate.scheme.lowercased() == "basic" {
      if let (user, password) = try lookupCredentials() {
-        currentAuthToken = BasicAuthentication(user: user, password: password)
+        await authenticationKeeper.set(BasicAuthentication(user: user, password: password))
      }

      return
@@ -361,7 +372,7 @@ class Registry {
        + "while retrieving an authentication token", details: data.asText())
    }

-    currentAuthToken = try TokenResponse.parse(fromData: data)
+    await authenticationKeeper.set(try TokenResponse.parse(fromData: data))
  }

  private func lookupCredentials() throws -> (String, String)? {
@@ -382,11 +393,13 @@ class Registry {
  private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
    var request = request

-    if let token = currentAuthToken {
-      let (name, value) = token.header()
+    if let (name, value) = await authenticationKeeper.header() {
      request.addValue(value, forHTTPHeaderField: name)
    }

+    request.setValue("Tart/\(CI.version) (\(DeviceInfo.os); \(DeviceInfo.model))",
+                     forHTTPHeaderField: "User-Agent")
+
    return try await Fetcher.fetch(request, viaFile: viaFile)
  }
 }
@@ -8,6 +8,11 @@ class PIDLock {
  init(lockURL: URL) throws {
    url = lockURL
    fd = open(lockURL.path, O_RDWR)
+    if fd == -1 {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.PIDLockFailed("failed to open lock file \(url): \(details)")
+    }
  }

  deinit {
@@ -1,6 +1,12 @@
 import Virtualization

-struct Darwin: Platform {
+struct UnsupportedHostOSError: Error, CustomStringConvertible {
+  var description: String {
+    "error: host macOS version is outdated to run this virtual machine"
+  }
+}
+
+struct Darwin: PlatformSuspendable {
  var ecid: VZMacMachineIdentifier
  var hardwareModel: VZMacHardwareModel

@@ -50,11 +56,18 @@ struct Darwin: Platform {
    VZMacOSBootLoader()
  }

-  func platform(nvramURL: URL) -> VZPlatformConfiguration {
+  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
    let result = VZMacPlatformConfiguration()

    result.machineIdentifier = ecid
-    result.auxiliaryStorage = VZMacAuxiliaryStorage(contentsOf: nvramURL)
+    result.auxiliaryStorage = VZMacAuxiliaryStorage(url: nvramURL)
+
+    if !hardwareModel.isSupported {
+      // At the moment support of M1 chip is not yet dropped in any macOS version
+      // This mean that host software is not supporting this hardware model and should be updated
+      throw UnsupportedHostOSError()
+    }
+
    result.hardwareModel = hardwareModel

    return result
@@ -85,13 +98,35 @@ struct Darwin: Platform {
    return result
  }

-  func pointingDevices() -> [VZPointingDeviceConfiguration] {
-    if #available(macOS 13, *) {
-      // Trackpad is only supported starting with macOS Ventura
-      // macOS Monterey will continue using a USB device == .darwin 
-      return [VZMacTrackpadConfiguration(), VZUSBScreenCoordinatePointingDeviceConfiguration()]
+  func keyboards() -> [VZKeyboardConfiguration] {
+    if #available(macOS 14, *) {
+      // Mac keyboard is only supported by guests starting with macOS Ventura
+      return [VZMacKeyboardConfiguration(), VZUSBKeyboardConfiguration()]
    } else {
-      return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+      return [VZUSBKeyboardConfiguration()]
+    }
+  }
+
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration] {
+    if #available(macOS 14, *) {
+      return [VZMacKeyboardConfiguration()]
+    } else {
+      // fallback to the regular configuration
+      return keyboards()
+    }
+  }
+
+  func pointingDevices() -> [VZPointingDeviceConfiguration] {
+    // Trackpad is only supported by guests starting with macOS Ventura
+    [VZMacTrackpadConfiguration(), VZUSBScreenCoordinatePointingDeviceConfiguration()]
+  }
+
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration] {
+    if #available(macOS 14, *) {
+      return [VZMacTrackpadConfiguration()]
+    } else {
+      // fallback to the regular configuration
+      return pointingDevices()
    }
  }
 }
@@ -14,7 +14,7 @@ struct Linux: Platform {
    return result
  }

-  func platform(nvramURL: URL) -> VZPlatformConfiguration {
+  func platform(nvramURL: URL) throws -> VZPlatformConfiguration {
    VZGenericPlatformConfiguration()
  }

@@ -31,6 +31,10 @@ struct Linux: Platform {
    return result
  }

+  func keyboards() -> [VZKeyboardConfiguration] {
+    [VZUSBKeyboardConfiguration()]
+  }
+
  func pointingDevices() -> [VZPointingDeviceConfiguration] {
    [VZUSBScreenCoordinatePointingDeviceConfiguration()]
  }
@@ -3,7 +3,13 @@ import Virtualization
 protocol Platform: Codable {
  func os() -> OS
  func bootLoader(nvramURL: URL) throws -> VZBootLoader
-  func platform(nvramURL: URL) -> VZPlatformConfiguration
+  func platform(nvramURL: URL) throws -> VZPlatformConfiguration
  func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
+  func keyboards() -> [VZKeyboardConfiguration]
  func pointingDevices() -> [VZPointingDeviceConfiguration]
 }
+
+protocol PlatformSuspendable: Platform {
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration]
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration]
+}
@@ -16,6 +16,7 @@ struct Root: AsyncParsableCommand {
      Get.self,
      List.self,
      Login.self,
+      Logout.self,
      IP.self,
      Pull.self,
      Push.self,
@@ -57,6 +58,11 @@ struct Root: AsyncParsableCommand {
      }
    }

+    // Add commands that are only available on specific macOS versions
+    if #available(macOS 14, *) {
+      configuration.subcommands.append(Suspend.self)
+    }
+
    // Ensure the default SIGINT handled is disabled,
    // otherwise there's a race between two handlers
    signal(SIGINT, SIG_IGN);
@@ -76,10 +82,12 @@ struct Root: AsyncParsableCommand {
      var command = try parseAsRoot()

      // Run garbage-collection before each command (shouldn't take too long)
-      do {
-        try Config().gc()
-      } catch {
-        fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+      if type(of: command) != type(of: Pull()) && type(of: command) != type(of: Clone()){
+        do {
+          try Config().gc()
+        } catch {
+          fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+        }
      }

      if var asyncCommand = command as? AsyncParsableCommand {
@@ -94,7 +102,7 @@ struct Root: AsyncParsableCommand {

      // Handle a non-ArgumentParser's exception that requires a specific exit code to be set
      if let errorWithExitCode = error as? HasExitCode {
-        print(error)
+        fputs("\(error)\n", stderr)

        Foundation.exit(errorWithExitCode.exitCode)
      }
@@ -0,0 +1,44 @@
+import Foundation
+
+func createPTY() -> Int32 {
+  var tty_fd: Int32 = -1
+  var sfd: Int32 = -1
+  var termios_ = termios()
+  let tty_path = UnsafeMutablePointer<CChar>.allocate(capacity: 1024)
+
+  var res = openpty(&tty_fd, &sfd, tty_path, nil, nil);
+  if (res < 0) {
+    perror("openpty error")
+    return -1
+  }
+
+  // close slave file descriptor
+  close(sfd)
+
+  res = fcntl(tty_fd, F_GETFL)
+  if (res < 0) {
+    perror("fcntl F_GETFL error")
+    return res
+  }
+
+  // set serial nonblocking
+  res = fcntl(tty_fd, F_SETFL, res | O_NONBLOCK)
+  if (res < 0) {
+    perror("fcntl F_SETFL O_NONBLOCK error")
+    return res
+  }
+
+  // set baudrate to 115200
+  tcgetattr(tty_fd, &termios_)
+  cfsetispeed(&termios_, speed_t(B115200))
+  cfsetospeed(&termios_, speed_t(B115200))
+  if (tcsetattr(tty_fd, TCSANOW, &termios_) != 0) {
+    perror("tcsetattr error")
+    return -1
+  }
+
+  print("Successfully open pty \(String(cString: tty_path))")
+
+  tty_path.deallocate()
+  return tty_fd
+}
@@ -5,34 +5,26 @@ import Dynamic
 // Kudos to @saagarjha's VirtualApple for finding about _VZVirtualMachineStartOptions

 extension VZVirtualMachine {
-  @available(macOS 12, *)
+  @MainActor @available(macOS 12, *)
  func start(_ recovery: Bool) async throws {
    if !recovery {
      // just use the regular API
-      return try await withCheckedThrowingContinuation { continuation in
-        DispatchQueue.main.async {
-          self.start(completionHandler: { result in
-            continuation.resume(with: result)
-          })
-        }
-      }
+      return try await self.start()
    }

    // use some private stuff only for recovery
    return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
-      DispatchQueue.main.async {
-        let handler: @convention(block) (_ result: Any?) -> Void = { result in
-          if let error = result as? Error {
-            continuation.resume(throwing: error)
-          } else {
-            continuation.resume(returning: ())
-          }
+      let handler: @convention(block) (_ result: Any?) -> Void = { result in
+        if let error = result as? Error {
+          continuation.resume(throwing: error)
+        } else {
+          continuation.resume(returning: ())
        }
-        // dynamic magic
-        let options = Dynamic._VZVirtualMachineStartOptions()
-        options.bootMacOSRecovery = recovery
-        Dynamic(self)._start(withOptions: options, completionHandler: handler)
      }
+      // dynamic magic
+      let options = Dynamic._VZVirtualMachineStartOptions()
+      options.bootMacOSRecovery = recovery
+      Dynamic(self)._start(withOptions: options, completionHandler: handler)
    }
  }
 }
@@ -26,6 +26,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
  // Virtualization.Framework's virtual machine
  @Published var virtualMachine: VZVirtualMachine

+  // Virtualization.Framework's virtual machine configuration
+  var configuration: VZVirtualMachineConfiguration
+
  // Semaphore used to communicate with the VZVirtualMachineDelegate
  var sema = DispatchSemaphore(value: 0)

@@ -39,8 +42,10 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {

  init(vmDir: VMDirectory,
       network: Network = NetworkShared(),
-       additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
-       directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = []
+       additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
+       directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
+       serialPorts: [VZSerialPortConfiguration] = [],
+       suspendable: Bool = false
  ) throws {
    name = vmDir.name
    config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -51,10 +56,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {

    // Initialize the virtual machine and its configuration
    self.network = network
-    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
-                                                    nvramURL: vmDir.nvramURL, vmConfig: config,
-                                                    network: network, additionalDiskAttachments: additionalDiskAttachments,
-                                                    directorySharingDevices: directorySharingDevices
+    configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
+                                                nvramURL: vmDir.nvramURL, vmConfig: config,
+                                                network: network, additionalStorageDevices: additionalStorageDevices,
+                                                directorySharingDevices: directorySharingDevices,
+                                                serialPorts: serialPorts,
+                                                suspendable: suspendable
    )
    virtualMachine = VZVirtualMachine(configuration: configuration)

@@ -64,9 +71,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {

  static func retrieveIPSW(remoteURL: URL) async throws -> URL {
    // Check if we already have this IPSW in cache
-    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+    var headRequest = URLRequest(url: remoteURL)
+    headRequest.httpMethod = "HEAD"
+    let (_, headResponse) = try await Fetcher.fetch(headRequest, viaFile: false)

-    if let hash = response.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
+    if let hash = headResponse.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
      let ipswLocation = try IPSWCache().locationFor(fileName: "sha256:\(hash).ipsw")

      if FileManager.default.fileExists(atPath: ipswLocation.path) {
@@ -80,6 +89,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    // Download the IPSW
    defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")

+    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+
    let progress = Progress(totalUnitCount: response.expectedContentLength)
    ProgressObserver(progress).log(defaultLogger)

@@ -131,8 +142,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    ipswURL: URL,
    diskSizeGB: UInt16,
    network: Network = NetworkShared(),
-    additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
-    directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = []
+    additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
+    directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
+    serialPorts: [VZSerialPortConfiguration] = []
  ) async throws {
    var ipswURL = ipswURL

@@ -176,10 +188,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {

    // Initialize the virtual machine and its configuration
    self.network = network
-    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
-                                                    vmConfig: config, network: network,
-                                                    additionalDiskAttachments: additionalDiskAttachments,
-                                                    directorySharingDevices: directorySharingDevices
+    configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
+                                                vmConfig: config, network: network,
+                                                additionalStorageDevices: additionalStorageDevices,
+                                                directorySharingDevices: directorySharingDevices,
+                                                serialPorts: serialPorts
    )
    virtualMachine = VZVirtualMachine(configuration: configuration)

@@ -216,27 +229,18 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    return try VM(vmDir: vmDir)
  }

-  func run(_ recovery: Bool) async throws {
+  func start(recovery: Bool, resume shouldResume: Bool) async throws {
    try network.run(sema)

-    let startTask = DispatchQueue.main.sync {
-      Task {
-        if #available(macOS 13, *) {
-          // new API introduced in Ventura
-          let startOptions = VZMacOSVirtualMachineStartOptions()
-          startOptions.startUpFromMacOSRecovery = recovery
-          try await virtualMachine.start(options: startOptions)
-        } else {
-          // use method that also available on Monterey
-          try await virtualMachine.start(recovery)
-        }
-      }
+    if shouldResume {
+      try await resume()
+    } else {
+      try await start(recovery)
    }
+  }

-    try await withTaskCancellationHandler(operation: {
-      // Await on VZVirtualMachine.start() result
-      _ = try await startTask.value
-
+  func run() async throws {
+    await withTaskCancellationHandler(operation: {
      // Wait for the VM to finish running
      // or for the exit condition
      sema.wait()
@@ -245,23 +249,38 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    })

    if Task.isCancelled {
-      DispatchQueue.main.sync {
-        Task {
-          try await self.virtualMachine.stop()
-        }
-      }
+      try await stop()
    }

    try await network.stop()
  }

+  @MainActor
+  private func start(_ recovery: Bool) async throws {
+    let startOptions = VZMacOSVirtualMachineStartOptions()
+    startOptions.startUpFromMacOSRecovery = recovery
+    try await virtualMachine.start(options: startOptions)
+  }
+
+  @MainActor
+  private func resume() async throws {
+    try await virtualMachine.resume()
+  }
+
+  @MainActor
+  private func stop() async throws {
+    try await self.virtualMachine.stop()
+  }
+
  static func craftConfiguration(
    diskURL: URL,
    nvramURL: URL,
    vmConfig: VMConfig,
    network: Network = NetworkShared(),
-    additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment],
-    directorySharingDevices: [VZDirectorySharingDeviceConfiguration]
+    additionalStorageDevices: [VZStorageDeviceConfiguration],
+    directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
+    serialPorts: [VZSerialPortConfiguration],
+    suspendable: Bool = false
  ) throws -> VZVirtualMachineConfiguration {
    let configuration = VZVirtualMachineConfiguration()

@@ -273,41 +292,80 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
    configuration.memorySize = vmConfig.memorySize

    // Platform
-    configuration.platform = vmConfig.platform.platform(nvramURL: nvramURL)
+    configuration.platform = try vmConfig.platform.platform(nvramURL: nvramURL)

    // Display
    configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]

    // Audio
-    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
-    let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-    inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
-    let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
-    outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
-    soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
-    configuration.audioDevices = [soundDeviceConfiguration]
+    if !suspendable {
+      let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
+      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
+      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+      outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
+      configuration.audioDevices = [soundDeviceConfiguration]
+    }

    // Keyboard and mouse
-    configuration.keyboards = [VZUSBKeyboardConfiguration()]
-    configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
+      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
+      configuration.pointingDevices = platformSuspendable.pointingDevicesSuspendable()
+    } else {
+      configuration.keyboards = vmConfig.platform.keyboards()
+      configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    }

    // Networking
-    let vio = VZVirtioNetworkDeviceConfiguration()
-    vio.attachment = network.attachment()
-    vio.macAddress = vmConfig.macAddress
-    configuration.networkDevices = [vio]
+    configuration.networkDevices = network.attachments().map {
+      let vio = VZVirtioNetworkDeviceConfiguration()
+      vio.attachment = $0
+      vio.macAddress = vmConfig.macAddress
+      return vio
+    }

    // Storage
-    var attachments = [try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)]
-    attachments.append(contentsOf: additionalDiskAttachments)
-    configuration.storageDevices = attachments.map { VZVirtioBlockDeviceConfiguration(attachment: $0) }
+    let attachment: VZDiskImageStorageDeviceAttachment = vmConfig.os == .linux ?
+      // Use "cached" caching mode for virtio drive to prevent fs corruption on linux
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false, cachingMode: .cached, synchronizationMode: .full) :
+      try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)
+
+    var device: VZStorageDeviceConfiguration
+    if #available(macOS 14, *), vmConfig.os == .linux {
+      device = VZNVMExpressControllerDeviceConfiguration(attachment: attachment)
+    } else {
+      device = VZVirtioBlockDeviceConfiguration(attachment: attachment)
+    }
+    var devices: [VZStorageDeviceConfiguration] = [device]
+    devices.append(contentsOf: additionalStorageDevices)
+    configuration.storageDevices = devices

    // Entropy
-    configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()]
+    if !suspendable {
+      configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()]
+    }

    // Directory sharing devices
    configuration.directorySharingDevices = directorySharingDevices

+    // Serial Port
+    configuration.serialPorts = serialPorts
+
+    // Version console device
+    //
+    // A dummy console device useful for implementing
+    // host feature checks in the guest agent software.
+    if !suspendable {
+      let consolePort = VZVirtioConsolePortConfiguration()
+      consolePort.name = "tart-version-\(CI.version)"
+
+      let consoleDevice = VZVirtioConsoleDeviceConfiguration()
+      consoleDevice.ports[0] = consolePort
+
+      configuration.consoleDevices.append(consoleDevice)
+    }
+
    try configuration.validate()

    return configuration
@@ -97,11 +97,7 @@ struct VMConfig: Codable {
    case .darwin:
      platform = try Darwin(from: decoder)
    case .linux:
-      if #available(macOS 13, *) {
-        platform = try Linux(from: decoder)
-      } else {
-        throw UnsupportedOSError("Linux VMs", "are")
-      }
+      platform = try Linux(from: decoder)
    }
    cpuCountMin = try container.decode(Int.self, forKey: .cpuCountMin)
    cpuCount = try container.decode(Int.self, forKey: .cpuCount)
@@ -136,20 +132,30 @@ struct VMConfig: Codable {
  }

  mutating func setCPU(cpuCount: Int) throws {
-    if cpuCount < cpuCountMin {
+    if os == .darwin && cpuCount < cpuCountMin {
      throw LessThanMinimalResourcesError("VM should have \(cpuCountMin) CPU cores"
        + " at minimum (requested \(cpuCount))")
    }

+    if cpuCount < VZVirtualMachineConfiguration.minimumAllowedCPUCount {
+      throw LessThanMinimalResourcesError("VM should have \(VZVirtualMachineConfiguration.minimumAllowedCPUCount) CPU cores"
+        + " at minimum (requested \(cpuCount))")
+    }
+
    self.cpuCount = cpuCount
  }

  mutating func setMemory(memorySize: UInt64) throws {
-    if memorySize < memorySizeMin {
+    if os == .darwin && memorySize < memorySizeMin {
      throw LessThanMinimalResourcesError("VM should have \(memorySizeMin) bytes"
        + " of memory at minimum (requested \(memorySize))")
    }

+    if memorySize < VZVirtualMachineConfiguration.minimumAllowedMemorySize {
+      throw LessThanMinimalResourcesError("VM should have \(VZVirtualMachineConfiguration.minimumAllowedMemorySize) bytes"
+        + " of memory at minimum (requested \(memorySize))")
+    }
+
    self.memorySize = memorySize
  }
 }
@@ -81,7 +81,8 @@ extension VMDirectory {
      try? decodeStream.close()
    }

-    guard let extractStream = ArchiveStream.extractStream(extractingTo: FilePath(baseURL.path)) else {
+    guard let extractStream = ArchiveStream.extractStream(extractingTo: FilePath(baseURL.path),
+                                                          flags: [.ignoreOperationNotPermitted]) else {
      let details = Errno(rawValue: CInt(errno))

      throw RuntimeError.ImportFailed("ArchiveStream.extractStream() failed: \(details)")
@@ -1,33 +1,30 @@
 import Foundation
-import Compression
 import Sentry

 enum OCIError: Error {
  case ShouldBeExactlyOneLayer
  case ShouldBeAtLeastOneLayer
  case FailedToCreateVmFile
+  case LayerIsMissingUncompressedSizeAnnotation
+  case LayerIsMissingUncompressedDigestAnnotation
 }

 extension VMDirectory {
  private static let bufferSizeBytes = 64 * 1024 * 1024
  private static let layerLimitBytes = 500 * 1000 * 1000

-  private static let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
-  private static let diskMediaType = "application/vnd.cirruslabs.tart.disk.v1"
-  private static let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
-
-  func pullFromRegistry(registry: Registry, reference: String) async throws {
+  func pullFromRegistry(registry: Registry, reference: String, concurrency: UInt) async throws {
    defaultLogger.appendNewLine("pulling manifest...")

    let (manifest, _) = try await registry.pullManifest(reference: reference)

-    return try await pullFromRegistry(registry: registry, manifest: manifest)
+    return try await pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
  }

-  func pullFromRegistry(registry: Registry, manifest: OCIManifest) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt) async throws {
    // Pull VM's config file layer and re-serialize it into a config file
    let configLayers = manifest.layers.filter {
-      $0.mediaType == Self.configMediaType
+      $0.mediaType == configMediaType
    }
    if configLayers.count != 1 {
      throw OCIError.ShouldBeExactlyOneLayer
@@ -41,50 +38,36 @@ extension VMDirectory {
    }
    try configFile.close()

-    // Pull VM's disk layers and decompress them sequentially into a disk file
-    let diskLayers = manifest.layers.filter {
-      $0.mediaType == Self.diskMediaType
-    }
-    if diskLayers.isEmpty {
+    // Pull VM's disk layers and decompress them into a disk file
+    let diskImplType: Disk.Type
+    let layers: [OCIManifestLayer]
+
+    if manifest.layers.contains(where: { $0.mediaType == diskV1MediaType }) {
+      diskImplType = DiskV1.self
+      layers = manifest.layers.filter { $0.mediaType == diskV1MediaType }
+    } else if manifest.layers.contains(where: { $0.mediaType == diskV2MediaType }) {
+      diskImplType = DiskV2.self
+      layers = manifest.layers.filter { $0.mediaType == diskV2MediaType }
+    } else {
      throw OCIError.ShouldBeAtLeastOneLayer
    }
-    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
-      throw OCIError.FailedToCreateVmFile
-    }
-    let disk = try FileHandle(forWritingTo: diskURL)
-    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
-      if let data = data {
-        disk.write(data)
-      }
-    }

-    // Progress
-    let diskCompressedSize: Int64 = Int64(diskLayers.map {
-      $0.size
-    }
-    .reduce(0) {
-      $0 + $1
-    })
+    let diskCompressedSize = layers.map { Int64($0.size) }.reduce(0, +)
+    SentrySDK.span?.setMeasurement(name: "compressed_disk_size", value: diskCompressedSize as NSNumber, unit: MeasurementUnitInformation.byte)
+
    let prettyDiskSize = String(format: "%.1f", Double(diskCompressedSize) / 1_000_000_000.0)
    defaultLogger.appendNewLine("pulling disk (\(prettyDiskSize) GB compressed)...")
+
    let progress = Progress(totalUnitCount: diskCompressedSize)
    ProgressObserver(progress).log(defaultLogger)

-    for diskLayer in diskLayers {
-      try await registry.pullBlob(diskLayer.digest) { data in
-        try filter.write(data)
-        progress.completedUnitCount += Int64(data.count)
-      }
-    }
-    try filter.finalize()
-    try disk.close()
-    SentrySDK.span?.setMeasurement(name: "compressed_disk_size", value: diskCompressedSize as NSNumber, unit: MeasurementUnitInformation.byte);
+    try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL, concurrency: concurrency, progress: progress)

    // Pull VM's NVRAM file layer and store it in an NVRAM file
    defaultLogger.appendNewLine("pulling NVRAM...")

    let nvramLayers = manifest.layers.filter {
-      $0.mediaType == Self.nvramMediaType
+      $0.mediaType == nvramMediaType
    }
    if nvramLayers.count != 1 {
      throw OCIError.ShouldBeExactlyOneLayer
@@ -99,7 +82,7 @@ extension VMDirectory {
    try nvram.close()
  }

-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String) async throws -> RemoteName {
    var layers = Array<OCIManifestLayer>()

    // Read VM's config and push it as blob
@@ -107,32 +90,22 @@ extension VMDirectory {
    let configJSON = try JSONEncoder().encode(config)
    defaultLogger.appendNewLine("pushing config...")
    let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb)
-    layers.append(OCIManifestLayer(mediaType: Self.configMediaType, size: configJSON.count, digest: configDigest))
+    layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest))

-    // Progress
+    // Compress the disk file as multiple chunks and push them as disk layers
    let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64

    defaultLogger.appendNewLine("pushing disk... this will take a while...")
    let progress = Progress(totalUnitCount: diskSize)
    ProgressObserver(progress).log(defaultLogger)

-    // Read VM's compressed disk as chunks
-    // and sequentially upload them as blobs
-    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
-    let mappedDiskSize = mappedDisk.count
-    var mappedDiskReadOffset = 0
-    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
-      let bytesRead = min(length, mappedDiskSize - mappedDiskReadOffset)
-      let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
-      mappedDiskReadOffset += bytesRead
-
-      progress.completedUnitCount = Int64(mappedDiskReadOffset)
-
-      return data
-    }
-    while let compressedLayerData = try compressingFilter.readData(ofLength: Self.layerLimitBytes) {
-      let layerDigest = try await registry.pushBlob(fromData: compressedLayerData, chunkSizeMb: chunkSizeMb)
-      layers.append(OCIManifestLayer(mediaType: Self.diskMediaType, size: compressedLayerData.count, digest: layerDigest))
+    switch diskFormat {
+    case "v1":
+      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+    case "v2":
+      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+    default:
+      throw RuntimeError.OCIUnsupportedDiskFormat(diskFormat)
    }

    // Read VM's NVRAM and push it as blob
@@ -140,7 +113,7 @@ extension VMDirectory {

    let nvram = try FileHandle(forReadingFrom: nvramURL).readToEnd()!
    let nvramDigest = try await registry.pushBlob(fromData: nvram, chunkSizeMb: chunkSizeMb)
-    layers.append(OCIManifestLayer(mediaType: Self.nvramMediaType, size: nvram.count, digest: nvramDigest))
+    layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest))

    // Craft a stub OCI config for Docker Hub compatibility
    let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os).toJSON()
@@ -148,7 +121,8 @@ extension VMDirectory {
    let manifest = OCIManifest(
      config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
      layers: layers,
-      uncompressedDiskSize: UInt64(mappedDiskReadOffset)
+      uncompressedDiskSize: UInt64(diskSize),
+      uploadDate: Date()
    )

    // Manifest
@@ -159,7 +133,7 @@ extension VMDirectory {
    }

    let pushedReference = Reference(digest: try manifest.digest())
-    return RemoteName(host: registry.baseURL.host!, namespace: registry.namespace, reference: pushedReference)
+    return RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference)
  }
 }

@@ -1,5 +1,6 @@
 import Foundation
 import Virtualization
+import CryptoKit

 struct VMDirectory: Prunable {
  var baseURL: URL
@@ -13,6 +14,9 @@ struct VMDirectory: Prunable {
  var nvramURL: URL {
    baseURL.appendingPathComponent("nvram.bin")
  }
+  var stateURL: URL {
+    baseURL.appendingPathComponent("state.vzvmsave")
+  }

  var explicitlyPulledMark: URL {
    baseURL.appendingPathComponent(".explicitly-pulled")
@@ -26,6 +30,33 @@ struct VMDirectory: Prunable {
    baseURL
  }

+  func lock() throws -> PIDLock {
+    try PIDLock(lockURL: configURL)
+  }
+
+  func running() throws -> Bool {
+    // The most common reason why PIDLock() instantiation fails is a race with "tart delete" (ENOENT),
+    // which is fine to report as "not running".
+    //
+    // The other reasons are unlikely and the cost of getting a false positive is way less than
+    // the cost of crashing with an exception when calling "tart list" on a busy machine, for example.
+    guard let lock = try? lock() else {
+      return false
+    }
+
+    return try lock.pid() != 0
+  }
+
+  func state() throws -> String {
+    if try running() {
+      return "running"
+    } else if FileManager.default.fileExists(atPath: stateURL.path) {
+      return "suspended"
+    } else {
+      return "stopped"
+    }
+  }
+
  static func temporary() throws -> VMDirectory {
    let tmpDir = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString)
    try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: false)
@@ -33,6 +64,17 @@ struct VMDirectory: Prunable {
    return VMDirectory(baseURL: tmpDir)
  }

+  //Create tmp directory with hashing
+  static func temporaryDeterministic(key: String) throws -> VMDirectory {
+    let keyData = Data(key.utf8)
+    let hash = Insecure.MD5.hash(data: keyData)
+    // Convert hash to string
+    let hashString = hash.compactMap { String(format: "%02x", $0) }.joined()
+    let tmpDir = try Config().tartTmpDir.appendingPathComponent(hashString)
+    try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true)
+    return VMDirectory(baseURL: tmpDir)
+  }
+
  var initialized: Bool {
    FileManager.default.fileExists(atPath: configURL.path) &&
      FileManager.default.fileExists(atPath: diskURL.path) &&
@@ -51,9 +93,9 @@ struct VMDirectory: Prunable {
    try? FileManager.default.removeItem(at: nvramURL)
  }

-  func validate() throws {
+  func validate(userFriendlyName: String) throws {
    if !FileManager.default.fileExists(atPath: baseURL.path) {
-      throw RuntimeError.VMDoesNotExist(name: baseURL.lastPathComponent)
+      throw RuntimeError.VMDoesNotExist(name: userFriendlyName)
    }

    if !initialized {
@@ -66,6 +108,7 @@ struct VMDirectory: Prunable {
    try FileManager.default.copyItem(at: configURL, to: to.configURL)
    try FileManager.default.copyItem(at: nvramURL, to: to.nvramURL)
    try FileManager.default.copyItem(at: diskURL, to: to.diskURL)
+    try? FileManager.default.copyItem(at: stateURL, to: to.stateURL)

    // Re-generate MAC address
    if generateMAC {
@@ -81,6 +124,8 @@ struct VMDirectory: Prunable {
    var vmConfig = try VMConfig(fromURL: configURL)

    vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
+    // cleanup state if any
+    try? FileManager.default.removeItem(at: stateURL)

    try vmConfig.save(toURL: configURL)
  }
@@ -89,14 +134,31 @@ struct VMDirectory: Prunable {
    if !FileManager.default.fileExists(atPath: diskURL.path) {
      FileManager.default.createFile(atPath: diskURL.path, contents: nil, attributes: nil)
    }
+
    let diskFileHandle = try FileHandle.init(forWritingTo: diskURL)
-    // macOS considers kilo being 1000 and not 1024
-    try diskFileHandle.truncate(atOffset: UInt64(sizeGB) * 1000 * 1000 * 1000)
+    let currentDiskFileLength = try diskFileHandle.seekToEnd()
+    let desiredDiskFileLength = UInt64(sizeGB) * 1000 * 1000 * 1000
+    if desiredDiskFileLength < currentDiskFileLength {
+      let currentLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(currentDiskFileLength))
+      let desiredLengthHuman = ByteCountFormatter().string(fromByteCount: Int64(desiredDiskFileLength))
+      throw RuntimeError.InvalidDiskSize("new disk size of \(desiredLengthHuman) should be larger " +
+        "than the current disk size of \(currentLengthHuman)")
+    } else if desiredDiskFileLength > currentDiskFileLength {
+      try diskFileHandle.truncate(atOffset: desiredDiskFileLength)
+    }
    try diskFileHandle.close()
  }

  func delete() throws {
+    let lock = try lock()
+
+    if try !lock.trylock() {
+      throw RuntimeError.VMIsRunning(name)
+    }
+
    try FileManager.default.removeItem(at: baseURL)
+
+    try lock.unlock()
  }

  func accessDate() throws -> Date {
@@ -34,19 +34,28 @@ class VMStorageHelper {
  }
 }

+extension NSError {
+  func isFileNotFound() -> Bool {
+    return self.code == NSFileNoSuchFileError || self.code == NSFileReadNoSuchFileError
+  }
+}
+
 extension Error {
  func isFileNotFound() -> Bool {
-    (self as NSError).code == NSFileReadNoSuchFileError
+    (self as NSError).isFileNotFound() || (self as NSError).underlyingErrors.contains(where: { $0.isFileNotFound() })
  }
 }

 enum RuntimeError : Error {
+  case VMConfigurationError(_ message: String)
  case VMDoesNotExist(name: String)
  case VMMissingFiles(_ message: String)
-  case VMNotRunning(_ message: String)
+  case VMIsRunning(_ name: String)
+  case VMNotRunning(_ name: String)
  case VMAlreadyRunning(_ message: String)
  case NoIPAddressFound(_ message: String)
  case DiskAlreadyInUse(_ message: String)
+  case InvalidDiskSize(_ message: String)
  case FailedToUpdateAccessDate(_ message: String)
  case PIDLockFailed(_ message: String)
  case FailedToParseRemoteName(_ message: String)
@@ -56,6 +65,9 @@ enum RuntimeError : Error {
  case ExportFailed(_ message: String)
  case ImportFailed(_ message: String)
  case SoftnetFailed(_ message: String)
+  case OCIStorageError(_ message: String)
+  case OCIUnsupportedDiskFormat(_ format: String)
+  case SuspendFailed(_ message: String)
 }

 protocol HasExitCode {
@@ -65,18 +77,24 @@ protocol HasExitCode {
 extension RuntimeError : CustomStringConvertible {
  public var description: String {
    switch self {
+    case .VMConfigurationError(let message):
+      return message
    case .VMDoesNotExist(let name):
      return "the specified VM \"\(name)\" does not exist"
    case .VMMissingFiles(let message):
      return message
-    case .VMNotRunning(let message):
-      return message
+    case .VMIsRunning(let name):
+      return "VM \"\(name)\" is running"
+    case .VMNotRunning(let name):
+      return "VM \"\(name)\" is not running"
    case .VMAlreadyRunning(let message):
      return message
    case .NoIPAddressFound(let message):
      return message
    case .DiskAlreadyInUse(let message):
      return message
+    case .InvalidDiskSize(let message):
+      return message
    case .FailedToUpdateAccessDate(let message):
      return message
    case .PIDLockFailed(let message):
@@ -95,6 +113,12 @@ extension RuntimeError : CustomStringConvertible {
      return "VM import failed: \(message)"
    case .SoftnetFailed(let message):
      return "Softnet failed: \(message)"
+    case .OCIStorageError(let message):
+      return "OCI storage error: \(message)"
+    case .OCIUnsupportedDiskFormat(let format):
+      return "OCI disk format \(format) is not supported by this version of Tart"
+    case .SuspendFailed(let message):
+      return "Failed to suspend the VM: \(message)"
    }
  }
 }
@@ -102,6 +126,8 @@ extension RuntimeError : CustomStringConvertible {
 extension RuntimeError : HasExitCode {
  var exitCode: Int32 {
    switch self {
+    case .VMDoesNotExist:
+      return 2
    case .VMNotRunning:
      return 2
    case .VMAlreadyRunning:
@@ -1,6 +1,6 @@
 import Foundation

-class VMStorageLocal {
+class VMStorageLocal: PrunableStorage {
  let baseURL: URL = try! Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)

  private func vmURL(_ name: String) -> URL {
@@ -14,7 +14,9 @@ class VMStorageLocal {
  func open(_ name: String) throws -> VMDirectory {
    let vmDir = VMDirectory(baseURL: vmURL(name))

-    try vmDir.validate()
+    try vmDir.validate(userFriendlyName: name)
+
+    try vmDir.baseURL.updateAccessDate()

    return vmDir
  }
@@ -37,7 +39,7 @@ class VMStorageLocal {
  }

  func delete(_ name: String) throws {
-    try FileManager.default.removeItem(at: vmURL(name))
+    try VMDirectory(baseURL: vmURL(name)).delete()
  }

  func list() throws -> [(String, VMDirectory)] {
@@ -63,6 +65,10 @@ class VMStorageLocal {
    }
  }

+  func prunables() throws -> [Prunable] {
+    try list().map { (_, vmDir) in vmDir }
+  }
+
  func hasVMsWithMACAddress(macAddress: String) throws -> Bool {
    try list().contains { try $1.macAddress() == macAddress }
  }
@@ -16,10 +16,20 @@ class VMStorageOCI: PrunableStorage {
    VMDirectory(baseURL: vmURL(name)).initialized
  }

+  func digest(_ name: RemoteName) throws -> String {
+    let digest = vmURL(name).resolvingSymlinksInPath().lastPathComponent
+
+    if !digest.starts(with: "sha256:") {
+      throw RuntimeError.OCIStorageError("\(name) is not a digest and doesn't point to a digest")
+    }
+
+    return digest
+  }
+
  func open(_ name: RemoteName) throws -> VMDirectory {
    let vmDir = VMDirectory(baseURL: vmURL(name))

-    try vmDir.validate()
+    try vmDir.validate(userFriendlyName: name.description)

    try vmDir.baseURL.updateAccessDate()

@@ -122,7 +132,11 @@ class VMStorageOCI: PrunableStorage {
    try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
  }

-  func pull(_ name: RemoteName, registry: Registry) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
+    SentrySDK.configureScope { scope in
+      scope.setContext(value: ["imageName": name], key: "OCI")
+    }
+
    defaultLogger.appendNewLine("pulling manifest...")

    let (manifest, manifestData) = try await registry.pullManifest(reference: name.reference.value)
@@ -130,6 +144,12 @@ class VMStorageOCI: PrunableStorage {
    let digestName = RemoteName(host: name.host, namespace: name.namespace,
                                reference: Reference(digest: Digest.hash(manifestData)))

+    if exists(name) && exists(digestName) && linked(from: name, to: digestName) {
+      // optimistically check if we need to do anything at all before locking
+      defaultLogger.appendNewLine("\(digestName) image is already cached and linked!")
+      return
+    }
+
    // Ensure that host directory for given RemoteName exists in OCI storage
    let hostDirectoryURL = hostDirectoryURL(digestName)
    try FileManager.default.createDirectory(at: hostDirectoryURL, withIntermediateDirectories: true)
@@ -150,7 +170,7 @@ class VMStorageOCI: PrunableStorage {

    if !exists(digestName) {
      let transaction = SentrySDK.startTransaction(name: name.description, operation: "pull", bindToScope: true)
-      let tmpVMDir = try VMDirectory.temporary()
+      let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description)

      // Lock the temporary VM directory to prevent it's garbage collection
      let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
@@ -158,40 +178,17 @@ class VMStorageOCI: PrunableStorage {

      // Try to reclaim some cache space if we know the VM size in advance
      if let uncompressedDiskSize = manifest.uncompressedDiskSize() {
-        let requiredCapacityBytes = UInt64(uncompressedDiskSize + 128 * 1024 * 1024)
-
-        let attrs = try Config().tartCacheDir.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey, .volumeAvailableCapacityKey])
-        let capacityImportant = attrs.volumeAvailableCapacityForImportantUsage!
-        let capacityAvailable = attrs.volumeAvailableCapacity!
-        let availableCapacityBytes = max(UInt64(capacityImportant), UInt64(capacityAvailable))
-
-        if capacityImportant == 0 || capacityAvailable == 0 {
-          SentrySDK.capture(message: "Zero capacity") { scope in
-            scope.setLevel(.warning)
-
-            scope.setContext(value: [
-              "volumeAvailableCapacityForImportantUsageKey": capacityImportant,
-              "volumeAvailableCapacityKey": capacityAvailable,
-            ], key: "Attributes")
-          }
+        SentrySDK.configureScope { scope in
+          scope.setContext(value: ["imageUncompressedDiskSize": uncompressedDiskSize], key: "OCI")
        }

-        // There is a suspicious that occasionally capacity is returned as zero which can't be true.
-        // Let's validate to avoid unnecessary pruning.
-        if 0 < availableCapacityBytes && availableCapacityBytes < requiredCapacityBytes {
-          let transaction = SentrySDK.startTransaction(name: "Automatically Pruning Cache", operation: "prune", bindToScope: true)
-          transaction.setData(value: name, key: "name")
-          transaction.setData(value: uncompressedDiskSize, key: "uncompressedDiskSize")
-          transaction.setData(value: availableCapacityBytes, key: "availableCapacity")
-          transaction.setData(value: requiredCapacityBytes, key: "requiredCapacity")
-          defer { transaction.finish() }
+        let otherVMFilesSize: UInt64 = 128 * 1024 * 1024

-          try Prune.pruneReclaim(reclaimBytes: requiredCapacityBytes - availableCapacityBytes)
-        }
+        try Prune.reclaimIfNeeded(uncompressedDiskSize + otherVMFilesSize)
      }

      try await withTaskCancellationHandler(operation: {
-        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest)
+        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
        try move(digestName, from: tmpVMDir)
        transaction.finish()
      }, onCancel: {
@@ -212,10 +209,17 @@ class VMStorageOCI: PrunableStorage {
    }
  }

-  func link(from: RemoteName, to: RemoteName) throws {
-    if FileManager.default.fileExists(atPath: vmURL(from).path) {
-      try FileManager.default.removeItem(at: vmURL(from))
+  func linked(from: RemoteName, to: RemoteName) -> Bool {
+    do {
+      let resolvedFrom = try FileManager.default.destinationOfSymbolicLink(atPath: vmURL(from).path)
+      return resolvedFrom == vmURL(to).path
+    } catch {
+      return false
    }
+  }
+
+  func link(from: RemoteName, to: RemoteName) throws {
+    try? FileManager.default.removeItem(at: vmURL(from))

    try FileManager.default.createSymbolicLink(at: vmURL(from), withDestinationURL: vmURL(to))

@@ -0,0 +1,32 @@
+import XCTest
+@testable import tart
+
+final class DirectoryShareTests: XCTestCase {
+  func testNamedParsing() throws {
+    let share = try DirectoryShare(parseFrom: "build:/Users/admin/build")
+    XCTAssertEqual(share.name, "build")
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+  }
+  
+  func testNamedReadOnlyParsing() throws {
+    let share = try DirectoryShare(parseFrom: "build:/Users/admin/build:ro")
+    XCTAssertEqual(share.name, "build")
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(share.readOnly)
+  }
+  
+  func testOptionalNameParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+  }
+  
+  func testOptionalNameReadOnlyParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build:ro")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(share.readOnly)
+  }
+}
@@ -0,0 +1,16 @@
+import XCTest
+@testable import tart
+
+final class DockerConfigTests: XCTestCase {
+  func testHelpers() throws {
+    let config = DockerConfig(credHelpers: [
+      "(.*).dkr.ecr.(.*).amazonaws.com": "ecr-login",
+      "gcr.io": "gcloud"
+    ])
+
+    XCTAssertEqual(try config.findCredHelper(host: "gcr.io"), "gcloud")
+    XCTAssertEqual(try config.findCredHelper(host: "123.dkr.ecr.eu-west-1.amazonaws.com"), "ecr-login")
+    XCTAssertEqual(try config.findCredHelper(host: "456.dkr.ecr.us-east-1.amazonaws.com"), "ecr-login")
+    XCTAssertNil(try config.findCredHelper(host: "ghcr.io"))
+  }
+}
@@ -0,0 +1,90 @@
+import XCTest
+@testable import tart
+
+final class LayerizerTests: XCTestCase {
+  var registryRunner: RegistryRunner?
+
+  var registry: Registry {
+    registryRunner!.registry
+  }
+
+  override func setUp() async throws {
+    try await super.setUp()
+
+    do {
+      registryRunner = try await RegistryRunner()
+    } catch {
+      try XCTSkipIf(ProcessInfo.processInfo.environment["CI"] == nil)
+    }
+  }
+
+  override func tearDown() async throws {
+    try await super.tearDown()
+
+    registryRunner = nil
+  }
+
+  func testDiskV1() async throws {
+    // Original disk file to be pushed to the registry
+    let originalDiskFileURL = try fileWithRandomData(sizeBytes: 5 * 1024 * 1024 * 1024)
+    addTeardownBlock {
+      try FileManager.default.removeItem(at: originalDiskFileURL)
+    }
+
+    // Disk file to be pulled from the registry
+    // and compared against the original disk file
+    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+
+    print("pushing disk...")
+    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+
+    print("pulling disk...")
+    try await DiskV1.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
+
+    print("comparing disks...")
+    try XCTAssertEqual(Digest.hash(originalDiskFileURL), Digest.hash(pulledDiskFileURL))
+  }
+
+  func testDiskV2() async throws {
+    // Original disk file to be pushed to the registry
+    let originalDiskFileURL = try fileWithRandomData(sizeBytes: 5 * 1024 * 1024 * 1024)
+    addTeardownBlock {
+      try FileManager.default.removeItem(at: originalDiskFileURL)
+    }
+
+    // Disk file to be pulled from the registry
+    // and compared against the original disk file
+    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+
+    print("pushing disk...")
+    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+
+    print("pulling disk...")
+    try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
+
+    print("comparing disks...")
+    try XCTAssertEqual(Digest.hash(originalDiskFileURL), Digest.hash(pulledDiskFileURL))
+  }
+
+  private func fileWithRandomData(sizeBytes: Int) throws -> URL {
+    let devUrandom = try FileHandle(forReadingFrom: URL(filePath: "/dev/urandom"))
+
+    let temporaryFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+    FileManager.default.createFile(atPath: temporaryFileURL.path, contents: nil)
+    let temporaryFile = try FileHandle(forWritingTo: temporaryFileURL)
+
+    var remainingBytes = sizeBytes
+
+    while remainingBytes > 0 {
+      let randomData = try devUrandom.read(upToCount: min(64 * 1024 * 1024, remainingBytes))!
+      remainingBytes -= randomData.count
+      try temporaryFile.write(contentsOf: randomData)
+    }
+
+    try devUrandom.close()
+
+    try temporaryFile.close()
+
+    return temporaryFileURL
+  }
+}
@@ -0,0 +1,18 @@
+import XCTest
+@testable import tart
+
+import Network
+import SwiftRadix
+
+final class LeaseTests: XCTestCase {
+  func testCorrectTimezone() throws {
+    let lease = Lease(fromRawLease: [
+      "hw_address": "1,11:22:33:44:55:66",
+      "ip_address": "1.2.3.4",
+      "lease": "0x6565da9e",
+    ])
+
+    XCTAssertNotNil(lease)
+    XCTAssertEqual(lease!.expiresAt.toISO(), "2023-11-28T12:18:38Z")
+  }
+}
@@ -0,0 +1,38 @@
+import XCTest
+@testable import tart
+
+import Network
+import SwiftDate
+
+final class LeasesTests: XCTestCase {
+  func testNoExpired() throws {
+    let macAddress = MACAddress(fromString: "11:22:33:44:55:66")!
+
+    let leases = try Leases("""
+    {
+      name=whatever
+      ip_address=66.66.66.66
+      hw_address=1,\(macAddress)
+      identifier=1,\(macAddress)
+      lease=\(Int((Date() - 1.seconds).timeIntervalSince1970).hex)
+
+    }
+    {
+      name=whatever
+      ip_address=1.2.3.4
+      hw_address=1,\(macAddress)
+      identifier=1,\(macAddress)
+      lease=\(Int((Date() + 10.minutes).timeIntervalSince1970).hex)
+    }
+    {
+      name=whatever
+      ip_address=66.66.66.66
+      hw_address=1,\(macAddress)
+      identifier=1,\(macAddress)
+      lease=\(Int((Date() - 1.seconds).timeIntervalSince1970).hex)
+    }
+    """)
+
+    XCTAssertEqual(IPv4Address("1.2.3.4"), leases.ResolveMACAddress(macAddress: macAddress))
+  }
+}
@@ -8,11 +8,12 @@ final class MACAddressResolverTests: XCTestCase {
    {
      ip_address=1.2.3.4
      hw_address=1,00:11:22:33:44:55
+      lease=0x7fffffff
    }
    """)

    XCTAssertEqual(IPv4Address("1.2.3.4"),
-                   try leases.resolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
+                   leases.ResolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
  }

  func testMultipleEntries() throws {
@@ -20,16 +21,18 @@ final class MACAddressResolverTests: XCTestCase {
    {
      ip_address=1.2.3.4
      hw_address=1,00:11:22:33:44:55
+      lease=0x7fffffff
    }
    {
      ip_address=5.6.7.8
      hw_address=1,AA:BB:CC:DD:EE:FF
+      lease=0x7fffffff
    }
    """)

    XCTAssertEqual(IPv4Address("1.2.3.4"),
-                   try leases.resolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
+                   leases.ResolveMACAddress(macAddress: MACAddress(fromString: "00:11:22:33:44:55")!))
    XCTAssertEqual(IPv4Address("5.6.7.8"),
-                   try leases.resolveMACAddress(macAddress: MACAddress(fromString: "AA:BB:CC:DD:EE:FF")!))
+                   leases.ResolveMACAddress(macAddress: MACAddress(fromString: "AA:BB:CC:DD:EE:FF")!))
  }
 }
@@ -0,0 +1,12 @@
+"default": true
+"MD002": false # First heading should be a top level heading
+"MD007": # Unordered list indentation
+  indent: 4
+"MD009": false # Trailing spaces
+"MD013": false # Line length
+"MD025": false # Multiple top level headings in the same document
+"MD026": false # Trailing punctuation in heading
+"MD033": false # Inline HTML
+"MD041": false # First line in file should be a top level heading
+"MD045": false # OK not to have a description for an image
+"MD046": false #  Code block style [Expected: fenced; Actual: indented]
@@ -1,8 +1,9 @@
-edigaryev:
-  name: Nikolay Edigaryev
-  description: Creator
-  avatar: https://github.com/edigaryev.png
-fkorotkov:
-  name: Fedor Korotkov
-  description: Creator
-  avatar: https://github.com/fkorotkov.png
+authors:
+  edigaryev:
+    name: Nikolay Edigaryev
+    description: Creator
+    avatar: https://github.com/edigaryev.png
+  fkorotkov:
+    name: Fedor Korotkov
+    description: Creator
+    avatar: https://github.com/fkorotkov.png
@@ -13,7 +13,7 @@ categories:

 **TLDR:** We are transitioning Tart's licensing from AGPL-3.0 to [Fair Source 100](https://fair.io/). This change will
 permit unlimited installations on personal computers, but organizations that exceed a certain number of server
-installations utilizing 100 CPU cores will be required to obtain a paid sponsorship.
+installations utilizing 100 CPU cores will be required to obtain a paid license.

 ## Background

@@ -55,15 +55,15 @@ of Tart virtual machines on a cluster of Apple Silicon servers. Concurrently, we
 which will establish a stable API and offer long-term support under a new Fair Source 100 license.

 The Fair Source 100 license for Tart means that once a certain threshold of server installations utilizing 100 CPU cores
-is exceeded, a paid sponsorship will be required. A "server installation" refers to the installation of Tart on a physical
+is exceeded, a paid license will be required. A "server installation" refers to the installation of Tart on a physical
 device without a physical display connected. For example, a Mac Mini with a HDMI Dummy Plug is considered a server,
 but a Mac Mini on a desk with a connected physical display is considered a personal computer. **Usage on personal computers
 and before reaching the 100 CPU cores limit is royalty-free and does not have the viral properties of AGPL.**

-When an organization surpasses the 100 CPU cores limit, they will be required to obtain a [Gold Sponsorship](/licensing#sponsorships),
-which costs \$1000 per month. Upon reaching a limit of 500 CPU cores, a [Platinum Sponsorship](/licensing#sponsorships)
-(\$5000 per month) will be required, and for organizations that exceed 5000 CPU cores, a custom [Diamond Sponsorship](/licensing#sponsorships)
-(\$1 per core per month) will be necessary. **All sponsorships will include priority feature development and SLAs on support with urgent issues.**
+When an organization surpasses the 100 CPU cores limit, they will be required to obtain a [Gold Tier License](/licensing#license-tiers),
+which costs \$1000 per month. Upon reaching a limit of 500 CPU cores, a [Platinum Tier License](/licensing#license-tiers)
+(\$5000 per month) will be required, and for organizations that exceed 5000 CPU cores, a custom [Diamond Tier License](/licensing#license-tiers)
+(\$1 per core per month) will be necessary. **All paid license tiers will include priority feature development and SLAs on support with urgent issues.**

 ## Have we considered alternatives?

@@ -0,0 +1,94 @@
+---
+draft: false
+date: 2023-04-25
+search:
+  exclude: true
+authors:
+  - fkorotkov
+categories:
+  - announcement
+  - orchard
+---
+
+# Announcing Orchard orchestration for managing macOS virtual machines at scale
+
+Today we are happy to announce general availability of Orchard – our new orchestrator to manage Tart virtual machines at scale.
+In this post we’ll cover the motivation behind creating yet another orchestrator and why we didn’t go with Kubernetes or Nomad integration.
+
+## What problem are we trying to solve?
+
+After releasing Tart we pretty quickly started getting requests about managing macOS virtual machines on a cluster of
+Apple Silicon machines rather than just a single host which only allows a maximum of two virtual machines at a time.
+By the end of 2022 the requests reached a tipping point, and we started planning.
+
+<!-- more -->
+
+First, we established some constraints about the end users and potential workload our solution should handle.
+Running macOS or Linux virtual machines on Apple Silicon is a very niche use case. These VMs are either used in
+automation solutions like CI/CD or for managing remote desktop environments. In this case **we are aiming to manage
+only thousands of virtual machines and not millions**.
+
+Second, **operators of such solutions won’t have experience of operating Kubernetes or Nomad**. Operators will most likely
+come with experience of using such systems but not managing them. And again, having built-in things like RBAC and
+ability to scale to millions were appealing but it seemed like it would be a solution for a few rather than a solution
+for everybody to use. Additionally Orchard should provide **first class support for accessing virtual machines over SSH/VNC**
+and support script execution.
+
+By that time, the idea of building a simple opinionated orchestrator got more and more appealing. Plus we kind of already did it
+for [Cirrus CI’s persistent workers](https://cirrus-ci.org/guide/persistent-workers/) feature.
+
+## Technical constraints
+
+With the UX constraints and expectations in place we started thinking about architecture for the orchestrator that we
+started calling **Orchard**.
+
+<script src="https://unpkg.com/@dotlottie/player-component@latest/dist/dotlottie-player.js"></script>
+<dotlottie-player
+src="/assets/animations/Orchard.lottie"
+mode="normal"
+style="width: 100%; height: 360px; margin: auto; background-color: rgb(5 62 94)"
+autoplay
+loop
+/>
+
+Since Orchard will manage a maximum of a couple thousands virtual machines and not millions we **decided to not think much
+about horizontal scalability.** Just a single instance of Orchard controller should be enough if it can restart quickly and
+persist state between restarts.
+
+**Orchard should be secure by default**. All the communication between a controller and workers should be secure.
+All external API requests to Orchard controller should be authorized.
+
+During development it’s crucial to have a quick feedback cycle. **It should be extremely easy to run Orchard in development**.
+Configuring a production cluster should be also easy for novice operators.
+
+## High-level implementation details
+
+Cirrus Labs started as a predominantly Kotlin shop with a little Go. But over the years we gradually moved a lot of things to Go.
+We love the expressibility of Kotlin as a language but the ecosystem for writing system utilities and services is superb in Go.
+
+Orchard is a single Go project that implements both controller server interface and worker client logic in a single repository.
+This simplifies code sharing and testability of the both components and allows to change them in a single pull request.
+
+Another benefit is that Orchard can be distributed as a single binary. We intend to run Orchard controller on a single host.
+Data model for the orchestration didn’t look complex as well. These observations lead us to exploring the use of an embedded database.
+Just imagine! **Orchard can be distributed as a single binary with no external dependencies on any database or runtime!**
+
+And we did exactly that! Orchard is distributed as a single binary that can be run in “controller” mode on a Linux/macOS host and
+in “worker” mode on macOS hosts. Orchard controller is using extremely fast [BadgerDB](https://dgraph.io/docs/badger/) key-value storage to persist data.
+
+## Conclusion
+
+Please give [Orchard](https://github.com/cirruslabs/orchard) a try! To run it locally in development mode on any Apple Silicon device
+please run the following command:
+
+```bash
+brew install cirruslabs/cli/orchard
+orchard dev
+```
+
+This will launch a development cluster with a single worker on your machine. Refer to [Orchard documentation](https://github.com/cirruslabs/orchard#creating-virtual-machines)
+on how to create your first virtual machine and access it.
+
+In a [separate blog post](/blog/2023/04/28/ssh-over-grpc-or-how-orchard-simplifies-accessing-vms-in-private-networks/)
+we’ll cover how Orchard implements seamless SSH access over a gRPC connection. Stay tuned and please don’t hesitate to
+[reach out](https://github.com/cirruslabs/orchard/discussions/landing)! 
--- a/Show More
+++ b/Show More