Introduce AuthenticationKeeper actor to serialize authn modification (#647)

* Fetcher.fetchViaFile(): use an mmap(2)-ed file, similarly to DiskV1

* No need to convert Data to Data

.cirrus.yml

@@ -1,14 +1,16 @@
 use_compute_credits: true
 
 env: 
-  XCODE_TAG: 15-beta-2
+  XCODE_TAG: 15
 
 task:
-  name: Test on Ventura
+  name: Test on Sonoma
   alias: test
   persistent_worker:
     labels:
       name: dev-mini
+    resources:
+      tart-vms: 1
   test_script:
     - swift test
   integration_test_script:
@@ -27,11 +29,19 @@ task:
     path: "integration-tests/pytest-junit.xml"
     format: junit
 
+task:
+  name: Markdown Lint
+  only_if: $CIRRUS_BRANCH != 'gh-pages' && changesInclude('**.md')
+  container:
+    image: node:latest
+  install_script: npm install -g markdownlint-cli
+  lint_script: markdownlint --config=docs/.markdownlint.yml docs/
+
 task:
   name: Lint
   alias: lint
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   lint_script:
     - swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore --lint --report swiftformat.json .
   always:
@@ -40,24 +50,24 @@ task:
       format: swiftformat
 
 task:
+  only_if: $CIRRUS_TAG == ''
   name: Build
   alias: build
-  only_if: $CIRRUS_TAG == ''
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   build_script: swift build --product tart
   sign_script: codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart
   binary_artifacts:
     path: .build/debug/tart
 
 task:
-  name: Release (Dry Run)
   only_if: $CIRRUS_TAG == '' && ($CIRRUS_USER_PERMISSION == 'write' || $CIRRUS_USER_PERMISSION == 'admin')
+  name: Release (Dry Run)
   depends_on:
     - lint
     - build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   env:
     MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -92,7 +102,7 @@ task:
     - test
     - build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:$XCODE_TAG
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:$XCODE_TAG
   env:
     MACOS_CERTIFICATE: ENCRYPTED[552b9d275d1c2bdbc1bff778b104a8f9a53cbd0d59344d4b7f6d0ca3c811a5cefb97bef9ba0ef31c219cb07bdacdd2c2]
     AC_PASSWORD: ENCRYPTED[4a761023e7e06fe2eb350c8b6e8e7ca961af193cb9ba47605f25f1d353abc3142606f412e405be48fd897a78787ea8c2]
@@ -123,9 +133,11 @@ task:
     # Generate and upload symbols
     - dsymutil tart
     - sentry-cli debug-files upload tart.dSYM/
+    - SENTRY_PROJECT=tart sentry-cli debug-files upload tart.dSYM/
     # Bundle and upload sources
     - sentry-cli debug-files bundle-sources tart.dSYM
     - sentry-cli debug-files upload tart.src.zip
+    - SENTRY_PROJECT=tart sentry-cli debug-files upload tart.src.zip
   create_sentry_release_script:
     - export SENTRY_RELEASE="tart@$CIRRUS_TAG"
     - sentry-cli releases new $SENTRY_RELEASE

.goreleaser.yml

@@ -4,6 +4,7 @@ before:
   hooks:
     - .ci/set-version.sh
     - swift build -c release --product tart
+    - gon gon.hcl
     - mkdir -p tart.app/Contents/MacOS
     - cp .build/arm64-apple-macosx/release/tart tart.app/Contents/MacOS/
 
@@ -16,9 +17,6 @@ builds:
     binary: tart.app/Contents/MacOS/tart
     prebuilt:
       path: tart.app/Contents/MacOS/tart
-    hooks:
-      post:
-        - gon gon.hcl
 
 archives:
   - name_template: "{{ .ProjectName }}"
@@ -47,7 +45,7 @@ brews:
       libexec.install Dir["*"]
       bin.write_exec_script "#{libexec}/tart.app/Contents/MacOS/tart"
     custom_block: |
-      depends_on :macos => :monterey
+      depends_on :macos => :ventura
       
       on_macos do
         unless Hardware::CPU.arm?

CONTRIBUTING.md

@@ -0,0 +1,40 @@
+# Contributing to Tart
+
+Table of Contents
+-----------------
+
+- [How to Build](#how-to-build)
+- [How to Create an Issue/Enhancement](#how-to-create-an-issueenhancement)
+- [Style Guidelines](#style-guidelines)
+- [Pull Requests](#Pull-Requests)
+
+## How to Build
+
+1. Fork the repository to your own GitHub account
+2. Clone the forked repository to your local machine
+3. If using Xcode, use from Xcode 15 or newer
+4. Run ./scripts/run-signed.sh from the root of your repository
+
+```bash
+./scripts/run-signed.sh list
+```
+## How to Create an Issue/Enhancement
+
+1. Go to the [Issue page](https://github.com/cirruslabs/tart/issues) of the repository
+2. Click on the "New Issue" button
+3. Provide a descriptive title and detailed description of the issue or enhancement you're suggesting
+4. Submit the issue
+
+## Style Guidelines
+
+1. Code should follow camel case
+2. Code should follow [SwiftFormat](https://github.com/nicklockwood/SwiftFormat#swift-package-manager-plugin) guidelines. You can auto-format the code by running the following command:
+```bash
+swift package plugin --allow-writing-to-package-directory swiftformat --cache ignore .
+```
+
+## Pull Requests
+
+1. Provide a detailed description of the changes you made in the pull request
+2. Wait for pull request to be reviewed 
+3. Make adjustments if necessary

Package.resolved

@@ -23,8 +23,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/getsentry/sentry-cocoa",
       "state" : {
-        "revision" : "ac224c437a3070ffe34460137ac8761eddaf2852",
-        "version" : "8.3.3"
+        "revision" : "d277532e1c8af813981ba01f591b15bbdd735615",
+        "version" : "8.8.0"
       }
     },
     {

Package.swift

@@ -4,7 +4,7 @@ import PackageDescription
 let package = Package(
   name: "Tart",
   platforms: [
-    .macOS(.v12)
+    .macOS(.v13)
   ],
   products: [
     .executable(name: "tart", targets: ["tart"])
@@ -18,7 +18,7 @@ let package = Package(
     .package(url: "https://github.com/antlr/antlr4", branch: "dev"),
     .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.0.0")),
     .package(url: "https://github.com/nicklockwood/SwiftFormat", from: "0.50.6"),
-    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.3.3"),
+    .package(url: "https://github.com/getsentry/sentry-cocoa", from: "8.8.0"),
     .package(url: "https://github.com/cfilipov/TextTable", branch: "master"),
     .package(url: "https://github.com/sersoft-gmbh/swift-sysctl.git", from: "1.0.0"),
   ],

README.md

@@ -6,9 +6,18 @@ Built by CI engineers for your automation needs. Here are some highlights of Tar
 * Tart uses Apple's own `Virtualization.Framework` for [near-native performance](https://browser.geekbench.com/v5/cpu/compare/20382844?baseline=20382722).
 * Push/Pull virtual machines from any OCI-compatible container registry.
 * Use Tart Packer Plugin to automate VM creation.
-* Built-in CI integration.
+* Easily integrates with any CI system.
 
-*Tart* is already adopted by several automation services:
+Tart powers [Cirrus Runners](https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral)
+service — a drop-in replacement for the standard GitHub-hosted runners, offering 2-3 times better performance for a fraction of the price.
+
+<p align="center">
+  <a href="https://tart.run/integrations/github-actions/?utm_source=github&utm_medium=referral" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/CirrusRunnersForGHA.png" height="65"/>
+  </a>
+</p>
+
+Tart is also adopted by several other automation services:
 
 <p align="center">
   <a href="https://cirrus-ci.org/guide/macOS/" target=_blank>
@@ -43,6 +52,9 @@ Many more companies are using Tart in their internal setups. Here are a few of t
   <a href="https://transloadit.com/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png" height="65"/>
   </a>
+  <a href="https://uphold.com/" target=_blank>
+    <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png" height="65"/>
+  </a>
   <a href="https://www.pitsdatarecovery.net/" target=_blank>
     <img src="https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png" height="65"/>
   </a>
@@ -52,12 +64,12 @@ Many more companies are using Tart in their internal setups. Here are a few of t
 
 ## Usage
 
-Try running a Tart VM on your Apple Silicon device running macOS 12.0 (Monterey) or later (will download a 25 GB image):
+Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura) or later (will download a 25 GB image):
 
 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
-tart run ventura-base
+tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+tart run sonoma-base
 ```
 
 Please check the [official documentation](https://tart.run) for more information and/or feel free to use [discussions](https://github.com/cirruslabs/tart/discussions)

Resources/tart-dev.entitlements

@@ -4,5 +4,7 @@
 <dict>
 	<key>com.apple.security.virtualization</key>
 	<true/>
+	<key>com.apple.security.get-task-allow</key>
+	<true/>
 </dict>
 </plist>

Sources/tart/Commands/Clone.swift

@@ -3,7 +3,20 @@ import Foundation
 import SystemConfiguration
 
 struct Clone: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Clone a VM")
+  static var configuration = CommandConfiguration(
+    abstract: "Clone a VM",
+    discussion: """
+    Creates a local virtual machine by cloning either a remote or another local virtual machine.
+
+    Due to copy-on-write magic in Apple File System a cloned VM won't actually claim all the space right away.
+    Only changes to a cloned disk will be written and claim new space. By default, Tart checks available capacity
+    in Tart's home directory and checks if there is enough space for the worst possible scenario: when the whole disk
+    will be modified.
+
+    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable. This might be helpful
+    for use cases when the original image is very big and a workload is known to only modify a fraction of the cloned disk.
+    """
+  )
 
   @Argument(help: "source VM name")
   var sourceName: String
@@ -14,10 +27,17 @@ struct Clone: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+  var concurrency: UInt = 4
+
   func validate() throws {
     if newName.contains("/") {
       throw ValidationError("<new-name> should be a local name")
     }
+
+    if concurrency < 1 {
+      throw ValidationError("network concurrency cannot be less than 1")
+    }
   }
 
   func run() async throws {
@@ -27,12 +47,10 @@ struct Clone: AsyncParsableCommand {
     if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
       // Pull the VM in case it's OCI-based and doesn't exist locally yet
       let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
-      try await ociStorage.pull(remoteName, registry: registry)
+      try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency)
     }
 
     let sourceVM = try VMStorageHelper.open(sourceName)
-    try Prune.reclaimIfNeeded(UInt64(sourceVM.sizeBytes()))
-
     let tmpVMDir = try VMDirectory.temporary()
 
     // Lock the temporary VM directory to prevent it's garbage collection
@@ -45,10 +63,17 @@ struct Clone: AsyncParsableCommand {
       try lock.lock()
 
       let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
+        && sourceVM.state() != "suspended"
       try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)
+
       try localStorage.move(newName, from: tmpVMDir)
 
       try lock.unlock()
+
+      // APFS is doing copy-on-write so the above cloning operation (just copying files on disk)
+      // is not actually claiming new space until the VM is started and it writes something to disk.
+      // So once we clone the VM let's try to claim a little bit of space for the VM to run.
+      try Prune.reclaimIfNeeded(UInt64(sourceVM.sizeBytes()), sourceVM)
     }, onCancel: {
       try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
     })

Sources/tart/Commands/Create.swift

@@ -47,11 +47,7 @@ struct Create: AsyncParsableCommand {
       }
 
       if linux {
-        if #available(macOS 13, *) {
-          _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
-        } else {
-          throw UnsupportedOSError("Linux VMs", "are")
-        }
+        _ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
       }
 
       try VMStorageLocal().move(name, from: tmpVMDir)

Sources/tart/Commands/Get.swift

@@ -7,6 +7,7 @@ fileprivate struct VMInfo: Encodable {
   let Disk: Int
   let Display: String
   let Running: Bool
+  let State: String
 }
 
 struct Get: AsyncParsableCommand {
@@ -25,7 +26,7 @@ struct Get: AsyncParsableCommand {
     let memorySizeInMb = vmConfig.memorySize / 1024 / 1024
 
     let info = VMInfo(CPU: vmConfig.cpuCount, Memory: memorySizeInMb, Disk: diskSizeInGb,
-                      Display: vmConfig.display.description, Running: try vmDir.running())
+                      Display: vmConfig.display.description, Running: try vmDir.running(), State: try vmDir.state())
     print(format.renderSingle(info))
   }
 }

Sources/tart/Commands/List.swift

@@ -7,6 +7,7 @@ fileprivate struct VMInfo: Encodable {
   let Name: String
   let Size: Int
   let Running: Bool
+  let State: String
 }
 
 struct List: AsyncParsableCommand {
@@ -36,13 +37,13 @@ struct List: AsyncParsableCommand {
 
     if source == nil || source == "local" {
       infos += sortedInfos(try VMStorageLocal().list().map { (name, vmDir) in
-        try VMInfo(Source: "local", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running())
+        try VMInfo(Source: "local", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running(), State: vmDir.state())
       })
     }
 
     if source == nil || source == "oci" {
       infos += sortedInfos(try VMStorageOCI().list().map { (name, vmDir, _) in
-        try VMInfo(Source: "oci", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running())
+        try VMInfo(Source: "oci", Name: name, Size: vmDir.sizeGB(), Running: vmDir.running(), State: vmDir.state())
       })
     }
 

Sources/tart/Commands/Login.swift

@@ -17,6 +17,9 @@ struct Login: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Flag(help: "skip validation of the registry's credentials before logging-in")
+  var noValidate: Bool = false
+
   func validate() throws {
     let usernameProvided = username != nil
     let passwordProvided = passwordStdin
@@ -45,12 +48,14 @@ struct Login: AsyncParsableCommand {
       host: (user, password)
     ])
 
-    do {
-      let registry = try Registry(host: host, namespace: "", insecure: insecure,
-                                  credentialsProviders: [credentialsProvider])
-      try await registry.ping()
-    } catch {
-      throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
+    if !noValidate {
+      do {
+        let registry = try Registry(host: host, namespace: "", insecure: insecure,
+                                    credentialsProviders: [credentialsProvider])
+        try await registry.ping()
+      } catch {
+        throw RuntimeError.InvalidCredentials("invalid credentials: \(error)")
+      }
     }
 
     try KeychainCredentialsProvider().store(host: host, user: user, password: password)

Sources/tart/Commands/Logout.swift

@@ -0,0 +1,14 @@
+import ArgumentParser
+import Dispatch
+import SwiftUI
+
+struct Logout: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(abstract: "Logout from a registry")
+
+  @Argument(help: "host")
+  var host: String
+
+  func run() async throws {
+    try KeychainCredentialsProvider().remove(host: host)
+  }
+}

Sources/tart/Commands/Prune.swift

@@ -5,23 +5,40 @@ import SwiftUI
 import SwiftDate
 
 struct Prune: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches")
+  static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches or local VMs")
 
-  @Option(help: ArgumentHelp("Remove cache entries last accessed more than n days ago",
+  @Option(help: ArgumentHelp("Entries to remove: \"caches\" targets OCI and IPSW caches and \"vms\" targets local VMs."))
+  var entries: String = "caches"
+
+  @Option(help: ArgumentHelp("Remove entries that were last accessed more than n days ago",
                              discussion: "For example, --older-than=7 will remove entries that weren't accessed by Tart in the last 7 days.",
                              valueName: "n"))
   var olderThan: UInt?
 
-  @Option(help: ArgumentHelp("Remove least recently used cache entries that do not fit the specified cache size budget n, expressed in gigabytes",
-                             discussion: "For example, --cache-budget=50 will effectively shrink all caches to a total size of 50 gigabytes.",
-                             valueName: "n"))
+  @Option(help: .hidden)
   var cacheBudget: UInt?
 
+  @Option(help: ArgumentHelp("Remove the least recently used entries that do not fit the specified space size budget n, expressed in gigabytes",
+                             discussion: "For example, --space-budget=50 will effectively shrink all entries to a total size of 50 gigabytes.",
+                             valueName: "n"))
+  var spaceBudget: UInt?
+
   @Flag(help: .hidden)
   var gc: Bool = false
 
-  func validate() throws {
-    if olderThan == nil && cacheBudget == nil && !gc {
+  mutating func validate() throws {
+    // --cache-budget deprecation logic
+    if let cacheBudget = cacheBudget {
+      fputs("--cache-budget is deprecated, please use --space-budget\n", stderr)
+
+      if spaceBudget != nil {
+        throw ValidationError("--cache-budget is deprecated, please use --space-budget")
+      }
+
+      spaceBudget = cacheBudget
+    }
+
+    if olderThan == nil && spaceBudget == nil && !gc {
       throw ValidationError("at least one pruning criteria must be specified")
     }
   }
@@ -31,43 +48,53 @@ struct Prune: AsyncParsableCommand {
       try VMStorageOCI().gc()
     }
 
+    // Build a list of prunable storages that we're going to prune based on user's request
+    let prunableStorages: [PrunableStorage]
+
+    switch entries {
+    case "caches":
+      prunableStorages = [VMStorageOCI(), try IPSWCache()]
+    case "vms":
+      prunableStorages = [VMStorageLocal()]
+    default:
+      throw ValidationError("unsupported --entries value, please specify either \"caches\" or \"vms\"")
+    }
+
     // Clean up cache entries based on last accessed date
     if let olderThan = olderThan {
       let olderThanInterval = Int(exactly: olderThan)!.days.timeInterval
-      let olderThanDate = Date().addingTimeInterval(olderThanInterval)
+      let olderThanDate = Date() - olderThanInterval
 
-      try Prune.pruneOlderThan(olderThanDate: olderThanDate)
+      try Prune.pruneOlderThan(prunableStorages: prunableStorages, olderThanDate: olderThanDate)
     }
 
     // Clean up cache entries based on imposed cache size limit and entry's last accessed date
-    if let cacheBudget = cacheBudget {
-      try Prune.pruneCacheBudget(cacheBudgetBytes: UInt64(cacheBudget) * 1024 * 1024 * 1024)
+    if let spaceBudget = spaceBudget {
+      try Prune.pruneSpaceBudget(prunableStorages: prunableStorages, spaceBudgetBytes: UInt64(spaceBudget) * 1024 * 1024 * 1024)
     }
   }
 
-  static func pruneOlderThan(olderThanDate: Date) throws {
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
+  static func pruneOlderThan(prunableStorages: [PrunableStorage], olderThanDate: Date) throws {
     let prunables: [Prunable] = try prunableStorages.flatMap { try $0.prunables() }
 
     try prunables.filter { try $0.accessDate() <= olderThanDate }.forEach { try $0.delete() }
   }
 
-  static func pruneCacheBudget(cacheBudgetBytes: UInt64) throws {
-    let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
+  static func pruneSpaceBudget(prunableStorages: [PrunableStorage], spaceBudgetBytes: UInt64) throws {
     let prunables: [Prunable] = try prunableStorages
       .flatMap { try $0.prunables() }
       .sorted { try $0.accessDate() > $1.accessDate() }
 
-    var cacheBudgetBytes = cacheBudgetBytes
+    var spaceBudgetBytes = spaceBudgetBytes
     var prunablesToDelete: [Prunable] = []
 
     for prunable in prunables {
       let prunableSizeBytes = UInt64(try prunable.sizeBytes())
 
-      if prunableSizeBytes <= cacheBudgetBytes {
+      if prunableSizeBytes <= spaceBudgetBytes {
         // Don't mark for deletion as
         // there's a budget available
-        cacheBudgetBytes -= prunableSizeBytes
+        spaceBudgetBytes -= prunableSizeBytes
       } else {
         // Mark for deletion
         prunablesToDelete.append(prunable)
@@ -77,7 +104,11 @@ struct Prune: AsyncParsableCommand {
     try prunablesToDelete.forEach { try $0.delete() }
   }
 
-  static func reclaimIfNeeded(_ requiredBytes: UInt64) throws {
+  static func reclaimIfNeeded(_ requiredBytes: UInt64, _ initiator: Prunable? = nil) throws {
+    if ProcessInfo.processInfo.environment.keys.contains("TART_NO_AUTO_PRUNE") {
+      return
+    }
+
     SentrySDK.configureScope { scope in
       scope.setContext(value: ["requiredBytes": requiredBytes], key: "Prune")
     }
@@ -114,10 +145,10 @@ struct Prune: AsyncParsableCommand {
       return
     }
 
-    try Prune.reclaimIfPossible(requiredBytes - volumeAvailableCapacityCalculated)
+    try Prune.reclaimIfPossible(requiredBytes - volumeAvailableCapacityCalculated, initiator)
   }
 
-  private static func reclaimIfPossible(_ reclaimBytes: UInt64) throws {
+  private static func reclaimIfPossible(_ reclaimBytes: UInt64, _ initiator: Prunable? = nil) throws {
     let transaction = SentrySDK.startTransaction(name: "Pruning cache", operation: "prune", bindToScope: true)
     defer { transaction.finish() }
 
@@ -141,6 +172,11 @@ struct Prune: AsyncParsableCommand {
         break
       }
 
+      if prunable.url == initiator?.url.resolvingSymlinksInPath() {
+        // do not prune the initiator
+        continue
+      }
+
       try SentrySDK.span?.setData(value: prunable.sizeBytes(), key: prunable.url.path)
 
       cacheReclaimedBytes += try prunable.sizeBytes()

Sources/tart/Commands/Pull.swift

@@ -3,7 +3,16 @@ import Dispatch
 import SwiftUI
 
 struct Pull: AsyncParsableCommand {
-  static var configuration = CommandConfiguration(abstract: "Pull a VM from a registry")
+  static var configuration = CommandConfiguration(
+    abstract: "Pull a VM from a registry",
+    discussion: """
+    Pulls a virtual machine from a remote OCI-compatible registry. Supports authorization via Keychain (see "tart login --help"),
+    Docker credential helpers defined in ~/.docker/config.json or via TART_REGISTRY_USERNAME/TART_REGISTRY_PASSWORD environment variables.
+
+    By default, Tart checks available capacity in Tart's home directory and tries to reclaim minimum possible storage for the remote image to fit via "tart prune".
+    This behaviour can be disabled by setting TART_NO_AUTO_PRUNE environment variable.
+    """
+  )
 
   @Argument(help: "remote VM name")
   var remoteName: String
@@ -11,6 +20,15 @@ struct Pull: AsyncParsableCommand {
   @Flag(help: "connect to the OCI registry via insecure HTTP protocol")
   var insecure: Bool = false
 
+  @Option(help: "network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+  var concurrency: UInt = 4
+
+  func validate() throws {
+    if concurrency < 1 {
+      throw ValidationError("network concurrency cannot be less than 1")
+    }
+  }
+
   func run() async throws {
     // Be more liberal when accepting local image as argument,
     // see https://github.com/cirruslabs/tart/issues/36
@@ -25,6 +43,6 @@ struct Pull: AsyncParsableCommand {
 
     defaultLogger.appendNewLine("pulling \(remoteName)...")
 
-    try await VMStorageOCI().pull(remoteName, registry: registry)
+    try await VMStorageOCI().pull(remoteName, registry: registry, concurrency: concurrency)
   }
 }

Sources/tart/Commands/Push.swift

@@ -23,6 +23,9 @@ struct Push: AsyncParsableCommand {
                              """))
   var chunkSize: Int = 0
 
+  @Option(help: .hidden)
+  var diskFormat: String = "v2"
+
   @Flag(help: ArgumentHelp("cache pushed images locally",
                            discussion: "Increases disk usage, but saves time if you're going to pull the pushed images later."))
   var populateCache: Bool = false
@@ -69,7 +72,8 @@ struct Push: AsyncParsableCommand {
         pushedRemoteName = try await localVMDir.pushToRegistry(
           registry: registry,
           references: references,
-          chunkSizeMb: chunkSize
+          chunkSizeMb: chunkSize,
+          diskFormat: diskFormat
         )
         // Populate the local cache (if requested)
         if populateCache {
@@ -100,7 +104,7 @@ struct Push: AsyncParsableCommand {
       _ = try await registry.pushManifest(reference: reference, manifest: remoteManifest)
     }
 
-    return RemoteName(host: registry.baseURL.host!, namespace: registry.namespace,
+    return RemoteName(host: registry.host!, namespace: registry.namespace,
                       reference: Reference(digest: digest))
   }
 }

Sources/tart/Commands/Run.swift

@@ -1,5 +1,6 @@
 import ArgumentParser
 import Cocoa
+import Darwin
 import Dispatch
 import SwiftUI
 import Virtualization
@@ -51,10 +52,17 @@ struct Run: AsyncParsableCommand {
   var vncExperimental: Bool = false
 
   @Option(help: ArgumentHelp("""
-  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\")
+  Additional disk attachments with an optional read-only specifier\n(e.g. --disk=\"disk.bin\" --disk=\"ubuntu.iso:ro\" --disk=\"/dev/disk0\")
   """, discussion: """
+  Can be either a disk image file or a block device like a local SSD on AWS EC2 Mac instances.
+
   Learn how to create a disk image using Disk Utility here:
   https://support.apple.com/en-gb/guide/disk-utility/dskutl11888/mac
+
+  To work with block devices 'tart' binary must be executed as root which affects locating Tart VMs.
+  To workaround this issue pass TART_HOME explicitly:
+
+  sudo TART_HOME="$HOME/.tart" tart run sonoma --disk=/dev/disk0
   """, valueName: "path[:ro]"))
   var disk: [String] = []
 
@@ -73,13 +81,17 @@ struct Run: AsyncParsableCommand {
   var rosettaTag: String?
 
   @Option(help: ArgumentHelp("""
-  Additional directory shares with an optional read-only specifier\n(e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\")
+  Additional directory shares with an optional read-only specifier\n(e.g. --dir=\"~/src/build\" or --dir=\"~/src/sources:ro\")
   """, discussion: """
   Requires host to be macOS 13.0 (Ventura) or newer.
-  All shared directories are automatically mounted to "/Volumes/My Shared Files" directory on macOS,
+  A shared directory is automatically mounted to "/Volumes/My Shared Files" directory on macOS,
   while on Linux you have to do it manually: "mount -t virtiofs com.apple.virtio-fs.automount /mount/point".
   For macOS guests, they must be running macOS 13.0 (Ventura) or newer.
-  """, valueName: "name:path[:ro]"))
+
+  In case of passing multiple directories it is required to prefix them with names e.g. --dir=\"build:~/src/build\" --dir=\"sources:~/src/sources:ro\"
+  These names will be used as directory names under the mounting point inside guests. For the example above it will be
+  "/Volumes/My Shared Files/build" and "/Volumes/My Shared Files/sources" respectively.
+  """, valueName: "[name:]path[:ro]"))
   var dir: [String] = []
 
   @Option(help: ArgumentHelp("""
@@ -87,28 +99,60 @@ struct Run: AsyncParsableCommand {
   """, discussion: """
   Specify "list" as an interface name (--net-bridged=list) to list the available bridged interfaces.
   """, valueName: "interface name"))
-  var netBridged: String?
+  var netBridged: [String] = []
 
   @Flag(help: ArgumentHelp("Use software networking instead of the default shared (NAT) networking",
                            discussion: "Learn how to configure Softnet for use with Tart here: https://github.com/cirruslabs/softnet"))
   var netSoftnet: Bool = false
 
-  func validate() throws {
+  @Flag(help: ArgumentHelp("Disables audio and entropy devices and switches to only Mac-specific input devices.", discussion: "Useful for running a VM that can be suspended via \"tart suspend\"."))
+  var suspendable: Bool = false
+
+  mutating func validate() throws {
     if vnc && vncExperimental {
       throw ValidationError("--vnc and --vnc-experimental are mutually exclusive")
     }
-    if netBridged != nil && netSoftnet {
+
+    if netBridged.count > 0 && netSoftnet {
       throw ValidationError("--net-bridged and --net-softnet are mutually exclusive")
     }
 
     if graphics && noGraphics {
       throw ValidationError("--graphics and --no-graphics are mutually exclusive")
     }
+
+    let localStorage = VMStorageLocal()
+    let vmDir = try localStorage.open(name)
+    if try vmDir.state() == "suspended" {
+      suspendable = true
+    }
+
+    if suspendable {
+      if dir.count > 0 {
+        throw ValidationError("Suspending VMs with shared directories is not supported")
+      }
+    }
   }
 
   @MainActor
   func run() async throws {
-    let vmDir = try VMStorageLocal().open(name)
+    let localStorage = VMStorageLocal()
+    let vmDir = try localStorage.open(name)
+
+    let storageLock = try FileLock(lockURL: Config().tartHomeDir)
+    if try vmDir.state() == "suspended" {
+      try storageLock.lock() // lock before checking
+      let needToGenerateNewMac = try localStorage.list().contains {
+        // check if there is a running VM with the same MAC but different name
+        try $1.running() && $1.macAddress() == vmDir.macAddress() && $1.name != vmDir.name
+      }
+
+      if needToGenerateNewMac {
+        print("There is already a running VM with the same MAC address!")
+        print("Resetting VM to assign a new MAC address...")
+        try vmDir.regenerateMACAddress()
+      }
+    }
 
     if netSoftnet && isInteractiveSession() {
       try Softnet.configureSUIDBitIfNeeded()
@@ -116,20 +160,6 @@ struct Run: AsyncParsableCommand {
 
     let additionalDiskAttachments = try additionalDiskAttachments()
 
-    // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
-    // see https://github.com/cirruslabs/tart/issues/323 for more details.
-    for additionalDiskAttachment in additionalDiskAttachments {
-      // Read-only attachments do not seem to acquire the lock
-      if additionalDiskAttachment.isReadOnly {
-        continue
-      }
-
-      if try !FileLock(lockURL: additionalDiskAttachment.url).trylock() {
-        throw RuntimeError.DiskAlreadyInUse("disk \(additionalDiskAttachment.url.path) seems to be already in use, "
-          + "unmount it first in Finder")
-      }
-    }
-
     var serialPorts: [VZSerialPortConfiguration] = []
     if serial {
       let tty_fd = createPTY()
@@ -151,9 +181,10 @@ struct Run: AsyncParsableCommand {
     vm = try VM(
       vmDir: vmDir,
       network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
-      additionalDiskAttachments: additionalDiskAttachments,
+      additionalStorageDevices: additionalDiskAttachments,
       directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
-      serialPorts: serialPorts
+      serialPorts: serialPorts,
+      suspendable: suspendable
     )
 
     let vncImpl: VNC? = try {
@@ -184,8 +215,25 @@ struct Run: AsyncParsableCommand {
       throw RuntimeError.VMAlreadyRunning("VM \"\(name)\" is already running!")
     }
 
+    // now VM state will return "running" so we can unlock
+    try storageLock.unlock()
+
     let task = Task {
       do {
+        var resume = false
+
+        if #available(macOS 14, *) {
+          if FileManager.default.fileExists(atPath: vmDir.stateURL.path) {
+            print("restoring VM state from a snapshot...")
+            try await vm!.virtualMachine.restoreMachineStateFrom(url: vmDir.stateURL)
+            try FileManager.default.removeItem(at: vmDir.stateURL)
+            resume = true
+            print("resuming VM...")
+          }
+        }
+
+        try await vm!.start(recovery: recovery, resume: resume)
+
         if let vncImpl = vncImpl {
           let vncURL = try await vncImpl.waitForURL()
 
@@ -197,7 +245,7 @@ struct Run: AsyncParsableCommand {
           }
         }
 
-        try await vm!.run(recovery)
+        try await vm!.run()
 
         if let vncImpl = vncImpl {
           try vncImpl.stop()
@@ -215,17 +263,50 @@ struct Run: AsyncParsableCommand {
       }
     }
 
+    // "tart stop" support
     let sigintSrc = DispatchSource.makeSignalSource(signal: SIGINT)
     sigintSrc.setEventHandler {
       task.cancel()
     }
     sigintSrc.activate()
 
+    // "tart suspend" / UI window closing support
+    signal(SIGUSR1, SIG_IGN)
+    let sigusr1Src = DispatchSource.makeSignalSource(signal: SIGUSR1)
+    sigusr1Src.setEventHandler {
+      Task {
+        do {
+          if #available(macOS 14, *) {
+            try vm!.configuration.validateSaveRestoreSupport()
+
+            print("pausing VM to take a snapshot...")
+            try await vm!.virtualMachine.pause()
+
+            print("creating a snapshot...")
+            try await vm!.virtualMachine.saveMachineStateTo(url: vmDir.stateURL)
+
+            print("snapshot created successfully! shutting down the VM...")
+
+            task.cancel()
+          } else {
+            print(RuntimeError.SuspendFailed("this functionality is only supported on macOS 14 (Sonoma) or newer"))
+
+            Foundation.exit(1)
+          }
+        } catch (let e) {
+          print(RuntimeError.SuspendFailed(e.localizedDescription))
+
+          Foundation.exit(1)
+        }
+      }
+    }
+    sigusr1Src.activate()
+
     let useVNCWithoutGraphics = (vnc || vncExperimental) && !graphics
     if noGraphics || useVNCWithoutGraphics {
       dispatchMain()
     } else {
-      runUI()
+      runUI(suspendable)
     }
   }
 
@@ -250,23 +331,19 @@ struct Run: AsyncParsableCommand {
       return try Softnet(vmMACAddress: config.macAddress.string)
     }
 
-    if let netBridged = netBridged {
-      let matchingInterfaces = VZBridgedNetworkInterface.networkInterfaces.filter { interface in
-        interface.identifier == netBridged || interface.localizedDisplayName == netBridged
+    if netBridged.count > 0 {
+      func findBridgedInterface(_ name: String) throws -> VZBridgedNetworkInterface {
+        let interface = VZBridgedNetworkInterface.networkInterfaces.first { interface in
+          interface.identifier == name || interface.localizedDisplayName == name
+        }
+        if (interface == nil) {
+          throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
+            + "available interfaces: \(bridgeInterfaces())")
+        }
+        return interface!
       }
 
-      if matchingInterfaces.isEmpty {
-        let available = bridgeInterfaces().joined(separator: ", ")
-        throw ValidationError("no bridge interfaces matched \"\(netBridged)\", "
-          + "available interfaces: \(available)")
-      }
-
-      if matchingInterfaces.count > 1 {
-        throw ValidationError("more than one bridge interface matched \"\(netBridged)\", "
-          + "consider refining the search criteria")
-      }
-
-      return NetworkBridged(interface: matchingInterfaces.first!)
+      return NetworkBridged(interfaces: try netBridged.map { try findBridgedInterface($0) })
     }
 
     return nil
@@ -284,22 +361,43 @@ struct Run: AsyncParsableCommand {
     }
   }
 
-  func additionalDiskAttachments() throws -> [VZDiskImageStorageDeviceAttachment] {
-    var result: [VZDiskImageStorageDeviceAttachment] = []
+  func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
+    var result: [VZStorageDeviceConfiguration] = []
     let readOnlySuffix = ":ro"
     let expandedDiskPaths = disk.map { NSString(string:$0).expandingTildeInPath }
 
     for rawDisk in expandedDiskPaths {
-      if rawDisk.hasSuffix(readOnlySuffix) {
-        result.append(try VZDiskImageStorageDeviceAttachment(
-          url: URL(fileURLWithPath: String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count))),
-          readOnly: true
-        ))
+      let diskReadOnly = rawDisk.hasSuffix(readOnlySuffix)
+      let diskPath = diskReadOnly ? String(rawDisk.prefix(rawDisk.count - readOnlySuffix.count)) : rawDisk
+      let diskURL = URL(fileURLWithPath: diskPath)
+
+      // check if `diskPath` is a block device or a directory
+      if pathHasMode(diskPath, mode: S_IFBLK) || pathHasMode(diskPath, mode: S_IFDIR) {
+        print("Using block device\n")
+        guard #available(macOS 14, *) else {
+          throw UnsupportedOSError("attaching block devices", "are")
+        }
+        let fileHandle = FileHandle(forUpdatingAtPath: diskPath)
+        guard fileHandle != nil else {
+          if ProcessInfo.processInfo.userName != "root" {
+            throw RuntimeError.VMConfigurationError("need to run as root to work with block devices")
+          }
+          throw RuntimeError.VMConfigurationError("block device \(diskURL.url.path) seems to be already in use, unmount it first via 'diskutil unmount'")
+        }
+        let attachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: fileHandle!, readOnly: diskReadOnly, synchronizationMode: .full)
+        result.append(VZVirtioBlockDeviceConfiguration(attachment: attachment))
       } else {
-        result.append(try VZDiskImageStorageDeviceAttachment(
-          url: URL(fileURLWithPath: rawDisk),
-          readOnly: false
-        ))
+        // Error out if the disk is locked by the host (e.g. it was mounted in Finder),
+        // see https://github.com/cirruslabs/tart/issues/323 for more details.
+        if try !diskReadOnly && !FileLock(lockURL: diskURL).trylock() {
+          throw RuntimeError.DiskAlreadyInUse("disk \(diskURL.url.path) seems to be already in use, unmount it first in Finder")
+        }
+
+        let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(
+          url: diskURL,
+          readOnly: diskReadOnly
+        )
+        result.append(VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
       }
     }
 
@@ -315,46 +413,31 @@ struct Run: AsyncParsableCommand {
       throw UnsupportedOSError("directory sharing", "is")
     }
 
-    struct DirectoryShare {
-      let name: String
-      let path: URL
-      let readOnly: Bool
-    }
-
     var directoryShares: [DirectoryShare] = []
 
+    var allNamedShares = true
     for rawDir in dir {
-      let splits = rawDir.split(maxSplits: 2) { $0 == ":" }
-
-      if splits.count < 2 {
-        throw ValidationError("invalid --dir syntax: should at least include name and path, colon-separated")
+      let directoryShare = try DirectoryShare(parseFrom: rawDir)
+      if (directoryShare.name == nil) {
+        allNamedShares = false
       }
-
-      var readOnly: Bool = false
-
-      if splits.count == 3 {
-        if splits[2] == "ro" {
-          readOnly = true
-        } else {
-          throw ValidationError("invalid --dir syntax: optional read-only specifier can only be \"ro\"")
-        }
-      }
-
-      let (name, path) = (String(splits[0]), String(splits[1]))
-
-      directoryShares.append(DirectoryShare(
-        name: name,
-        path: URL(fileURLWithPath: NSString(string: path).expandingTildeInPath),
-        readOnly: readOnly)
-      )
+      directoryShares.append(directoryShare)
     }
 
-    var directories: [String : VZSharedDirectory] = Dictionary()
-    directoryShares.forEach { directories[$0.name] = VZSharedDirectory(url: $0.path, readOnly: $0.readOnly) }
 
     let automountTag = VZVirtioFileSystemDeviceConfiguration.macOSGuestAutomountTag
     let sharingDevice = VZVirtioFileSystemDeviceConfiguration(tag: automountTag)
-    sharingDevice.share = VZMultipleDirectoryShare(directories: directories)
+    if allNamedShares {
+      var directories: [String : VZSharedDirectory] = Dictionary()
+      try directoryShares.forEach { directories[$0.name!] = try $0.createConfiguration() }
+      sharingDevice.share = VZMultipleDirectoryShare(directories: directories)
+    } else if dir.count > 1 {
+      throw ValidationError("invalid --dir syntax: for multiple directory shares each one of them should be named")
+    } else if dir.count == 1 {
+      let directoryShare = directoryShares.first!
+      let singleDirectoryShare = VZSingleDirectoryShare(directory: try directoryShare.createConfiguration())
+      sharingDevice.share = singleDirectoryShare
+    }
 
     return [sharingDevice]
   }
@@ -384,7 +467,7 @@ struct Run: AsyncParsableCommand {
     return [device]
   }
 
-  private func runUI() {
+  private func runUI(_ suspendable: Bool) {
     let nsApp = NSApplication.shared
     nsApp.setActivationPolicy(.regular)
     nsApp.activate(ignoringOtherApps: true)
@@ -392,6 +475,8 @@ struct Run: AsyncParsableCommand {
     nsApp.applicationIconImage = NSImage(data: AppIconData)
 
     struct MainApp: App {
+      static var disappearSignal: Int32 = SIGINT
+
       @NSApplicationDelegateAdaptor private var appDelegate: MinimalMenuAppDelegate
 
       var body: some Scene {
@@ -400,7 +485,7 @@ struct Run: AsyncParsableCommand {
             VMView(vm: vm!).onAppear {
               NSWindow.allowsAutomaticWindowTabbing = false
             }.onDisappear {
-              let ret = kill(getpid(), SIGINT)
+              let ret = kill(getpid(), MainApp.disappearSignal)
               if ret != 0 {
                 // Fallback to the old termination method that doesn't
                 // propagate the cancellation to Task's in case graceful
@@ -436,11 +521,17 @@ struct Run: AsyncParsableCommand {
             Button("Request Stop") {
               Task { try vm!.virtualMachine.requestStop() }
             }
+            if #available(macOS 14, *) {
+              Button("Suspend") {
+                kill(getpid(), SIGUSR1)
+              }
+            }
           }
         }
       }
     }
 
+    MainApp.disappearSignal = suspendable ? SIGUSR1 : SIGINT
     MainApp.main()
   }
 }
@@ -512,3 +603,116 @@ struct VMView: NSViewRepresentable {
     nsView.virtualMachine = vm.virtualMachine
   }
 }
+
+struct DirectoryShare {
+  let name: String?
+  let path: URL
+  let readOnly: Bool
+
+  init(parseFrom: String) throws {
+    let readOnlySuffix = ":ro"
+    readOnly = parseFrom.hasSuffix(readOnlySuffix)
+    let maybeNameAndURL = readOnly ? String(parseFrom.dropLast(readOnlySuffix.count)) : parseFrom
+
+    if maybeNameAndURL.starts(with: "https://") || maybeNameAndURL.starts(with: "http://") {
+      // just a URL
+      name = nil
+      path = URL(string: maybeNameAndURL)!
+      return
+    }
+
+    let splits = maybeNameAndURL.split(separator: ":", maxSplits: 1)
+
+    if splits.count == 2 {
+      name = String(splits[0])
+      path = String(splits[1]).toRemoteOrLocalURL()
+    } else {
+      name = nil
+      path = String(splits[0]).toRemoteOrLocalURL()
+    }
+  }
+
+  func createConfiguration() throws -> VZSharedDirectory {
+    if (path.isFileURL) {
+      return VZSharedDirectory(url: path, readOnly: readOnly)
+    }
+
+    let urlCache = URLCache(memoryCapacity: 0, diskCapacity: 1 * 1024 * 1024 * 1024)
+
+    let archiveRequest = URLRequest(url: path, cachePolicy: .returnCacheDataElseLoad)
+    var response: CachedURLResponse? = urlCache.cachedResponse(for: archiveRequest)
+    if (response == nil) {
+      print("Downloading \(path)...")
+      // download and unarchive remote directories if needed here
+      // use old school API to prevent deadlocks since we are running via MainActor
+      let downloadSemaphore = DispatchSemaphore(value: 0)
+      Task {
+        do {
+          let (archiveData, archiveResponse) = try await URLSession.shared.data(for: archiveRequest)
+          urlCache.storeCachedResponse(CachedURLResponse(response: archiveResponse, data: archiveData, storagePolicy: .allowed), for: archiveRequest)
+          print("Cached for future invocations!")
+        } catch {
+          print("Download failed: \(error)")
+        }
+        downloadSemaphore.signal()
+      }
+      downloadSemaphore.wait()
+      response = urlCache.cachedResponse(for: archiveRequest)
+    } else {
+      print("Using cached archive for \(path)...")
+    }
+
+    if (response == nil) {
+      throw ValidationError("Failed to fetch a remote archive!")
+    }
+
+    let temporaryLocation = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString + ".volume")
+    try FileManager.default.createDirectory(atPath: temporaryLocation.path, withIntermediateDirectories: true)
+    let lock = try FileLock(lockURL: temporaryLocation)
+    try lock.lock()
+
+    guard let executableURL = resolveBinaryPath("tar") else {
+      throw ValidationError("tar not found in PATH")
+    }
+
+    let process = Process.init()
+    process.executableURL = executableURL
+    process.currentDirectoryURL = temporaryLocation
+    process.arguments = ["-xz"]
+
+    let inPipe = Pipe()
+    process.standardInput = inPipe
+    process.launch()
+
+    inPipe.fileHandleForWriting.write(response!.data)
+    try inPipe.fileHandleForWriting.close()
+    process.waitUntilExit()
+
+    if !(process.terminationReason == .exit && process.terminationStatus == 0) {
+      throw ValidationError("Unarchiving failed!")
+    }
+
+    print("Unarchived into a temporary directory!")
+
+    return VZSharedDirectory(url: temporaryLocation, readOnly: readOnly)
+  }
+}
+
+extension String {
+  func toRemoteOrLocalURL() -> URL {
+    if (starts(with: "https://") || starts(with: "https://")) {
+      URL(string: self)!
+    } else {
+      URL(fileURLWithPath: NSString(string: self).expandingTildeInPath)
+    }
+  }
+}
+
+func pathHasMode(_ path: String, mode: mode_t) -> Bool {
+  var st = stat()
+  let statRes = stat(path, &st)
+  guard statRes != -1 else {
+    return false
+  }
+  return (Int32(st.st_mode) & Int32(mode)) == Int32(mode)
+}

Sources/tart/Commands/Stop.swift

@@ -14,6 +14,21 @@ struct Stop: AsyncParsableCommand {
 
   func run() async throws {
     let vmDir = try VMStorageLocal().open(name)
+    switch try vmDir.state() {
+    case "suspended":
+      try stopSuspended(vmDir)
+    case "running":
+      try await stopRunning(vmDir)
+    default:
+      return
+    }
+  }
+
+  func stopSuspended(_ vmDir: VMDirectory) throws {
+    try? FileManager.default.removeItem(at: vmDir.stateURL)
+  }
+
+  func stopRunning(_ vmDir: VMDirectory) async throws {
     let lock = try PIDLock(lockURL: vmDir.configURL)
 
     // Find the VM's PID

Sources/tart/Commands/Suspend.swift

@@ -0,0 +1,28 @@
+import ArgumentParser
+import Foundation
+import System
+import SwiftDate
+
+struct Suspend: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(commandName: "suspend", abstract: "Suspend a VM")
+
+  @Argument(help: "VM name")
+  var name: String
+
+  func run() async throws {
+    let vmDir = try VMStorageLocal().open(name)
+    let lock = try PIDLock(lockURL: vmDir.configURL)
+
+    // Find the VM's PID
+    var pid = try lock.pid()
+    if pid == 0 {
+      throw RuntimeError.VMNotRunning("VM \"\(name)\" is not running")
+    }
+
+    // Tell the "tart run" process to suspend the VM
+    let ret = kill(pid, SIGUSR1)
+    if ret != 0 {
+      throw RuntimeError.SuspendFailed("failed to send SIGUSR1 signal to the \"tart run\" process running VM \"\(name)\"")
+    }
+  }
+}

Sources/tart/Credentials/DockerConfigCredentialsProvider.swift

@@ -11,7 +11,7 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
     if let credentialsFromAuth = config.auths?[host]?.decodeCredentials() {
       return credentialsFromAuth
     }
-    if let helperProgram = config.credHelpers?[host] {
+    if let helperProgram = try config.findCredHelper(host: host) {
       return try executeHelper(binaryName: "docker-credential-\(helperProgram)", host: host)
     }
 
@@ -59,6 +59,26 @@ class DockerConfigCredentialsProvider: CredentialsProvider {
 struct DockerConfig: Codable {
   var auths: Dictionary<String, DockerAuthConfig>? = Dictionary()
   var credHelpers: Dictionary<String, String>? = Dictionary()
+
+  func findCredHelper(host: String) throws -> String? {
+    // Tart supports wildcards in credHelpers
+    // Similar to what is requested from Docker: https://github.com/docker/cli/issues/2928
+
+    guard let credHelpers else {
+      return nil
+    }
+
+    for (hostPattern, helperProgram) in credHelpers {
+      if (hostPattern == host) {
+        return helperProgram
+      }
+      let compiledPattern = try? Regex(hostPattern)
+      if (try compiledPattern?.wholeMatch(in: host) != nil) {
+        return helperProgram
+      }
+    }
+    return nil
+  }
 }
 
 struct DockerAuthConfig: Codable {

Sources/tart/Credentials/EnvironmentCredentialsProvider.swift

@@ -2,6 +2,11 @@ import Foundation
 
 class EnvironmentCredentialsProvider: CredentialsProvider {
   func retrieve(host: String) throws -> (String, String)? {
+    if let tartRegistryHostname = ProcessInfo.processInfo.environment["TART_REGISTRY_HOSTNAME"],
+       tartRegistryHostname != host {
+      return nil
+    }
+
     let username = ProcessInfo.processInfo.environment["TART_REGISTRY_USERNAME"]
     let password = ProcessInfo.processInfo.environment["TART_REGISTRY_PASSWORD"]
     if let username = username, let password = password {

Sources/tart/Credentials/KeychainCredentialsProvider.swift

@@ -61,6 +61,24 @@ class KeychainCredentialsProvider: CredentialsProvider {
       throw CredentialsProviderError.Failed(message: "Keychain failed to find item: \(status.explanation())")
     }
   }
+
+  func remove(host: String) throws {
+    let query: [String: Any] = [kSecClass as String: kSecClassInternetPassword,
+                                kSecAttrServer as String: host,
+                                kSecAttrLabel as String: "Tart Credentials",
+    ]
+
+    let status = SecItemDelete(query as CFDictionary)
+
+    switch status {
+    case errSecSuccess:
+      return
+    case errSecItemNotFound:
+      return
+    default:
+      throw CredentialsProviderError.Failed(message: "Failed to remove Keychain item(s): \(status.explanation())")
+    }
+  }
 }
 
 extension OSStatus {

Sources/tart/Credentials/StdinCredentials.swift

@@ -13,7 +13,7 @@ class StdinCredentials {
     return (user, password)
   }
 
-  private static func readStdinCredential(name: String, prompt: String, maxCharacters: Int = 255, isSensitive: Bool) throws -> String {
+  private static func readStdinCredential(name: String, prompt: String, maxCharacters: Int = 1024, isSensitive: Bool) throws -> String {
     var buf = [CChar](repeating: 0, count: maxCharacters + 1 /* sentinel */ + 1 /* NUL */)
     guard let rawCredential = readpassphrase(prompt, &buf, buf.count, isSensitive ? RPP_ECHO_OFF : RPP_ECHO_ON) else {
       throw StdinCredentialsError.CredentialRequired(which: name)

Sources/tart/Fetcher.swift

@@ -35,12 +35,12 @@ class Fetcher {
     //
     // This keeps a working reference to that file, yet we don't
     // have to deal with the cleanup any more.
-    let fh = try FileHandle(forReadingFrom: fileURL)
+    let mappedFile = try Data(contentsOf: fileURL, options: [.alwaysMapped])
     try FileManager.default.removeItem(at: fileURL)
 
     Task {
-      while let data = try fh.read(upToCount: 64 * 1024 * 1024) {
-        await dataCh.send(data)
+      for chunk in (0 ..< mappedFile.count).chunks(ofCount: 64 * 1024 * 1024) {
+        await dataCh.send(mappedFile.subdata(in: chunk))
       }
 
       dataCh.finish()

Sources/tart/Formatter/Format.swift

@@ -26,10 +26,16 @@ enum Format: String, ExpressibleByArgument, CaseIterable {
       }
       let table = TextTable<T> { (item: T) in
         let mirroredObject = Mirror(reflecting: item)
-        return mirroredObject.children.enumerated().map { (_, element) in
-          let fieldName = element.label!
-          return Column(title: fieldName, value: element.value)
-        }
+        return mirroredObject.children.enumerated()
+          .filter {(_, element) in
+            // Deprecate the "Running" field: only make it available
+            // from JSON for backwards-compatibility
+            element.label! != "Running"
+          }
+          .map { (_, element) in
+            let fieldName = element.label!
+            return Column(title: fieldName, value: element.value)
+          }
       }
       return table.string(for: data, style: Style.plain)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
     case .json:

Sources/tart/Network/Network.swift

@@ -1,7 +1,7 @@
 import Virtualization
 
 protocol Network {
-  func attachment() -> VZNetworkDeviceAttachment
+  func attachments() -> [VZNetworkDeviceAttachment]
   func run(_ sema: DispatchSemaphore) throws
   func stop() async throws
 }

Sources/tart/Network/NetworkBridged.swift

@@ -2,14 +2,14 @@ import Foundation
 import Virtualization
 
 class NetworkBridged: Network {
-  let interface: VZBridgedNetworkInterface
+  let interfaces: [VZBridgedNetworkInterface]
 
-  init(interface: VZBridgedNetworkInterface) {
-    self.interface = interface
+  init(interfaces: [VZBridgedNetworkInterface]) {
+    self.interfaces = interfaces
   }
 
-  func attachment() -> VZNetworkDeviceAttachment {
-    VZBridgedNetworkDeviceAttachment(interface: interface)
+  func attachments() -> [VZNetworkDeviceAttachment] {
+    interfaces.map { VZBridgedNetworkDeviceAttachment(interface: $0) }
   }
 
   func run(_ sema: DispatchSemaphore) throws {

Sources/tart/Network/NetworkShared.swift

@@ -2,8 +2,8 @@ import Foundation
 import Virtualization
 
 class NetworkShared: Network {
-  func attachment() -> VZNetworkDeviceAttachment {
-    VZNATNetworkDeviceAttachment()
+  func attachments() -> [VZNetworkDeviceAttachment] {
+    [VZNATNetworkDeviceAttachment()]
   }
 
   func run(_ sema: DispatchSemaphore) throws {

Sources/tart/Network/Softnet.swift

@@ -92,9 +92,9 @@ class Softnet: Network {
     }
   }
 
-  func attachment() -> VZNetworkDeviceAttachment {
+  func attachments() -> [VZNetworkDeviceAttachment] {
     let fh = FileHandle.init(fileDescriptor: vmFD)
-    return VZFileHandleNetworkDeviceAttachment(fileHandle: fh)
+    return [VZFileHandleNetworkDeviceAttachment(fileHandle: fh)]
   }
 
   static func configureSUIDBitIfNeeded() throws {

Sources/tart/OCI/AuthenticationKeeper.swift

@@ -0,0 +1,25 @@
+import Foundation
+
+actor AuthenticationKeeper {
+  var authentication: Authentication? = nil
+
+  func set(_ authentication: Authentication) {
+    self.authentication = authentication
+  }
+
+  func header() -> (String, String)? {
+    if let authentication = authentication {
+      // Do not suggest any headers if the
+      // authentication token has expired
+      if !authentication.isValid() {
+        return nil
+      }
+
+      return authentication.header()
+    }
+
+    // Do not suggest any headers if the
+    // authentication token is not set
+    return nil
+  }
+}

Sources/tart/OCI/Digest.swift

@@ -1,6 +1,11 @@
 import Foundation
 import CryptoKit
 
+enum DigestError: Error {
+  case InvalidOffset
+  case InvalidSize
+}
+
 class Digest {
   var hash: SHA256 = SHA256()
 
@@ -15,6 +20,37 @@ class Digest {
   static func hash(_ data: Data) -> String {
     SHA256.hash(data: data).hexdigest()
   }
+
+  static func hash(_ url: URL) throws -> String {
+    hash(try Data(contentsOf: url))
+  }
+
+  static func hash(_ url: URL, offset: UInt64, size: UInt64) throws -> String {
+    // Sanity check
+    let fhSanity = try FileHandle(forReadingFrom: url)
+    try fhSanity.seekToEnd()
+    let fileSize = try fhSanity.offset()
+    try fhSanity.close()
+
+    if offset > fileSize {
+      throw DigestError.InvalidOffset
+    }
+
+    if (offset + size) > fileSize {
+      throw DigestError.InvalidSize
+    }
+
+    // Read a chunk of size ``size`` at offset ``offset``
+    // and calculate it's digest
+    let fh = try FileHandle(forReadingFrom: url)
+    defer { try! fh.close() }
+
+    try fh.seek(toOffset: offset)
+
+    let data = try fh.read(upToCount: Int(size))!
+
+    return hash(data)
+  }
 }
 
 extension SHA256.Digest {

Sources/tart/OCI/Layerizer/Disk.swift

@@ -0,0 +1,6 @@
+import Foundation
+
+protocol Disk {
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer]
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws
+}

Sources/tart/OCI/Layerizer/DiskV1.swift

@@ -0,0 +1,75 @@
+import Foundation
+import Compression
+
+class DiskV1: Disk {
+  private static let bufferSizeBytes = 4 * 1024 * 1024
+  private static let layerLimitBytes = 500 * 1000 * 1000
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [OCIManifestLayer] = []
+
+    // Open the disk file
+    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+    var mappedDiskReadOffset = 0
+
+    // Compress the disk file as a single stream
+    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
+      // Determine the size of the next chunk
+      let bytesRead = min(length, mappedDisk.count - mappedDiskReadOffset)
+
+      // Read the next uncompressed chunk
+      let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
+
+      // Advance the offset
+      mappedDiskReadOffset += bytesRead
+
+      // Provide the uncompressed chunk to the compressing filter
+      return data
+    }
+
+    // Cut the compressed stream into layers, each equal exactly ``Self.layerLimitBytes`` bytes,
+    // except for the last one, which may be smaller
+    while let compressedData = try compressingFilter.readData(ofLength: Self.layerLimitBytes) {
+      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+
+      pushedLayers.append(OCIManifestLayer(
+        mediaType: diskV1MediaType,
+        size: compressedData.count,
+        digest: layerDigest
+      ))
+
+      // Update progress using an absolute value
+      progress.completedUnitCount = Int64(mappedDiskReadOffset)
+    }
+
+    return pushedLayers
+  }
+
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+      throw OCIError.FailedToCreateVmFile
+    }
+
+    // Open the disk file
+    let disk = try FileHandle(forWritingTo: diskURL)
+    defer { try! disk.close() }
+
+    // Decompress the layers onto the disk in a single stream
+    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
+      if let data = data {
+        disk.write(data)
+      }
+    }
+
+    for diskLayer in diskLayers {
+      try await registry.pullBlob(diskLayer.digest) { data in
+        try filter.write(data)
+
+        // Update the progress
+        progress.completedUnitCount += Int64(data.count)
+      }
+    }
+
+    try filter.finalize()
+  }
+}

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -0,0 +1,163 @@
+import Foundation
+import Compression
+
+class DiskV2: Disk {
+  private static let bufferSizeBytes = 4 * 1024 * 1024
+  private static let layerLimitBytes = 500 * 1000 * 1000
+
+  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, progress: Progress) async throws -> [OCIManifestLayer] {
+    var pushedLayers: [OCIManifestLayer] = []
+
+    // Open the disk file
+    var mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
+
+    // Compress the disk file as multiple individually decompressible streams,
+    // each equal ``Self.layerLimitBytes`` bytes or slightly larger due to the
+    // internal compressor's buffer
+    var offset: UInt64 = 0
+
+    while let (compressedData, uncompressedSize, uncompressedDigest) = try compressNextLayerOfLimitBytesOrMore(mappedDisk: mappedDisk, offset: offset) {
+      offset += uncompressedSize
+
+      let layerDigest = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb)
+
+      pushedLayers.append(OCIManifestLayer(
+        mediaType: diskV2MediaType,
+        size: compressedData.count,
+        digest: layerDigest,
+        uncompressedSize: uncompressedSize,
+        uncompressedContentDigest: uncompressedDigest
+      ))
+
+      // Update progress using a relative value
+      progress.completedUnitCount += Int64(uncompressedSize)
+    }
+
+    return pushedLayers
+  }
+
+  static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress) async throws {
+    // Support resumable pulls
+    let pullResumed = FileManager.default.fileExists(atPath: diskURL.path)
+
+    if !pullResumed && !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
+      throw OCIError.FailedToCreateVmFile
+    }
+
+    // Calculate the uncompressed disk size
+    var uncompressedDiskSize: UInt64 = 0
+
+    for layer in diskLayers {
+      guard let uncompressedLayerSize = layer.uncompressedSize() else {
+        throw OCIError.LayerIsMissingUncompressedSizeAnnotation
+      }
+
+      uncompressedDiskSize += uncompressedLayerSize
+    }
+
+    // Truncate the target disk file so that it will be able
+    // to accomodate the uncompressed disk size
+    let disk = try FileHandle(forWritingTo: diskURL)
+    try disk.truncate(atOffset: uncompressedDiskSize)
+    try disk.close()
+
+    // Concurrently fetch and decompress layers
+    try await withThrowingTaskGroup(of: Void.self) { group in
+      var globalDiskWritingOffset: UInt64 = 0
+
+      for (index, diskLayer) in diskLayers.enumerated() {
+        // Respect the concurrency limit
+        if index >= concurrency {
+          try await group.next()
+        }
+
+        // Retrieve layer annotations
+        guard let uncompressedLayerSize = diskLayer.uncompressedSize() else {
+          throw OCIError.LayerIsMissingUncompressedSizeAnnotation
+        }
+        guard let uncompressedLayerContentDigest = diskLayer.uncompressedContentDigest() else {
+          throw OCIError.LayerIsMissingUncompressedDigestAnnotation
+        }
+
+        // Capture the current disk writing offset
+        let diskWritingOffset = globalDiskWritingOffset
+
+        // Launch a fetching and decompression task
+        group.addTask {
+          // No need to fetch and decompress anything if we've already done so
+          if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest {
+            // Update the progress
+            progress.completedUnitCount += Int64(diskLayer.size)
+
+            return
+          }
+
+          // Open the disk file at the specific offset
+          let disk = try FileHandle(forWritingTo: diskURL)
+          try disk.seek(toOffset: diskWritingOffset)
+
+          // Pull and decompress a single layer into the specific offset on disk
+          let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
+            if let data = data {
+              disk.write(data)
+            }
+          }
+
+          try await registry.pullBlob(diskLayer.digest) { data in
+            try filter.write(data)
+
+            // Update the progress
+            progress.completedUnitCount += Int64(data.count)
+          }
+
+          try filter.finalize()
+
+          try disk.close()
+        }
+
+        globalDiskWritingOffset += uncompressedLayerSize
+      }
+    }
+  }
+
+  private static func compressNextLayerOfLimitBytesOrMore(mappedDisk: Data, offset: UInt64) throws -> (Data, UInt64, String)? {
+    var compressedData = Data()
+    var bytesRead: UInt64 = 0
+    let digest = Digest()
+
+    // Create a compressing filter that we will terminate upon
+    // reaching ``Self.layerLimitBytes`` of compressed data
+    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: bufferSizeBytes) { (length: Int) -> Data? in
+      if compressedData.count >= Self.layerLimitBytes {
+        return nil
+      }
+
+      let readFromByte = Int(offset + bytesRead)
+
+      let numBytesToRead = min(mappedDisk.count - readFromByte, bufferSizeBytes)
+      if numBytesToRead == 0 {
+        return nil
+      }
+
+      let uncompressedChunk = mappedDisk.subdata(in: readFromByte ..< (readFromByte + numBytesToRead))
+
+      bytesRead += UInt64(uncompressedChunk.count)
+      digest.update(uncompressedChunk)
+
+      return uncompressedChunk
+    }
+
+    // Retrieve compressed data chunks, but normally no more than ``Self.layerLimitBytes`` bytes
+    while let compressedChunk = try compressingFilter.readData(ofLength: Self.bufferSizeBytes) {
+      compressedData.append(compressedChunk)
+    }
+
+    // Nothing was read this time from the disk,
+    // signal that to the consumer
+    if bytesRead == 0 {
+      return nil
+    }
+
+    return (compressedData, bytesRead, digest.finalize())
+  }
+}

Sources/tart/OCI/Manifest.swift

@@ -1,10 +1,22 @@
 import Foundation
 
+// OCI manifest and OCI config media types
 let ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json"
 let ociConfigMediaType = "application/vnd.oci.image.config.v1+json"
 
-// Annotations
+// Layer media types
+let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
+let diskV1MediaType = "application/vnd.cirruslabs.tart.disk.v1"
+let diskV2MediaType = "application/vnd.cirruslabs.tart.disk.v2"
+let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
+
+// Manifest annotations
 let uncompressedDiskSizeAnnotation = "org.cirruslabs.tart.uncompressed-disk-size"
+let uploadTimeAnnotation = "org.cirruslabs.tart.upload-time"
+
+// Layer annotations
+let uncompressedSizeAnnotation = "org.cirruslabs.tart.uncompressed-size"
+let uncompressedContentDigestAnnotation = "org.cirruslabs.tart.uncompressed-content-digest"
 
 struct OCIManifest: Codable, Equatable {
   var schemaVersion: Int = 2
@@ -13,15 +25,21 @@ struct OCIManifest: Codable, Equatable {
   var layers: [OCIManifestLayer] = Array()
   var annotations: Dictionary<String, String>?
 
-  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil) {
+  init(config: OCIManifestConfig, layers: [OCIManifestLayer], uncompressedDiskSize: UInt64? = nil, uploadDate: Date? = nil) {
     self.config = config
     self.layers = layers
 
+    var annotations: [String: String] = [:]
+
     if let uncompressedDiskSize = uncompressedDiskSize {
-      annotations = [
-        uncompressedDiskSizeAnnotation: String(uncompressedDiskSize)
-      ]
+      annotations[uncompressedDiskSizeAnnotation] = String(uncompressedDiskSize)
     }
+
+    if let uploadDate = uploadDate {
+      annotations[uploadTimeAnnotation] = uploadDate.toISO()
+    }
+
+    self.annotations = annotations
   }
 
   init(fromJSON: Data) throws {
@@ -64,6 +82,37 @@ struct OCIManifestLayer: Codable, Equatable {
   var mediaType: String
   var size: Int
   var digest: String
+  var annotations: Dictionary<String, String>?
+
+  init(mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil, uncompressedContentDigest: String? = nil) {
+    self.mediaType = mediaType
+    self.size = size
+    self.digest = digest
+
+    var annotations: [String: String] = [:]
+
+    if let uncompressedSize = uncompressedSize {
+      annotations[uncompressedSizeAnnotation] = String(uncompressedSize)
+    }
+
+    if let uncompressedContentDigest = uncompressedContentDigest {
+      annotations[uncompressedContentDigestAnnotation] = uncompressedContentDigest
+    }
+
+    self.annotations = annotations
+  }
+
+  func uncompressedSize() -> UInt64? {
+    guard let value = annotations?[uncompressedSizeAnnotation] else {
+      return nil
+    }
+
+    return UInt64(value)
+  }
+
+  func uncompressedContentDigest() -> String? {
+    annotations?[uncompressedContentDigestAnnotation]
+  }
 }
 
 struct Descriptor: Equatable {

Sources/tart/OCI/Registry.swift

@@ -99,11 +99,20 @@ struct TokenResponse: Decodable, Authentication {
 }
 
 class Registry {
-  let baseURL: URL
+  private let baseURL: URL
   let namespace: String
   let credentialsProviders: [CredentialsProvider]
+  let authenticationKeeper = AuthenticationKeeper()
 
-  var currentAuthToken: Authentication? = nil
+  var host: String? {
+    guard let host = baseURL.host else { return nil }
+
+    if let port = baseURL.port {
+      return "\(host):\(port)"
+    }
+
+    return host
+  }
 
   init(urlComponents: URLComponents,
        namespace: String,
@@ -232,7 +241,7 @@ class Registry {
     return digest
   }
 
-  public func pullBlob(_ digest: String, handler: (Data) throws -> Void) async throws {
+  public func pullBlob(_ digest: String, handler: (Data) async throws -> Void) async throws {
     let (channel, response) = try await channelRequest(.GET, endpointURL("\(namespace)/blobs/\(digest)"), viaFile: true)
     if response.statusCode != HTTPCode.Ok.rawValue {
       let body = try await channel.asData().asText()
@@ -243,7 +252,7 @@ class Registry {
     for try await part in channel {
       try Task.checkCancellation()
 
-      try handler(Data(part))
+      try await handler(part)
     }
   }
 
@@ -295,11 +304,6 @@ class Registry {
       request.httpBody = body
     }
 
-    // Invalidate token if it has expired
-    if currentAuthToken?.isValid() == false {
-      currentAuthToken = nil
-    }
-
     var (channel, response) = try await authAwareRequest(request: request, viaFile: viaFile)
 
     if doAuth && response.statusCode == HTTPCode.Unauthorized.rawValue {
@@ -321,7 +325,7 @@ class Registry {
 
     if wwwAuthenticate.scheme.lowercased() == "basic" {
       if let (user, password) = try lookupCredentials() {
-        currentAuthToken = BasicAuthentication(user: user, password: password)
+        await authenticationKeeper.set(BasicAuthentication(user: user, password: password))
       }
 
       return
@@ -368,7 +372,7 @@ class Registry {
         + "while retrieving an authentication token", details: data.asText())
     }
 
-    currentAuthToken = try TokenResponse.parse(fromData: data)
+    await authenticationKeeper.set(try TokenResponse.parse(fromData: data))
   }
 
   private func lookupCredentials() throws -> (String, String)? {
@@ -389,8 +393,7 @@ class Registry {
   private func authAwareRequest(request: URLRequest, viaFile: Bool = false) async throws -> (AsyncThrowingChannel<Data, Error>, HTTPURLResponse) {
     var request = request
 
-    if let token = currentAuthToken {
-      let (name, value) = token.header()
+    if let (name, value) = await authenticationKeeper.header() {
       request.addValue(value, forHTTPHeaderField: name)
     }
 

Sources/tart/PIDLock.swift

@@ -8,6 +8,11 @@ class PIDLock {
   init(lockURL: URL) throws {
     url = lockURL
     fd = open(lockURL.path, O_RDWR)
+    if fd == -1 {
+      let details = Errno(rawValue: CInt(errno))
+
+      throw RuntimeError.PIDLockFailed("failed to open lock file \(url): \(details)")
+    }
   }
 
   deinit {

Sources/tart/Platform/Darwin.swift

@@ -6,7 +6,7 @@ struct UnsupportedHostOSError: Error, CustomStringConvertible {
   }
 }
 
-struct Darwin: Platform {
+struct Darwin: PlatformSuspendable {
   var ecid: VZMacMachineIdentifier
   var hardwareModel: VZMacHardwareModel
 
@@ -60,7 +60,7 @@ struct Darwin: Platform {
     let result = VZMacPlatformConfiguration()
 
     result.machineIdentifier = ecid
-    result.auxiliaryStorage = VZMacAuxiliaryStorage(contentsOf: nvramURL)
+    result.auxiliaryStorage = VZMacAuxiliaryStorage(url: nvramURL)
 
     if !hardwareModel.isSupported {
       // At the moment support of M1 chip is not yet dropped in any macOS version
@@ -100,17 +100,33 @@ struct Darwin: Platform {
 
   func keyboards() -> [VZKeyboardConfiguration] {
     if #available(macOS 14, *) {
-      return [VZMacKeyboardConfiguration()]
+      // Mac keyboard is only supported by guests starting with macOS Ventura
+      return [VZMacKeyboardConfiguration(), VZUSBKeyboardConfiguration()]
     } else {
       return [VZUSBKeyboardConfiguration()]
     }
   }
 
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration] {
+    if #available(macOS 14, *) {
+      return [VZMacKeyboardConfiguration()]
+    } else {
+      // fallback to the regular configuration
+      return keyboards()
+    }
+  }
+
   func pointingDevices() -> [VZPointingDeviceConfiguration] {
-    if #available(macOS 13, *) {
+    // Trackpad is only supported by guests starting with macOS Ventura
+    [VZMacTrackpadConfiguration(), VZUSBScreenCoordinatePointingDeviceConfiguration()]
+  }
+
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration] {
+    if #available(macOS 14, *) {
       return [VZMacTrackpadConfiguration()]
     } else {
-      return [VZUSBScreenCoordinatePointingDeviceConfiguration()]
+      // fallback to the regular configuration
+      return pointingDevices()
     }
   }
 }

Sources/tart/Platform/Platform.swift

@@ -8,3 +8,8 @@ protocol Platform: Codable {
   func keyboards() -> [VZKeyboardConfiguration]
   func pointingDevices() -> [VZPointingDeviceConfiguration]
 }
+
+protocol PlatformSuspendable: Platform {
+  func pointingDevicesSuspendable() -> [VZPointingDeviceConfiguration]
+  func keyboardsSuspendable() -> [VZKeyboardConfiguration]
+}

Sources/tart/Root.swift

@@ -16,6 +16,7 @@ struct Root: AsyncParsableCommand {
       Get.self,
       List.self,
       Login.self,
+      Logout.self,
       IP.self,
       Pull.self,
       Push.self,
@@ -57,6 +58,11 @@ struct Root: AsyncParsableCommand {
       }
     }
 
+    // Add commands that are only available on specific macOS versions
+    if #available(macOS 14, *) {
+      configuration.subcommands.append(Suspend.self)
+    }
+
     // Ensure the default SIGINT handled is disabled,
     // otherwise there's a race between two handlers
     signal(SIGINT, SIG_IGN);
@@ -76,10 +82,12 @@ struct Root: AsyncParsableCommand {
       var command = try parseAsRoot()
 
       // Run garbage-collection before each command (shouldn't take too long)
-      do {
-        try Config().gc()
-      } catch {
-        fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+      if type(of: command) != type(of: Pull()) && type(of: command) != type(of: Clone()){
+        do {
+          try Config().gc()
+        } catch {
+          fputs("Failed to perform garbage collection!\n\(error)\n", stderr)
+        }
       }
 
       if var asyncCommand = command as? AsyncParsableCommand {

Sources/tart/VM.swift

@@ -26,6 +26,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
   // Virtualization.Framework's virtual machine
   @Published var virtualMachine: VZVirtualMachine
 
+  // Virtualization.Framework's virtual machine configuration
+  var configuration: VZVirtualMachineConfiguration
+
   // Semaphore used to communicate with the VZVirtualMachineDelegate
   var sema = DispatchSemaphore(value: 0)
 
@@ -39,9 +42,10 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
   init(vmDir: VMDirectory,
        network: Network = NetworkShared(),
-       additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
+       additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
        directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
-       serialPorts: [VZSerialPortConfiguration] = []
+       serialPorts: [VZSerialPortConfiguration] = [],
+       suspendable: Bool = false
   ) throws {
     name = vmDir.name
     config = try VMConfig.init(fromURL: vmDir.configURL)
@@ -52,11 +56,12 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
     // Initialize the virtual machine and its configuration
     self.network = network
-    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
-                                                    nvramURL: vmDir.nvramURL, vmConfig: config,
-                                                    network: network, additionalDiskAttachments: additionalDiskAttachments,
-                                                    directorySharingDevices: directorySharingDevices,
-                                                    serialPorts: serialPorts
+    configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
+                                                nvramURL: vmDir.nvramURL, vmConfig: config,
+                                                network: network, additionalStorageDevices: additionalStorageDevices,
+                                                directorySharingDevices: directorySharingDevices,
+                                                serialPorts: serialPorts,
+                                                suspendable: suspendable
     )
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
@@ -66,9 +71,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
   static func retrieveIPSW(remoteURL: URL) async throws -> URL {
     // Check if we already have this IPSW in cache
-    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+    var headRequest = URLRequest(url: remoteURL)
+    headRequest.httpMethod = "HEAD"
+    let (_, headResponse) = try await Fetcher.fetch(headRequest, viaFile: false)
 
-    if let hash = response.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
+    if let hash = headResponse.value(forHTTPHeaderField: "x-amz-meta-digest-sha256") {
       let ipswLocation = try IPSWCache().locationFor(fileName: "sha256:\(hash).ipsw")
 
       if FileManager.default.fileExists(atPath: ipswLocation.path) {
@@ -82,6 +89,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Download the IPSW
     defaultLogger.appendNewLine("Fetching \(remoteURL.lastPathComponent)...")
 
+    let (channel, response) = try await Fetcher.fetch(URLRequest(url: remoteURL), viaFile: true)
+
     let progress = Progress(totalUnitCount: response.expectedContentLength)
     ProgressObserver(progress).log(defaultLogger)
 
@@ -133,7 +142,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     ipswURL: URL,
     diskSizeGB: UInt16,
     network: Network = NetworkShared(),
-    additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [],
+    additionalStorageDevices: [VZStorageDeviceConfiguration] = [],
     directorySharingDevices: [VZDirectorySharingDeviceConfiguration] = [],
     serialPorts: [VZSerialPortConfiguration] = []
   ) async throws {
@@ -179,11 +188,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
     // Initialize the virtual machine and its configuration
     self.network = network
-    let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
-                                                    vmConfig: config, network: network,
-                                                    additionalDiskAttachments: additionalDiskAttachments,
-                                                    directorySharingDevices: directorySharingDevices,
-                                                    serialPorts: serialPorts
+    configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
+                                                vmConfig: config, network: network,
+                                                additionalStorageDevices: additionalStorageDevices,
+                                                directorySharingDevices: directorySharingDevices,
+                                                serialPorts: serialPorts
     )
     virtualMachine = VZVirtualMachine(configuration: configuration)
 
@@ -220,11 +229,17 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     return try VM(vmDir: vmDir)
   }
 
-  func run(_ recovery: Bool) async throws {
+  func start(recovery: Bool, resume shouldResume: Bool) async throws {
     try network.run(sema)
 
-    try await start(recovery)
+    if shouldResume {
+      try await resume()
+    } else {
+      try await start(recovery)
+    }
+  }
 
+  func run() async throws {
     await withTaskCancellationHandler(operation: {
       // Wait for the VM to finish running
       // or for the exit condition
@@ -242,15 +257,14 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
 
   @MainActor
   private func start(_ recovery: Bool) async throws {
-    if #available(macOS 13, *) {
-      // new API introduced in Ventura
-      let startOptions = VZMacOSVirtualMachineStartOptions()
-      startOptions.startUpFromMacOSRecovery = recovery
-      try await virtualMachine.start(options: startOptions)
-    } else {
-      // use method that also available on Monterey
-      try await virtualMachine.start(recovery)
-    }
+    let startOptions = VZMacOSVirtualMachineStartOptions()
+    startOptions.startUpFromMacOSRecovery = recovery
+    try await virtualMachine.start(options: startOptions)
+  }
+
+  @MainActor
+  private func resume() async throws {
+    try await virtualMachine.resume()
   }
 
   @MainActor
@@ -263,9 +277,10 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     nvramURL: URL,
     vmConfig: VMConfig,
     network: Network = NetworkShared(),
-    additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment],
+    additionalStorageDevices: [VZStorageDeviceConfiguration],
     directorySharingDevices: [VZDirectorySharingDeviceConfiguration],
-    serialPorts: [VZSerialPortConfiguration]
+    serialPorts: [VZSerialPortConfiguration],
+    suspendable: Bool = false
   ) throws -> VZVirtualMachineConfiguration {
     let configuration = VZVirtualMachineConfiguration()
 
@@ -283,31 +298,44 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]
 
     // Audio
-    let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
-    let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
-    inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
-    let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
-    outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
-    soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
-    configuration.audioDevices = [soundDeviceConfiguration]
+    if !suspendable {
+      let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+      let inputAudioStreamConfiguration = VZVirtioSoundDeviceInputStreamConfiguration()
+      inputAudioStreamConfiguration.source = VZHostAudioInputStreamSource()
+      let outputAudioStreamConfiguration = VZVirtioSoundDeviceOutputStreamConfiguration()
+      outputAudioStreamConfiguration.sink = VZHostAudioOutputStreamSink()
+      soundDeviceConfiguration.streams = [inputAudioStreamConfiguration, outputAudioStreamConfiguration]
+      configuration.audioDevices = [soundDeviceConfiguration]
+    }
 
     // Keyboard and mouse
-    configuration.keyboards = vmConfig.platform.keyboards()
-    configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    if suspendable, let platformSuspendable = vmConfig.platform.self as? PlatformSuspendable {
+      configuration.keyboards = platformSuspendable.keyboardsSuspendable()
+      configuration.pointingDevices = platformSuspendable.pointingDevicesSuspendable()
+    } else {
+      configuration.keyboards = vmConfig.platform.keyboards()
+      configuration.pointingDevices = vmConfig.platform.pointingDevices()
+    }
 
     // Networking
-    let vio = VZVirtioNetworkDeviceConfiguration()
-    vio.attachment = network.attachment()
-    vio.macAddress = vmConfig.macAddress
-    configuration.networkDevices = [vio]
+    configuration.networkDevices = network.attachments().map {
+      let vio = VZVirtioNetworkDeviceConfiguration()
+      vio.attachment = $0
+      vio.macAddress = vmConfig.macAddress
+      return vio
+    }
 
     // Storage
-    var attachments = [try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false)]
-    attachments.append(contentsOf: additionalDiskAttachments)
-    configuration.storageDevices = attachments.map { VZVirtioBlockDeviceConfiguration(attachment: $0) }
+    var devices: [VZStorageDeviceConfiguration] = [
+      VZVirtioBlockDeviceConfiguration(attachment: try VZDiskImageStorageDeviceAttachment(url: diskURL, readOnly: false))
+    ]
+    devices.append(contentsOf: additionalStorageDevices)
+    configuration.storageDevices = devices
 
     // Entropy
-    configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()]
+    if !suspendable {
+      configuration.entropyDevices = [VZVirtioEntropyDeviceConfiguration()]
+    }
 
     // Directory sharing devices
     configuration.directorySharingDevices = directorySharingDevices
@@ -315,6 +343,20 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
     // Serial Port
     configuration.serialPorts = serialPorts
 
+    // Version console device
+    //
+    // A dummy console device useful for implementing
+    // host feature checks in the guest agent software.
+    if !suspendable {
+      let consolePort = VZVirtioConsolePortConfiguration()
+      consolePort.name = "tart-version-\(CI.version)"
+
+      let consoleDevice = VZVirtioConsoleDeviceConfiguration()
+      consoleDevice.ports[0] = consolePort
+
+      configuration.consoleDevices.append(consoleDevice)
+    }
+
     try configuration.validate()
 
     return configuration

Sources/tart/VMConfig.swift

@@ -97,11 +97,7 @@ struct VMConfig: Codable {
     case .darwin:
       platform = try Darwin(from: decoder)
     case .linux:
-      if #available(macOS 13, *) {
-        platform = try Linux(from: decoder)
-      } else {
-        throw UnsupportedOSError("Linux VMs", "are")
-      }
+      platform = try Linux(from: decoder)
     }
     cpuCountMin = try container.decode(Int.self, forKey: .cpuCountMin)
     cpuCount = try container.decode(Int.self, forKey: .cpuCount)

Sources/tart/VMDirectory+OCI.swift

@@ -1,33 +1,30 @@
 import Foundation
-import Compression
 import Sentry
 
 enum OCIError: Error {
   case ShouldBeExactlyOneLayer
   case ShouldBeAtLeastOneLayer
   case FailedToCreateVmFile
+  case LayerIsMissingUncompressedSizeAnnotation
+  case LayerIsMissingUncompressedDigestAnnotation
 }
 
 extension VMDirectory {
   private static let bufferSizeBytes = 64 * 1024 * 1024
   private static let layerLimitBytes = 500 * 1000 * 1000
 
-  private static let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
-  private static let diskMediaType = "application/vnd.cirruslabs.tart.disk.v1"
-  private static let nvramMediaType = "application/vnd.cirruslabs.tart.nvram.v1"
-
-  func pullFromRegistry(registry: Registry, reference: String) async throws {
+  func pullFromRegistry(registry: Registry, reference: String, concurrency: UInt) async throws {
     defaultLogger.appendNewLine("pulling manifest...")
 
     let (manifest, _) = try await registry.pullManifest(reference: reference)
 
-    return try await pullFromRegistry(registry: registry, manifest: manifest)
+    return try await pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
   }
 
-  func pullFromRegistry(registry: Registry, manifest: OCIManifest) async throws {
+  func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt) async throws {
     // Pull VM's config file layer and re-serialize it into a config file
     let configLayers = manifest.layers.filter {
-      $0.mediaType == Self.configMediaType
+      $0.mediaType == configMediaType
     }
     if configLayers.count != 1 {
       throw OCIError.ShouldBeExactlyOneLayer
@@ -41,50 +38,36 @@ extension VMDirectory {
     }
     try configFile.close()
 
-    // Pull VM's disk layers and decompress them sequentially into a disk file
-    let diskLayers = manifest.layers.filter {
-      $0.mediaType == Self.diskMediaType
-    }
-    if diskLayers.isEmpty {
+    // Pull VM's disk layers and decompress them into a disk file
+    let diskImplType: Disk.Type
+    let layers: [OCIManifestLayer]
+
+    if manifest.layers.contains(where: { $0.mediaType == diskV1MediaType }) {
+      diskImplType = DiskV1.self
+      layers = manifest.layers.filter { $0.mediaType == diskV1MediaType }
+    } else if manifest.layers.contains(where: { $0.mediaType == diskV2MediaType }) {
+      diskImplType = DiskV2.self
+      layers = manifest.layers.filter { $0.mediaType == diskV2MediaType }
+    } else {
       throw OCIError.ShouldBeAtLeastOneLayer
     }
-    if !FileManager.default.createFile(atPath: diskURL.path, contents: nil) {
-      throw OCIError.FailedToCreateVmFile
-    }
-    let disk = try FileHandle(forWritingTo: diskURL)
-    let filter = try OutputFilter(.decompress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { data in
-      if let data = data {
-        disk.write(data)
-      }
-    }
 
-    // Progress
-    let diskCompressedSize: Int64 = Int64(diskLayers.map {
-      $0.size
-    }
-    .reduce(0) {
-      $0 + $1
-    })
+    let diskCompressedSize = layers.map { Int64($0.size) }.reduce(0, +)
+    SentrySDK.span?.setMeasurement(name: "compressed_disk_size", value: diskCompressedSize as NSNumber, unit: MeasurementUnitInformation.byte)
+
     let prettyDiskSize = String(format: "%.1f", Double(diskCompressedSize) / 1_000_000_000.0)
     defaultLogger.appendNewLine("pulling disk (\(prettyDiskSize) GB compressed)...")
+
     let progress = Progress(totalUnitCount: diskCompressedSize)
     ProgressObserver(progress).log(defaultLogger)
 
-    for diskLayer in diskLayers {
-      try await registry.pullBlob(diskLayer.digest) { data in
-        try filter.write(data)
-        progress.completedUnitCount += Int64(data.count)
-      }
-    }
-    try filter.finalize()
-    try disk.close()
-    SentrySDK.span?.setMeasurement(name: "compressed_disk_size", value: diskCompressedSize as NSNumber, unit: MeasurementUnitInformation.byte);
+    try await diskImplType.pull(registry: registry, diskLayers: layers, diskURL: diskURL, concurrency: concurrency, progress: progress)
 
     // Pull VM's NVRAM file layer and store it in an NVRAM file
     defaultLogger.appendNewLine("pulling NVRAM...")
 
     let nvramLayers = manifest.layers.filter {
-      $0.mediaType == Self.nvramMediaType
+      $0.mediaType == nvramMediaType
     }
     if nvramLayers.count != 1 {
       throw OCIError.ShouldBeExactlyOneLayer
@@ -99,7 +82,7 @@ extension VMDirectory {
     try nvram.close()
   }
 
-  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int) async throws -> RemoteName {
+  func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, diskFormat: String) async throws -> RemoteName {
     var layers = Array<OCIManifestLayer>()
 
     // Read VM's config and push it as blob
@@ -107,32 +90,22 @@ extension VMDirectory {
     let configJSON = try JSONEncoder().encode(config)
     defaultLogger.appendNewLine("pushing config...")
     let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb)
-    layers.append(OCIManifestLayer(mediaType: Self.configMediaType, size: configJSON.count, digest: configDigest))
+    layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest))
 
-    // Progress
+    // Compress the disk file as multiple chunks and push them as disk layers
     let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64
 
     defaultLogger.appendNewLine("pushing disk... this will take a while...")
     let progress = Progress(totalUnitCount: diskSize)
     ProgressObserver(progress).log(defaultLogger)
 
-    // Read VM's compressed disk as chunks
-    // and sequentially upload them as blobs
-    let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
-    let mappedDiskSize = mappedDisk.count
-    var mappedDiskReadOffset = 0
-    let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
-      let bytesRead = min(length, mappedDiskSize - mappedDiskReadOffset)
-      let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
-      mappedDiskReadOffset += bytesRead
-
-      progress.completedUnitCount = Int64(mappedDiskReadOffset)
-
-      return data
-    }
-    while let compressedLayerData = try compressingFilter.readData(ofLength: Self.layerLimitBytes) {
-      let layerDigest = try await registry.pushBlob(fromData: compressedLayerData, chunkSizeMb: chunkSizeMb)
-      layers.append(OCIManifestLayer(mediaType: Self.diskMediaType, size: compressedLayerData.count, digest: layerDigest))
+    switch diskFormat {
+    case "v1":
+      layers.append(contentsOf: try await DiskV1.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+    case "v2":
+      layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, progress: progress))
+    default:
+      throw RuntimeError.OCIUnsupportedDiskFormat(diskFormat)
     }
 
     // Read VM's NVRAM and push it as blob
@@ -140,7 +113,7 @@ extension VMDirectory {
 
     let nvram = try FileHandle(forReadingFrom: nvramURL).readToEnd()!
     let nvramDigest = try await registry.pushBlob(fromData: nvram, chunkSizeMb: chunkSizeMb)
-    layers.append(OCIManifestLayer(mediaType: Self.nvramMediaType, size: nvram.count, digest: nvramDigest))
+    layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest))
 
     // Craft a stub OCI config for Docker Hub compatibility
     let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os).toJSON()
@@ -148,7 +121,8 @@ extension VMDirectory {
     let manifest = OCIManifest(
       config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
       layers: layers,
-      uncompressedDiskSize: UInt64(mappedDiskReadOffset)
+      uncompressedDiskSize: UInt64(diskSize),
+      uploadDate: Date()
     )
 
     // Manifest
@@ -159,7 +133,7 @@ extension VMDirectory {
     }
 
     let pushedReference = Reference(digest: try manifest.digest())
-    return RemoteName(host: registry.baseURL.host!, namespace: registry.namespace, reference: pushedReference)
+    return RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference)
   }
 }
 

Sources/tart/VMDirectory.swift

@@ -1,5 +1,6 @@
 import Foundation
 import Virtualization
+import CryptoKit
 
 struct VMDirectory: Prunable {
   var baseURL: URL
@@ -13,6 +14,9 @@ struct VMDirectory: Prunable {
   var nvramURL: URL {
     baseURL.appendingPathComponent("nvram.bin")
   }
+  var stateURL: URL {
+    baseURL.appendingPathComponent("state.vzvmsave")
+  }
 
   var explicitlyPulledMark: URL {
     baseURL.appendingPathComponent(".explicitly-pulled")
@@ -27,7 +31,26 @@ struct VMDirectory: Prunable {
   }
 
   func running() throws -> Bool {
-    try PIDLock(lockURL: configURL).pid() != 0
+    // The most common reason why PIDLock() instantiation fails is a race with "tart delete" (ENOENT),
+    // which is fine to report as "not running".
+    //
+    // The other reasons are unlikely and the cost of getting a false positive is way less than
+    // the cost of crashing with an exception when calling "tart list" on a busy machine, for example.
+    guard let lock = try? PIDLock(lockURL: configURL) else {
+      return false
+    }
+
+    return try lock.pid() != 0
+  }
+
+  func state() throws -> String {
+    if try running() {
+      return "running"
+    } else if FileManager.default.fileExists(atPath: stateURL.path) {
+      return "suspended"
+    } else {
+      return "stopped"
+    }
   }
 
   static func temporary() throws -> VMDirectory {
@@ -37,6 +60,17 @@ struct VMDirectory: Prunable {
     return VMDirectory(baseURL: tmpDir)
   }
 
+  //Create tmp directory with hashing
+  static func temporaryDeterministic(key: String) throws -> VMDirectory {
+    let keyData = Data(key.utf8)
+    let hash = Insecure.MD5.hash(data: keyData)
+    // Convert hash to string
+    let hashString = hash.compactMap { String(format: "%02x", $0) }.joined()
+    let tmpDir = try Config().tartTmpDir.appendingPathComponent(hashString)
+    try FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true)
+    return VMDirectory(baseURL: tmpDir)
+  }
+
   var initialized: Bool {
     FileManager.default.fileExists(atPath: configURL.path) &&
       FileManager.default.fileExists(atPath: diskURL.path) &&
@@ -70,6 +104,7 @@ struct VMDirectory: Prunable {
     try FileManager.default.copyItem(at: configURL, to: to.configURL)
     try FileManager.default.copyItem(at: nvramURL, to: to.nvramURL)
     try FileManager.default.copyItem(at: diskURL, to: to.diskURL)
+    try? FileManager.default.copyItem(at: stateURL, to: to.stateURL)
 
     // Re-generate MAC address
     if generateMAC {
@@ -85,6 +120,8 @@ struct VMDirectory: Prunable {
     var vmConfig = try VMConfig(fromURL: configURL)
 
     vmConfig.macAddress = VZMACAddress.randomLocallyAdministered()
+    // cleanup state if any
+    try? FileManager.default.removeItem(at: stateURL)
 
     try vmConfig.save(toURL: configURL)
   }

Sources/tart/VMStorageHelper.swift

@@ -34,9 +34,15 @@ class VMStorageHelper {
   }
 }
 
+extension NSError {
+  func isFileNotFound() -> Bool {
+    return self.code == NSFileNoSuchFileError || self.code == NSFileReadNoSuchFileError
+  }
+}
+
 extension Error {
   func isFileNotFound() -> Bool {
-    (self as NSError).code == NSFileReadNoSuchFileError
+    (self as NSError).isFileNotFound() || (self as NSError).underlyingErrors.contains(where: { $0.isFileNotFound() })
   }
 }
 
@@ -58,6 +64,8 @@ enum RuntimeError : Error {
   case ImportFailed(_ message: String)
   case SoftnetFailed(_ message: String)
   case OCIStorageError(_ message: String)
+  case OCIUnsupportedDiskFormat(_ format: String)
+  case SuspendFailed(_ message: String)
 }
 
 protocol HasExitCode {
@@ -101,6 +109,10 @@ extension RuntimeError : CustomStringConvertible {
       return "Softnet failed: \(message)"
     case .OCIStorageError(let message):
       return "OCI storage error: \(message)"
+    case .OCIUnsupportedDiskFormat(let format):
+      return "OCI disk format \(format) is not supported by this version of Tart"
+    case .SuspendFailed(let message):
+      return "Failed to suspend the VM: \(message)"
     }
   }
 }
@@ -108,6 +120,8 @@ extension RuntimeError : CustomStringConvertible {
 extension RuntimeError : HasExitCode {
   var exitCode: Int32 {
     switch self {
+    case .VMDoesNotExist:
+      return 2
     case .VMNotRunning:
       return 2
     case .VMAlreadyRunning:

Sources/tart/VMStorageLocal.swift

@@ -1,6 +1,6 @@
 import Foundation
 
-class VMStorageLocal {
+class VMStorageLocal: PrunableStorage {
   let baseURL: URL = try! Config().tartHomeDir.appendingPathComponent("vms", isDirectory: true)
 
   private func vmURL(_ name: String) -> URL {
@@ -16,6 +16,8 @@ class VMStorageLocal {
 
     try vmDir.validate(userFriendlyName: name)
 
+    try vmDir.baseURL.updateAccessDate()
+
     return vmDir
   }
 
@@ -63,6 +65,10 @@ class VMStorageLocal {
     }
   }
 
+  func prunables() throws -> [Prunable] {
+    try list().map { (_, vmDir) in vmDir }
+  }
+
   func hasVMsWithMACAddress(macAddress: String) throws -> Bool {
     try list().contains { try $1.macAddress() == macAddress }
   }

Sources/tart/VMStorageOCI.swift

@@ -132,7 +132,7 @@ class VMStorageOCI: PrunableStorage {
     try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
   }
 
-  func pull(_ name: RemoteName, registry: Registry) async throws {
+  func pull(_ name: RemoteName, registry: Registry, concurrency: UInt) async throws {
     SentrySDK.configureScope { scope in
       scope.setContext(value: ["imageName": name], key: "OCI")
     }
@@ -170,7 +170,7 @@ class VMStorageOCI: PrunableStorage {
 
     if !exists(digestName) {
       let transaction = SentrySDK.startTransaction(name: name.description, operation: "pull", bindToScope: true)
-      let tmpVMDir = try VMDirectory.temporary()
+      let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description)
 
       // Lock the temporary VM directory to prevent it's garbage collection
       let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
@@ -188,7 +188,7 @@ class VMStorageOCI: PrunableStorage {
       }
 
       try await withTaskCancellationHandler(operation: {
-        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest)
+        try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency)
         try move(digestName, from: tmpVMDir)
         transaction.finish()
       }, onCancel: {

Tests/TartTests/DirecotryShareTests.swift

@@ -0,0 +1,32 @@
+import XCTest
+@testable import tart
+
+final class DirectoryShareTests: XCTestCase {
+  func testNamedParsing() throws {
+    let share = try DirectoryShare(parseFrom: "build:/Users/admin/build")
+    XCTAssertEqual(share.name, "build")
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+  }
+  
+  func testNamedReadOnlyParsing() throws {
+    let share = try DirectoryShare(parseFrom: "build:/Users/admin/build:ro")
+    XCTAssertEqual(share.name, "build")
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(share.readOnly)
+  }
+  
+  func testOptionalNameParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertFalse(share.readOnly)
+  }
+  
+  func testOptionalNameReadOnlyParsing() throws {
+    let share = try DirectoryShare(parseFrom: "/Users/admin/build:ro")
+    XCTAssertNil(share.name)
+    XCTAssertEqual(share.path, URL(filePath: "/Users/admin/build"))
+    XCTAssertTrue(share.readOnly)
+  }
+}

Tests/TartTests/DockerConfigTests.swift

@@ -0,0 +1,16 @@
+import XCTest
+@testable import tart
+
+final class DockerConfigTests: XCTestCase {
+  func testHelpers() throws {
+    let config = DockerConfig(credHelpers: [
+      "(.*).dkr.ecr.(.*).amazonaws.com": "ecr-login",
+      "gcr.io": "gcloud"
+    ])
+
+    XCTAssertEqual(try config.findCredHelper(host: "gcr.io"), "gcloud")
+    XCTAssertEqual(try config.findCredHelper(host: "123.dkr.ecr.eu-west-1.amazonaws.com"), "ecr-login")
+    XCTAssertEqual(try config.findCredHelper(host: "456.dkr.ecr.us-east-1.amazonaws.com"), "ecr-login")
+    XCTAssertNil(try config.findCredHelper(host: "ghcr.io"))
+  }
+}

Tests/TartTests/LayerizerTests.swift

@@ -0,0 +1,90 @@
+import XCTest
+@testable import tart
+
+final class LayerizerTests: XCTestCase {
+  var registryRunner: RegistryRunner?
+
+  var registry: Registry {
+    registryRunner!.registry
+  }
+
+  override func setUp() async throws {
+    try await super.setUp()
+
+    do {
+      registryRunner = try await RegistryRunner()
+    } catch {
+      try XCTSkipIf(ProcessInfo.processInfo.environment["CI"] == nil)
+    }
+  }
+
+  override func tearDown() async throws {
+    try await super.tearDown()
+
+    registryRunner = nil
+  }
+
+  func testDiskV1() async throws {
+    // Original disk file to be pushed to the registry
+    let originalDiskFileURL = try fileWithRandomData(sizeBytes: 5 * 1024 * 1024 * 1024)
+    addTeardownBlock {
+      try FileManager.default.removeItem(at: originalDiskFileURL)
+    }
+
+    // Disk file to be pulled from the registry
+    // and compared against the original disk file
+    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+
+    print("pushing disk...")
+    let diskLayers = try await DiskV1.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+
+    print("pulling disk...")
+    try await DiskV1.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
+
+    print("comparing disks...")
+    try XCTAssertEqual(Digest.hash(originalDiskFileURL), Digest.hash(pulledDiskFileURL))
+  }
+
+  func testDiskV2() async throws {
+    // Original disk file to be pushed to the registry
+    let originalDiskFileURL = try fileWithRandomData(sizeBytes: 5 * 1024 * 1024 * 1024)
+    addTeardownBlock {
+      try FileManager.default.removeItem(at: originalDiskFileURL)
+    }
+
+    // Disk file to be pulled from the registry
+    // and compared against the original disk file
+    let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+
+    print("pushing disk...")
+    let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, progress: Progress())
+
+    print("pulling disk...")
+    try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
+
+    print("comparing disks...")
+    try XCTAssertEqual(Digest.hash(originalDiskFileURL), Digest.hash(pulledDiskFileURL))
+  }
+
+  private func fileWithRandomData(sizeBytes: Int) throws -> URL {
+    let devUrandom = try FileHandle(forReadingFrom: URL(filePath: "/dev/urandom"))
+
+    let temporaryFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+    FileManager.default.createFile(atPath: temporaryFileURL.path, contents: nil)
+    let temporaryFile = try FileHandle(forWritingTo: temporaryFileURL)
+
+    var remainingBytes = sizeBytes
+
+    while remainingBytes > 0 {
+      let randomData = try devUrandom.read(upToCount: min(64 * 1024 * 1024, remainingBytes))!
+      remainingBytes -= randomData.count
+      try temporaryFile.write(contentsOf: randomData)
+    }
+
+    try devUrandom.close()
+
+    try temporaryFile.close()
+
+    return temporaryFileURL
+  }
+}

docs/.markdownlint.yml

@@ -0,0 +1,12 @@
+"default": true
+"MD002": false # First heading should be a top level heading
+"MD007": # Unordered list indentation
+  indent: 4
+"MD009": false # Trailing spaces
+"MD013": false # Line length
+"MD025": false # Multiple top level headings in the same document
+"MD026": false # Trailing punctuation in heading
+"MD033": false # Inline HTML
+"MD041": false # First line in file should be a top level heading
+"MD045": false # OK not to have a description for an image
+"MD046": false #  Code block style [Expected: fenced; Actual: indented]

docs/blog/.authors.yml

@@ -1,8 +1,9 @@
-edigaryev:
-  name: Nikolay Edigaryev
-  description: Creator
-  avatar: https://github.com/edigaryev.png
-fkorotkov:
-  name: Fedor Korotkov
-  description: Creator
-  avatar: https://github.com/fkorotkov.png
+authors:
+  edigaryev:
+    name: Nikolay Edigaryev
+    description: Creator
+    avatar: https://github.com/edigaryev.png
+  fkorotkov:
+    name: Fedor Korotkov
+    description: Creator
+    avatar: https://github.com/fkorotkov.png

docs/blog/posts/2023-04-28-orchard-ssh-over-grpc.md

@@ -56,7 +56,7 @@ On bootstrap, each Orchard worker establishes a `Watch()` RPC stream and waits f
 
 Once `PortForward` instruction is received, the worker connects to the specified VM and port locally and opens a new `PortForward()` RPC stream with the controller, carrying the unique `session` identifier in the gRPC metadata to help distinguish several port forwarding requests.
 
-We’re using a pretty ingenious Golang package that turns any gRPC stream into a `net.Conn`: https://github.com/mitchellh/go-grpc-net-conn. This allows us to abstract from the gRPC details and simply proxy two `net.Conns`, thus providing the port forwarding functionality.
+We’re using a pretty ingenious [Golang package that turns any gRPC stream into a `net.Conn`](https://github.com/mitchellh/go-grpc-net-conn). This allows us to abstract from the gRPC details and simply proxy two `net.Conns`, thus providing the port forwarding functionality.
 
 We’ve also initially considered using [Yamux](https://github.com/hashicorp/yamux) to only keep a single connection with each worker, however, that involves the burden of dealing with flow control and potential implementation bugs associated with it, so we’ve decided to simply open an additional connection for each port forwarding session and let the OS deal with it.
 
@@ -74,25 +74,25 @@ Secondly, we’ve exposed three commands in the Orchard CLI that all use this en
 
 Opens a TCP port locally and forwards everything sent to it to the specified VM (and vice versa).
 
-For example, `orchard port-forward vm ventura-builder 2222:22` will forward traffic from the local TCP port `2222` to the `ventura-builder` VM’s TCP port `22`.
+For example, `orchard port-forward vm sonoma-builder 2222:22` will forward traffic from the local TCP port `2222` to the `ventura-builder` VM’s TCP port `22`.
 
 ### `orchard ssh`
 
 Connects to the specified VM on the default SSH port `22`, optionally only launching a command (if specified), similarly to what the official OpenSSH client does.
 
-For example, `orchard ssh vm ventura-builder` will open an interactive session with the `ventura-builder` VM.
+For example, `orchard ssh vm sonoma-builder` will open an interactive session with the `ventura-builder` VM.
 
 You can also send local scripts for execution by utilizing redirection:
 
 ```shell
-orchard ssh vm ventura-builder 'sh -s' < script.sh
+orchard ssh vm sonoma-builder 'sh -s' < script.sh
 ```
 
 ### `orchard vnc`
 
 Establishes a port forwarding to the specified VM’s default VNC port `5900` and opens the default macOS Screen Sharing app.
 
-For example, `orchard vnc vm ventura-builder` will establish a port-forwarding to the `ventura-builder` VM's port `5900` under the hood and launch macOS Screen Sharing app.
+For example, `orchard vnc vm sonoma-builder` will establish a port-forwarding to the `ventura-builder` VM's port `5900` under the hood and launch macOS Screen Sharing app.
 
 Note that the SSH and VNC commands expect the VM resource to specify credentials in it’s definition (can be done via `orchard create vm`),  and will otherwise fall back to the credentials specified by `--username` and `--password`, or if none specified — to de-facto standard of `admin:admin` credentials.
 

docs/blog/posts/2023-09-20-tart-2.0.0.md

@@ -0,0 +1,104 @@
+---
+draft: false
+date: 2023-09-20
+search:
+  exclude: true
+authors:
+  - fkorotkov
+categories:
+  - announcement
+---
+
+# Tart 2.0.0 and community updates
+
+Today we'd like to share some news and updates around the Tart ecosystem since the Tart 1.0.0 release back in February.
+
+<!-- more -->
+
+## Community Growth
+
+In the last 7 months Tart community almost tripled and growth is continuing to accelerate. Tart just crossed 25,000 installations,
+dozens of companies that we know of are using Tart in their daily workflows. If your company is not in the list please consider
+[joining](https://github.com/cirruslabs/tart/blob/main/Resources/Users/HowToAddYourself.md)!
+
+<div class="grid cards" markdown>
+
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/Krisp.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/Mullvad.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/ahrefs.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/Suran.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/Symflower.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/Transloadit.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/PITSGlobalDataRecoveryServices.png){ height="65" }
+- ![](https://github.com/cirruslabs/tart/raw/main/Resources/Users/Uphold.png){ height="65" }
+
+</div>
+
+We are also very pleased by how the community responded to [the license change](2023-02-11-changing-tart-license.md).
+We now have a number of companies running Tart at scale under the new license. Revenue from the licensing allowed us to
+allocate time to continue improving Tart which brings us to the section below.
+
+## Recent updates and what's changing in Tart 2.0.0
+
+In the last 7 months we've had 12 feature releases that brought a lot of features requested by the community. Here are just
+a few of them to highlight:
+
+-[Custom GitLab Runner Executor](/integrations/gitlab-runner/).
+-[Cluster Management via Orchard](2023-04-25-orchard-ga.md).
+-Numerous compatibility improvements for all kinds of OCI-registries.
+-Sonoma Support (see details [below](#macos-sonoma-updates)).
+
+But one of the most requested features/complaints was around pulling huge Tart images from remote OCI-compatible registries.
+With an ideal network conditions `tart pull` worked pretty good but in case of any network issues it was required to
+restart the pull from scratch. Additionally, some registries are notably slow streaming a single blob but can stream
+multiple blobs in parallel. Finally, the initial format of storing Tart VMs was very naive: disk image is compressed
+via a single stream which is chunked up into blobs that are serially uploaded to a registry. A single compression stream
+means that Tart can also only decompress blobs serially.
+
+Given these three observations above we came up with an improved format of storing Tart VM disk images. In Tart 2.0.0
+disk images are chunked up first and compressed independently into blobs, when pushed, each blob has attached annotations
+of expected uncompressed size and a checksum. This way when Tart 2.0.0 is pulling an image pushed by Tart 2.0.0 each blob can
+be pulled, uncompressed and written at the right offset independently. Having checksums along expected uncompressed blob size
+also allowed to support resumable pulls. Upon a failure Tart 2.0.0 will compare checksums of chunks and will continue pulling
+only missing blobs.
+
+Overall in our experiments we saw a 10% improvement in compressed size of the images and **4 times faster pulls**.
+
+In order to try the new image format please upgrade Tart and try to pull any of [the Sonoma images](https://github.com/orgs/cirruslabs/packages?tab=packages&q=macos-sonoma):
+
+```bash
+brew upgrade cirruslabs/cli/tart
+tart pull ghcr.io/cirruslabs/macos-sonoma-base:latest
+```
+
+## macOS Sonoma Updates
+
+Tart VMs now can be run in a "suspendable" mode which will enable VM snapshotting instead of the standard shutdown.
+VMs with an existing snapshot will `run` from the same state as they got snapshotted. Please check demo down below:
+
+<div>
+    <blockquote class="twitter-tweet" data-theme="dark">
+      <p lang="en" dir="ltr">
+        Tart 1.8.0 brings macOS Sonoma updates! 🍏 Now you can suspend and resume your virtual machines for even faster startup times. Check out the demo below 👇 <a href="https://t.co/RoRFT8Nwst">pic.twitter.com/RoRFT8Nwst</a>
+      </p>&mdash; Cirrus Labs (@cirrus_labs) <a href="https://twitter.com/cirrus_labs/status/1677308360385765382?ref_src=twsrc%5Etfw">July 7, 2023</a>
+    </blockquote> 
+    <script src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
+</div>
+
+There are two caveats to the "suspendable" mode support:
+
+1. Both host and guest should be running macOS Sonoma.
+2. Snapshots are locally encrypted and can't be shared between physical hosts. Therefore `tart push` won't push the corresponding snapshotted state of the VM.
+
+Try the "suspendable" mode for yourself by passing `--suspendable` flag to a `tart run` command:
+
+```bash
+tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+tart run --suspendable sonoma-base
+```
+
+## Conclusion
+
+We are very excited about this major release of Tart. Please give it a try and let us know how it went!
+
+Stay tuned for new updates and announcements! There are a few coming up very shortly...

docs/blog/posts/2023-10-06-tart-on-aws.md

@@ -0,0 +1,71 @@
+---
+draft: false
+date: 2023-10-06
+search:
+  exclude: true
+authors:
+  - fkorotkov
+categories:
+  - announcement
+---
+
+# Tart is now available on AWS Marketplace
+
+Announcing [official AMIs for EC2 Mac Instances](https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws)
+with preconfigured Tart installation that is optimized to work within AWS infrastructure.
+
+EC2 Mac Instances is a gem of engineering powered by AWS Nitro devices. Just imagine there is a physical Mac Mini with
+a plugged in Nitro device that can push the physical power button!
+
+![EC2 M2 Pro](/blog/images/ec2-mac2-m2pro.png)
+
+This clever synergy between Apple Hardware and Nitro System allows seamless integration with VPC networking and booting macOS from an EBS volume.
+
+In this blog post we’ll see how a virtualization solution like Tart can compliment and elevate experience with EC2 Mac Instances.
+
+<!-- more -->
+
+Let’s start from the basics, what EC2 Mac Instances allow to do compared to physical Mac Minis seating in offices of
+many companies around the world?
+
+First and foremost, EC2 Mac Instances sit inside AWS data centers and can leverage all the goodies of VPC networking
+within your company's existing infrastructure. No need to connect your Macs in the office through a VPN and deal
+with networking and security.
+
+Additionally, EC2 Mac Instances are booting from EBS volumes which means it is possible to always have reproducible instances
+and apply all the best practices of Infrastructure-as-Code. Managing a fleet of physical Macs is a pain and it's very hard
+to make them configured in a reproducible and stable way. With booting from identical EBS volumes your team is always sure
+about the identical initial state of the fleet.
+
+## Compromises of EC2 Mac Instances
+
+The flexibility of EBS volumes for macOS comes with some compromises that virtualization solutions like Tart can help with.
+The initial boot from an EBS volume takes some time and not instant. macOS itself is pretty heavy and a Nitro device needs
+to download tens of gigabytes that macOS requires in order to boot. This means that **resetting a EC2 Mac Instance to a clean state
+is not instant and usually takes a couple of minutes** when you can’t utilize the precious resources for your workloads.
+
+It is much easier to tailor such EBS volumes with tools like Packer but there is still a **friction to test newly created EBS volumes**
+since one needs to start and run a EC2 Mac Instance and it’s not possible to test things locally. Similarly it is even harder
+to test beta versions of macOS that require manual interaction with a running instance.
+
+## Solution
+
+Tart can help with all the compromises! Tart virtual machines (VMs) have nearly native performance thanks to utilizing
+native `Virtualization.Framework` that was developed along the first Apple Silicon chip. **Tart VMs can be copied/disposed
+instantly and booting a fresh Tart VM takes only several seconds**. It is also possible to run two different Tart VMs in parallel
+that can have completely different versions of macOS and packages. For example, it is possible to have the latest stable macOS
+with the release version of Xcode along with the next version of macOS with the latest beta of Xcode.
+
+Creation of Tart VMs can be automated with [a Packer plugin](https://github.com/cirruslabs/packer-plugin-tart) the same way as
+creation of EC2 AMIs with one caveat that **Tart Packer Plugin works locally so you can test the same virtual machine locally
+as you would run it in the cloud**.
+
+Lightweight nature of Tart VMs with a focus on an easy-to-integrate Tart CLI compliments any macOS automation and helps to reduce
+the feedback cycle and improves reproducibility of macOS environments even further.
+
+## Conclusion
+
+We are excited to bring [official AMIs that include Tart installation optimized to work within AWS](https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws).
+In the coming weeks when macOS Sonoma will become available on AWS we’ll release another update specifically targeting EC2 Mac Instances. 
+This update will simplify access to local SSDs of Mac Instances that are slightly faster than EBS volumes. Stay tuned and don’t hesitate
+to ask any [questions](https://tart.run/licensing/).

docs/blog/posts/2023-11-03-cirrus-runners-dashboard.md

@@ -0,0 +1,59 @@
+---
+draft: false
+date: 2023-11-03
+search:
+  exclude: true
+authors:
+  - fkorotkov
+categories:
+  - announcement
+---
+
+# New dashboard with insights into performance of Cirrus Runners
+
+This month we are celebrating one year since launching Cirrus Runners — managed Apple Silicon infrastructure for your
+GitHub Actions. During the last 12 months we ran millions of workflows for our customers and now ready to share some insights
+into price performance of them for our customers.
+
+One of the key difference with Cirrus Runners is how they are getting billed for. Customers purchase Cirrus Runners via monthly subscription
+that costs $150 per each Cirrus Runner. Each runner can be used 24 hours a day 7 days a week to run GitHub Actions workflows
+for an organization. If there are more outstanding jobs than available runners then they are queued and executed as soon as
+there is a free runner. This is different from how GitHub-managed GitHub Actions are billed for — you pay for each minute of execution time.
+
+The benefit of a fixed price is that you can run as many jobs as you want without worrying about the cost. The downside is that
+you need to make sure that you are using your runners efficiently. This is where the new dashboard comes in handy.
+
+<!-- more -->
+
+But first, **let's see theoretically the lowest price per minute** of a Cirrus Runners. If you run 24 hours a day 7 days a week
+then you will get 43,200 minutes of execution time per month. This means that the price per minute is $0.0035 if your runners
+utilization is 100%. But even if your engineering teams is located in a single time zone and works 8 hours a day 5 days a week
+then you will get 9,600 minutes of execution time per month which comes down to $0.015 per-minute. This is still more than 10 times cheaper
+than recently announced Apple Silicon GitHub-manged runners that cost $0.16 per minute.
+
+Now lets take a look at the new Cirrus Runners dashboard of a real customers that run their workflows on Cirrus Runners
+and **practically pushing the price performance pretty close to the theoretical minimum**.
+
+![Cirrus Runners Dashboard](/blog/images/runners-price-performance-2.png)
+
+As you can see above Cirrus Runners Dashboard focuses on 4 core metrics:
+
+1. **Minutes Used** — overall amount of minutes that Cirrus Runners were executing jobs.
+2. **Workflow Runs** — absolute number of workflow runs that were executed on Cirrus Runners.
+3. **Queue Size** — number of jobs that were queued and waiting for a free Cirrus Runner.
+4. **Queue Time** — average time that jobs were waiting in the queue.
+
+In this particular example price performance of Cirrus Runners is $0.006 per minute which is 2 times more than the theoretical minimum
+and **26 times better than GitHub-managed Apple Silicon runners**. But this is a extreme example, looking at queue time and queue size
+we can see that the downside of such great price performance is that jobs are waiting in the queue on average around 5 minutes.
+
+Here is another example of Cirrus Runners Dashboard for a different customer that has a slightly higher price performance of $0.017 per minute
+but at the same time doesn't experience queue time at all. **Note that $0.017 is still 10 times cheaper than GitHub-managed Apple Silicon runners**.
+
+![Cirrus Runners Dashboard](/blog/images/runners-price-performance-3.png)
+
+## Conclusion
+
+Having a fixed price for Cirrus Runners is a great way to save money on your CI/CD infrastructure and just in general have predictable budged.
+But it requires keeping the balance between price per minute and queue time. Cirrus Runners Dashboard helps you to keep an eye on this balance
+and make sure that you are getting the most out of your Cirrus Runners.

docs/integrations/cirrus-cli.md

@@ -13,7 +13,7 @@ task:
   name: hello
   macos_instance:
     # can be a remote or a local virtual machine
-    image: ghcr.io/cirruslabs/macos-monterey-base:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-base:latest
   hello_script:
     - echo "Hello from within a Tart VM!"
     - echo "Here is my CPU info:"
@@ -45,7 +45,7 @@ exposes it via [`artifacts` instruction](https://cirrus-ci.org/guide/writing-tas
 task:
   name: Build
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-xcode:latest
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:latest
   build_script: swift build --product tart
   binary_artifacts:
     path: .build/debug/tart

docs/integrations/github-actions.md

@@ -1,32 +1,115 @@
-# GitHub Actions
+# Cirrus Runners for GitHub Actions
 
-Tart already powers several CI services mentioned above including our own [Cirrus CI](https://cirrus-ci.org/guide/macOS/) which offers unlimited concurrency with per-second billing.
-For services that haven't leveraged Tart yet, we offer fully managed runners via a monthly subscription.
-*Cirrus Runners* is the fastest way to get your current CI workflows to benefit from Apple Silicon hardware. No need to manage infrastructure or migrate to another CI provider.
+*Cirrus Runners* is the fastest and most cost-efficient way to get your current CI workflows to benefit from Apple Silicon hardware. No need to manage infrastructure or migrate to another CI provider.
+Your actions will be executed in clean macOS virtual machines with 4 Apple M2 cores.
 
 ## Testimonials from customers
 
-Sebastian Jachec, Mobile Engineer at [Daybridge](https://www.daybridge.com/).
-
-> It’s been plain-sailing with the Cirrus Runners — they’ve been great! They’re consistently 60+% faster on workflows that we previously used Github Actions’ macOS runners for.
-
-Max Lapides, Senior Mobile Engineer at [Tonal](https://www.tonal.com/).
+Max Lapides, Senior Mobile Engineer at [Tonal](https://www.tonal.com/):
 
 > Previously, we were using the GitHub‑hosted macOS runners and our iOS build took ~30 minutes. Now with Cirrus Runners, the iOS build only takes ~12 minutes. That’s a huge boost to our productivity, and for only $150/month per runner it is much less expensive too.
 
+John A., Software Engineer at [GitKraken](https://www.gitkraken.com/):
+
+> GitHub Actions MacOS-x86 runners have become increasingly unreliable, so we're moving our Mac builds over to arm64 because Cirrus Labs' M1 runners are not only ~3 times faster, they've also been far more stable.
+
+Sebastian Jachec, Mobile Engineer at [Daybridge](https://www.daybridge.com/):
+
+> It’s been plain-sailing with the Cirrus Runners — they’ve been great! They’re consistently 60+% faster on workflows that we previously used Github Actions’ macOS runners for.
+
+## Pricing
+
+Each Cirrus Runner costs $150 a month and there is no limit on the amount of minutes for your actions.
+We recommend to purchase several Cirrus Runners depending on your team size, so you can run actions in
+parallel. Note that you can change your subscription at any time via [this page](https://billing.stripe.com/p/login/3cs7vNbzo92p7fy3cc)
+or by emailing [support@cirruslabs.org](mailto:support@cirruslabs.org).
+
+### Priority Support
+
+Subscriptions of 20 or more Cirrus Runners include access to [Priority Support](../licensing.md#priority-support).
+Please contact [sales@cirruslabs.org](mailto:sales@cirruslabs.org) in order to get all the details.
+
+### CPU and Memory resources of Cirrus Runners
+
+By default, a single Cirrus Runner is allocated with 4 M2 cores and 12 GB of unified memory which is enough for most of the workloads.
+For workloads that require more resources it is possible to use XL Cirrus Runners which have twice the resources: a full M2 chip with 8 cores
+and 24 GB of unified memory. Note that a single XL Cirrus Runner also uses twice the concurrency.
+
+In order to use an XL Cirrus Runner for a job please append `-xl` suffix to your `runs-on` property. More on that down below.
+
+## Installation
+
+Once you configure [Cirrus Runners App](https://github.com/apps/cirrus-runners) for your organization, you'll be redirected
+to a checkout page powered by Stripe. During the checkout process you'll be able to configure a subscription for
+a desired amount of parallel Cirrus Runners and try it for free for 10 days.
+
+Once configured, please follow instruction below. If you have any questions please contact [support@cirruslabs.org](mailto:support@cirruslabs.org).
+Subscriptions with more than 10 runners also include Priority Support 
 
 ## Configuring Cirrus Runners
 
-Configuring Cirrus Runners for GitHub Actions is as simple as installing [Cirrus Runners App](https://github.com/apps/cirrus-runners).
-After successful installation and subscription configuration, use any of [Ventura images managed by us](https://github.com/cirruslabs/macos-image-templates) in `runs-on`:
+In order for Cirrus Runners to be used by your GitHub Actions workflow jobs, specify a desired image in the `runs-on` property.
 
-```yaml
-name: Test Suite
-jobs:
-  test:
-    runs-on: ghcr.io/cirruslabs/macos-ventura-xcode:latest
-```
+=== "Default Cirrus Runner"
+
+    ```yaml
+    name: Tests
+    jobs:
+      test:
+        runs-on: ghcr.io/cirruslabs/macos-sonoma-xcode:latest
+    ```
+
+=== "XL Cirrus Runner"
+
+    ```yaml
+    name: Integration Tests
+    jobs:
+      test:
+        runs-on: ghcr.io/cirruslabs/macos-sonoma-xcode:latest-xl
+    ```
+
+List of all available images can be found in [this repository](https://github.com/cirruslabs/macos-image-templates).
+
+Note that Tart VM images don't have the same set of pre-installed packages as the official Intel GitHub runners.
+If something is missing please [create an issue within this repository](https://github.com/cirruslabs/macos-image-templates/issues/new).
 
 When workflows are executing you'll see Cirrus on-demand runners on your organization's settings page at `https://github.com/organizations/<ORGANIZATION>/settings/actions/runners`.
+Note that Cirrus Runners will get added to the default runner group.
+
+!!! tip "Using Cirrus Runners with public repositories"
+
+    By default, only private repositories can access runners in a default runner group, but you can override this in your organization's settings:
+
+    ```https://github.com/organizations/<YOUR ORGANIZATION NAME>/settings/actions/runner-groups/1```
 
 ![](/assets/images/TartGHARunners.png)
+
+### Dashboard
+
+You can also see the status of your runners on the [Cirrus Runners Dashboard](https://cirrus-runners.app/). This dashboard
+also provides insights into price performance of your Cirrus Runners. Please check out [this blog post](/blog/2023/11/03/new-dashboard-with-insights-into-performance-of-cirrus-runners/)
+to learn more about what this dashboard can do for you.
+
+![](/assets/images/RunnersDashboard.png)
+
+## Data handling flow
+
+By design Cirrus Runners service never sees any of your secrets or source code and acts as compute platform with the lastest
+Apple Silicon hardware that can quickly allocate CPU/Memory resources for your jobs.
+
+Here is a high-level overview of how Cirrus Runners service manages runners for your organization:
+
+- Cirrus Runner GitHub App is subscribed to [`workflow_job`](https://docs.github.com/en/webhooks/webhook-events-and-payloads#workflow_job).
+- Upon receiving a new event targeting Cirrus Runners via `runs-on` property the following steps take place:
+
+    - Non-personal information about your job is saved to perform health checking of Cirrus Runners execution.
+    - Cirrus Runners GitHub App has only one permission that allows generating temporary registration tokens for
+      self-hosted GitHub Actions Runners. Note that Cirrus Runners GitHub App itself doesn't have access to contents of
+      repositories in your organization.
+    - Cirrus Runners Service creates a new single use Tart VM, generates a temporary registration tokens for self-hosted runners
+      and passes it without storing inside the VM for the GitHub Actions Runner service to [start a ephemeral runner](https://github.blog/changelog/2021-09-20-github-actions-ephemeral-self-hosted-runners-new-webhooks-for-auto-scaling/).
+
+- Cirrus Runners service continuously monitors health of the Tart VM executing your job to make sure it runs to completion.
+- After the job finishes the ephemeral Tart VM is getting destroyed with all the information of the job run.
+
+If you have any questions or concerns please feel free to reach out to [support@cirruslabs.org](mailto:support@cirruslabs.org).

docs/integrations/gitlab-runner.md

@@ -37,7 +37,7 @@ Now you can use Tart Images in your `.gitlab-ci.yml`:
 ```yaml
 # You can use any remote Tart Image.
 # Tart Executor will pull it from the registry and use it for creating ephemeral VMs.
-image: ghcr.io/cirruslabs/macos-ventura-base:latest
+image: ghcr.io/cirruslabs/macos-sonoma-base:latest
 
 test:
   tags:

docs/integrations/vm-management.md

@@ -11,8 +11,8 @@ Tart can create VMs from `*.ipsw` files. You can download a specific `*.ipsw` fi
 use `latest` instead of a path to `*.ipsw` to download the latest available version:
 
 ```bash
-tart create --from-ipsw=latest monterey-vanilla
-tart run monterey-vanilla
+tart create --from-ipsw=latest sonoma-vanilla
+tart run sonoma-vanilla
 ```
 
 After the initial booting of the VM you'll need to manually go through the macOS installation process. As a convention we recommend creating an `admin` user with an `admin` password. After the regular installation please do some additional modifications in the VM:
@@ -54,7 +54,7 @@ Please refer to `tart set --help` for additional details.
 ## Building with Packer
 
 Please refer to [Tart Packer Plugin repository](https://github.com/cirruslabs/packer-plugin-tart) for setup instructions.
-Here is an example of a template to build `monterey-base` local image based of a remote image:
+Here is an example of a template to build a local image based of a remote image:
 
 ```hcl
 packer {
@@ -67,8 +67,8 @@ packer {
 }
 
 source "tart-cli" "tart" {
-  vm_base_name = "ghcr.io/cirruslabs/macos-ventura-base:latest"
-  vm_name      = "my-custom-ventura"
+  vm_base_name = "ghcr.io/cirruslabs/macos-sonoma-base:latest"
+  vm_name      = "my-custom-sonoma"
   cpu_count    = 4
   memory_gb    = 8
   disk_size_gb = 70
@@ -92,7 +92,9 @@ Here is a [repository with Packer templates](https://github.com/cirruslabs/macos
 
 ## Working with a Remote OCI Container Registry
 
+<!-- markdownlint-disable MD034 -->
 For example, let's say you want to push/pull images to a registry hosted at https://acme.io/.
+<!-- markdownlint-enable MD034 -->
 
 ### Registry Authorization
 

docs/licensing.md

@@ -8,21 +8,59 @@ are licensed under [Fair Source License](https://fair.io/). Usage on personal co
 but organizations that exceed a certain number of server installations (100 CPU cores for Tart and/or 4 hosts for Orchard)
 will be required to obtain a paid license.
 
-??? note "Performance and Efficiency Cores"
-    The virtual CPU cores in Tart VMs do not differentiate between the high-performance  and high-efficient cores
-    of the host CPU. Instead, Tart VMs automatically alternate between these types of cores depending on the workload
-    being executed within the virtual machines. As a result, both performance and energy-efficient cores of the host CPU
-    are treated equally in terms of licensing.
+??? note "Host CPU Core usage"
+    The virtual CPU cores of Tart VMs are not tied to specific physical cores of the host CPU. Instead, for optimal performance
+    Tart VMs will automatically try to balance compute between all available cores of the host CPU. As a result,
+    all performance and energy-efficient cores of the host CPU are always counted towards the license usage.
 
 # License Tiers
 
-When an organization surpasses the 100 CPU cores limit, it is required to obtain a Gold Tier License, which costs \$1000 per month.
-Upon reaching a limit of 500 CPU cores, a Platinum Tier License (\$5000 per month) will be required, and for organizations
-that exceed 5000 CPU cores, a custom Diamond Tier License (\$1 per core per month) will be necessary.
+## Free Tier
+
+By default, when no [license is purchased](#get-the-license), it is assumed that an organization is using a Free Tier license.
+You can find the Free Tier license text in [Tart](https://github.com/cirruslabs/tart/blob/main/LICENSE) and [Orchard](https://github.com/cirruslabs/orchard/blob/main/LICENSE) repositories.
+
+Free Tier license has a 100 CPU core limit for Tart and 4 Orchard Workers limit for Orchard.
+
+??? info "Usage Scenarios Examples"
+
+    Here are a few examples that fit into the free tier:
+
+      - Using Tart on 12 Mac Minis with 8 CPUs each running up to 24 VMs in parallel.
+      - Creating an Orchard cluster of 4 Mac Studio workers with 24 CPUs each.
+
+    Here are a few examples that do not fit into the free tier:
+
+      - Using Tart on 13 Mac Minis with 8 CPUs each.
+      - Creating an Orchard cluster of 5 Mac Minis workers with 8 CPUs each.
+
+## Gold Tier
+
+If an organization wishes to exceed the limits of the Free Tier license, a purchase of the [Gold Tier License](#get-the-license) is required, which costs \$1000 per month.
+
+Gold Tier license has a 500 CPU core limit for Tart and 20 Orchard Workers limit for Orchard.
+
+## Platinum Tier
+
+If an organization wishes to exceed the limits of the Gold Tier license, a purchase of the [Platinum Tier License](#get-the-license) is required, which costs \$5000 per month.
+
+Platinum Tier license has a 5,000 CPU core limit for Tart and 200 Orchard Workers limit for Orchard.
+
+## Diamond Tier
+
+For organizations that wish to exceed the limits of the Platinum Tier license, a purchase of a [custom Diamond Tier License](#get-the-license) is required, which costs \$1 per CPU core per month and gives the ability to run unlimited Orchard Workers.
+
+# Get the license
 
 If your organization is interested in purchasing one of the license tiers, please email [licensing@cirruslabs.org](mailto:licensing@cirruslabs.org).
+
 You can see a template of a license subscription agreement [here](assets/TartLicenseSubscription.pdf).
 
+!!! info "Running on AWS?"
+
+    There are [official AMIs for EC2 Mac Instances](https://aws.amazon.com/marketplace/pp/prodview-qczco34wlkdws)
+    with preconfigured Tart installation that is optimized to work within AWS infrastructure.
+
 # General Support
 
 The best way to ask general questions about particular use cases is to email our support team at [support@cirruslabs.org](mailto:support@cirruslabs.org).

docs/quick-start.md

@@ -3,12 +3,12 @@ hide:
   - navigation
 ---
 
-Try running a Tart VM on your Apple Silicon device running macOS 12.0 (Monterey) or later (will download a 25 GB image):
+Try running a Tart VM on your Apple Silicon device running macOS 13.0 (Ventura) or later (will download a 25 GB image):
 
 ```bash
 brew install cirruslabs/cli/tart
-tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
-tart run ventura-base
+tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+tart run sonoma-base
 ```
 
 ??? info "Manual installation from a release archive"
@@ -17,8 +17,8 @@ tart run ventura-base
     ```bash
     curl -LO https://github.com/cirruslabs/tart/releases/latest/download/tart.tar.gz
     tar -xzvf tart.tar.gz
-    ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
-    ./tart.app/Contents/MacOS/tart run ventura-base
+    ./tart.app/Contents/MacOS/tart clone ghcr.io/cirruslabs/macos-sonoma-base:latest sonoma-base
+    ./tart.app/Contents/MacOS/tart run sonoma-base
     ```
 
     Please note that `./tart.app/Contents/MacOS/tart` binary is required to be used in order to trick macOS
@@ -33,9 +33,19 @@ tart run ventura-base
 If the guest VM is running and configured to accept incoming SSH connections you can conveniently connect to it like so:
 
 ```bash
-ssh admin@$(tart ip macos-ventura-base)
+ssh admin@$(tart ip sonoma-base)
 ```
 
+!!! tip "Running scripts inside Tart virtual machines"
+    We recommend using [Cirrus CLI](integrations/cirrus-cli.md) to run scripts and/or retrieve artifacts
+    from within Tart virtual machines. Alternatively, you can use plain ssh connection and `tart ip` command:
+
+    ```bash
+    brew install sshpass
+    sshpass -p admin ssh -o "StrictHostKeyChecking no" admin@$(tart ip sonoma-base) "uname -a"
+    sshpass -p admin ssh -o "StrictHostKeyChecking no" admin@$(tart ip sonoma-base) < script.sh
+    ```
+
 ## Mounting directories
 
 To mount a directory, run the VM with the `--dir` argument:
@@ -70,6 +80,17 @@ The directory we've mounted above will be accessible from the `/Volumes/My Share
 
 Note: to use the directory mounting feature, the guest VM needs to run macOS 13.0 (Ventura) or newer.
 
+??? tip "Changing mount location"
+    It is possible to remount the directories after a virtual machine is started by running the following commands:
+    
+    ```bash
+    sudo umount "/Volumes/My Shared Files"
+    mkdir ~/workspace
+    mount_virtiofs com.apple.virtio-fs.automount ~/workspace
+    ```
+
+    After running the above commands the direcory will be available at `~/workspace/project`
+
 ### Accessing mounted directories in Linux guests
 
 To be able to access the shared directories from the Linux guest, you need to manually mount the virtual filesystem first:
@@ -79,4 +100,3 @@ mount -t virtiofs com.apple.virtio-fs.automount /mnt/shared
 ```
 
 The directory we've mounted above will be accessible from the `/mnt/shared/project` path inside a guest VM.
-

docs/theme/overrides/home.html

@@ -1,11 +1,5 @@
 {% extends "base.html" %}
 
-{% block announce %}
-<a href="/blog/2023/04/25/announcing-orchard-orchestration-for-managing-macos-virtual-machines-at-scale/">
-  🚀🚀🚀&nbsp&nbspAnnouncing <strong>Orchard</strong> orchestration for managing macOS virtual machines at scale&nbsp;&nbsp;🚀🚀🚀
-</a>
-{% endblock %}
-
 <!-- Render landing page under tabs -->
 {% block tabs %} {{ super() }}
 
@@ -81,7 +75,7 @@
   }
 </style>
 
-<script src="https://unpkg.com/@dotlottie/player-component@latest/dist/dotlottie-player.js"></script>
+<script src="https://unpkg.com/@dotlottie/player-component@1.4.2/dist/dotlottie-player.js"></script>
 
 <!-- landing page for landing page -->
 <!-- Hero -->
@@ -242,12 +236,15 @@
             }
             let counterElement = document.getElementById('installation-counter');
             if (counterElement) {
-              counterElement.textContent = Math.round(allDownloads / 1000) + ",000"
+              // Live installation count is available starting version 1.0.0
+              // Prior Tart was installed a little over 14,000 times, let's count them too
+              let installationPriorV1 = 14
+              counterElement.textContent = (installationPriorV1 + Math.round(allDownloads / 1000)) + ",000"
             }
           })
       </script>
       <h2>
-        With more than <strong id="installation-counter">10,000</strong> installations to date, Tart has been adopted for various scenarios.
+        With more than <strong id="installation-counter">25,000</strong> installations to date, Tart has been adopted for various scenarios.
         Its applications range from powering CI/CD pipelines and reproducible local development environments,
         to helping in the testing of device management systems without actual physical devices.
       </h2>

integration-tests/requirements.txt

@@ -3,3 +3,4 @@ testcontainers
 requests
 bitmath
 pytest-dependency
+paramiko

integration-tests/tart.py

@@ -7,10 +7,9 @@ class Tart:
     def __init__(self):
         self.tmp_dir = tempfile.TemporaryDirectory(dir=os.environ.get("CIRRUS_WORKING_DIR"))
 
-        # Link to the users IPSW cache to make things faster
-        src = os.path.join(os.path.expanduser("~"), ".tart", "cache", "IPSWs")
-        dst = os.path.join(self.tmp_dir.name, "cache", "IPSWs")
-        os.makedirs(os.path.join(self.tmp_dir.name, "cache"))
+        # Link to the users cache to make things faster
+        src = os.path.join(os.path.expanduser("~"), ".tart", "cache")
+        dst = os.path.join(self.tmp_dir.name, "cache")
         os.symlink(src, dst)
 
     def __enter__(self):
@@ -31,3 +30,9 @@ class Tart:
         completed_process.check_returncode()
 
         return completed_process.stdout.decode("utf-8"), completed_process.stderr.decode("utf-8")
+
+    def run_async(self, args) -> subprocess.Popen:
+        env = os.environ.copy()
+        env.update({"TART_HOME": self.tmp_dir.name})
+
+        return subprocess.Popen(["tart"] + args, env=env)

integration-tests/test_clone.py

@@ -6,5 +6,5 @@ def test_clone(tart):
     tart.run(["clone", "debian", "ubuntu"])
 
     # Ensure that we have now 2 VMs
-    stdout, _, = tart.run(["list", "--quiet"])
+    stdout, _, = tart.run(["list", "--source", "local", "--quiet"])
     assert stdout == "debian\nubuntu\n"

integration-tests/test_create.py

@@ -3,7 +3,7 @@ def test_create_macos(tart):
     tart.run(["create", "--from-ipsw", "latest", "macos-vm"])
 
     # Ensure that the VM was created
-    stdout, _ = tart.run(["list", "--quiet"])
+    stdout, _ = tart.run(["list", "--source", "local", "--quiet"])
     assert stdout == "macos-vm\n"
 
 
@@ -12,5 +12,5 @@ def test_create_linux(tart):
     tart.run(["create", "--linux", "linux-vm"])
 
     # Ensure that the VM was created
-    stdout, _ = tart.run(["list", "--quiet"])
+    stdout, _ = tart.run(["list", "--source", "local", "--quiet"])
     assert stdout == "linux-vm\n"

integration-tests/test_delete.py

@@ -3,12 +3,12 @@ def test_delete(tart):
     tart.run(["create", "--linux", "debian"])
 
     # Ensure that the VM exists
-    stdout, _, = tart.run(["list", "--quiet"])
+    stdout, _, = tart.run(["list", "--source", "local", "--quiet"])
     assert stdout == "debian\n"
 
     # Delete the VM
     tart.run(["delete", "debian"])
 
     # Ensure that the VM was removed
-    stdout, _, = tart.run(["list", "--quiet"])
+    stdout, _, = tart.run(["list", "--source", "local", "--quiet"])
     assert stdout == ""

integration-tests/test_rename.py

@@ -6,5 +6,5 @@ def test_rename(tart):
     tart.run(["rename", "debian", "ubuntu"])
 
     # Ensure that the VM is now named "ubuntu"
-    stdout, _, = tart.run(["list", "--quiet"])
+    stdout, _, = tart.run(["list", "--source", "local", "--quiet"])
     assert stdout == "ubuntu\n"

integration-tests/test_run.py

@@ -0,0 +1,30 @@
+import uuid
+
+from paramiko.client import SSHClient, AutoAddPolicy
+
+
+def test_run(tart):
+    vm_name = f"integration-test-run-{uuid.uuid4()}"
+
+    # Instantiate a VM with admin:admin SSH access
+    tart.run(["clone", "ghcr.io/cirruslabs/macos-sonoma-base:latest", vm_name])
+
+    # Run the VM asynchronously
+    tart_run_process = tart.run_async(["run", vm_name])
+
+    # Obtain the VM's IP
+    stdout, _ = tart.run(["ip", vm_name, "--wait", "120"])
+    ip = stdout.strip()
+
+    # Connect to the VM over SSH and shutdown it
+    client = SSHClient()
+    client.set_missing_host_key_policy(AutoAddPolicy)
+    client.connect(ip, username="admin", password="admin")
+    client.exec_command("sudo shutdown -h now")
+
+    # Wait for the "tart run" to finish successfully
+    tart_run_process.wait()
+    assert tart_run_process.returncode == 0
+
+    # Delete the VM
+    _, _ = tart.run(["delete", vm_name])

mkdocs.yml

@@ -82,6 +82,8 @@ markdown_extensions:
   - pymdownx.tasklist:
       custom_checkbox: true
   - pymdownx.tilde
+  - attr_list
+  - md_in_html
 
 nav:
   - "Home": index.md

scripts/run-signed.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # helper script to build and run a signed tart binary
-# usage: ./scripts/run-signed.sh run ventura-base
+# usage: ./scripts/run-signed.sh run sonoma-base
 
 set -e