Merge 92b64f6b9a into f87b57bbc5

Sources/tart/Commands/Save.swift

@@ -0,0 +1,63 @@
+import ArgumentParser
+import Foundation
+
+struct Save: AsyncParsableCommand {
+  static var configuration = CommandConfiguration(abstract: "Save a VM to an OCI archive file")
+
+  @Argument(help: "local VM name", completion: .custom(completeMachines))
+  var localName: String
+
+  @Argument(help: "output archive path", completion: .file())
+  var path: String
+
+  @Option(help: "concurrency for disk layer compression")
+  var concurrency: UInt = 4
+
+  @Option(name: [.customLong("label")], help: ArgumentHelp("additional metadata to attach to the OCI image configuration in key=value format",
+                                                           discussion: "Can be specified multiple times to attach multiple labels."))
+  var labels: [String] = []
+
+  @Option(help: "tag to assign to the saved image (default: latest)")
+  var tag: String?
+
+  func run() async throws {
+    let localVMDir = try VMStorageHelper.open(localName)
+    let lock = try localVMDir.lock()
+    if try !lock.trylock() {
+      throw RuntimeError.VMIsRunning(localName)
+    }
+
+    let resolvedPath: String
+    if path.hasPrefix("/") {
+      resolvedPath = path
+    } else {
+      resolvedPath = FileManager.default.currentDirectoryPath + "/" + path
+    }
+
+    try await localVMDir.saveToArchive(
+      path: resolvedPath,
+      concurrency: concurrency,
+      labels: parseLabels(),
+      tag: tag
+    )
+  }
+
+  func parseLabels() -> [String: String] {
+    var result = [String: String]()
+
+    for label in labels {
+      let parts = label.trimmingCharacters(in: .whitespaces).split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false)
+
+      let key = parts.count > 0 ? String(parts[0]) : ""
+      let value = parts.count > 1 ? String(parts[1]) : ""
+
+      if key.isEmpty {
+        continue
+      }
+
+      result[key] = value
+    }
+
+    return result
+  }
+}

Sources/tart/OCI/BlobStorage.swift

@@ -0,0 +1,7 @@
+import Foundation
+
+protocol BlobStorage {
+  func pushBlob(fromData: Data, chunkSizeMb: Int, digest: String?) async throws -> String
+  func blobExists(_ digest: String) async throws -> Bool
+  func pushManifest(reference: String, manifest: OCIManifest) async throws -> String
+}

Sources/tart/OCI/Layerizer/Disk.swift

@@ -1,6 +1,6 @@
 import Foundation
 
 protocol Disk {
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
+  static func push(diskURL: URL, registry: any BlobStorage, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
   static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
 }

Sources/tart/OCI/Layerizer/DiskV2.swift

@@ -22,7 +22,7 @@ class DiskV2: Disk {
   private static let holeGranularityBytes = 4 * 1024 * 1024
   private static let zeroChunk = Data(count: holeGranularityBytes)
 
-  static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
+  static func push(diskURL: URL, registry: any BlobStorage, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
     var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []
 
     // Open the disk file

Sources/tart/OCI/Manifest.swift

@@ -4,6 +4,10 @@ import Foundation
 let ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json"
 let ociConfigMediaType = "application/vnd.oci.image.config.v1+json"
 
+// Docker manifest and config media types (schema v2)
+let dockerManifestMediaType = "application/vnd.docker.distribution.manifest.v2+json"
+let dockerConfigMediaType = "application/vnd.docker.container.image.v1+json"
+
 // Layer media types
 let configMediaType = "application/vnd.cirruslabs.tart.config.v1"
 let diskV2MediaType = "application/vnd.cirruslabs.tart.disk.v2"

Sources/tart/OCI/OCIArchiveWriter.swift

@@ -0,0 +1,119 @@
+import Foundation
+
+class OCIArchiveWriter {
+  private let tmpDir: URL
+  private let blobsDir: URL
+  private let lock: FileLock
+  private var manifestDigest: String?
+  private var manifestSize: Int?
+  private var manifestReferences: [String] = []
+  private var manifestData: Data?
+
+  init() throws {
+    tmpDir = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString)
+    blobsDir = tmpDir.appendingPathComponent("blobs/sha256")
+    try FileManager.default.createDirectory(at: blobsDir, withIntermediateDirectories: true)
+    lock = try FileLock(lockURL: tmpDir)
+    if try !lock.trylock() {
+      throw RuntimeError.Generic("failed to lock archive staging directory")
+    }
+  }
+
+  deinit {
+    try? lock.unlock()
+    try? FileManager.default.removeItem(at: tmpDir)
+  }
+}
+
+extension OCIArchiveWriter: BlobStorage {
+  func pushBlob(fromData: Data, chunkSizeMb: Int, digest: String?) async throws -> String {
+    let resolvedDigest = digest ?? Digest.hash(fromData)
+    let hex = resolvedDigest.replacingOccurrences(of: "sha256:", with: "")
+    let blobPath = blobsDir.appendingPathComponent(hex)
+    try fromData.write(to: blobPath)
+    return resolvedDigest
+  }
+
+  func blobExists(_ digest: String) async throws -> Bool {
+    let hex = digest.replacingOccurrences(of: "sha256:", with: "")
+    let blobPath = blobsDir.appendingPathComponent(hex)
+    return FileManager.default.fileExists(atPath: blobPath.path)
+  }
+
+  func pushManifest(reference: String, manifest: OCIManifest) async throws -> String {
+    if let existingDigest = manifestDigest, let existingData = manifestData {
+      let newData = try manifest.toJSON()
+      if newData == existingData {
+        manifestReferences.append(reference)
+        return existingDigest
+      }
+    }
+
+    let data = try manifest.toJSON()
+    let digest = Digest.hash(data)
+    let hex = digest.replacingOccurrences(of: "sha256:", with: "")
+    let blobPath = blobsDir.appendingPathComponent(hex)
+    try data.write(to: blobPath)
+    manifestDigest = digest
+    manifestSize = data.count
+    manifestData = data
+    manifestReferences.append(reference)
+    return digest
+  }
+
+  func finalize(path: String, tag: String? = nil) throws {
+    guard let manifestDigest = manifestDigest, let manifestSize = manifestSize else {
+      throw RuntimeError.Generic("no manifest was pushed")
+    }
+
+    let ociLayoutData = try JSONSerialization.data(withJSONObject: ["imageLayoutVersion": "1.0.0"])
+    try ociLayoutData.write(to: tmpDir.appendingPathComponent("oci-layout"))
+
+    var manifests: [[String: Any]] = []
+
+    let baseDescriptor: [String: Any] = [
+      "mediaType": ociManifestMediaType,
+      "digest": manifestDigest,
+      "size": manifestSize,
+    ]
+
+    let refs = manifestReferences.isEmpty
+      ? (tag.map { [$0] } ?? ["latest"])
+      : manifestReferences
+
+    for ref in refs {
+      var entry = baseDescriptor
+      entry["annotations"] = [
+        "org.opencontainers.image.ref.name": ref
+      ]
+      manifests.append(entry)
+    }
+
+    let index: [String: Any] = [
+      "schemaVersion": 2,
+      "manifests": manifests
+    ]
+
+    let indexData = try JSONSerialization.data(withJSONObject: index, options: [.prettyPrinted, .sortedKeys])
+    try indexData.write(to: tmpDir.appendingPathComponent("index.json"))
+
+    let absolutePath = URL(fileURLWithPath: path).path
+
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/usr/bin/tar")
+    process.arguments = ["-cf", absolutePath, "-C", tmpDir.path, "."]
+
+    let pipe = Pipe()
+    process.standardError = pipe
+
+    try process.run()
+    process.waitUntilExit()
+
+    if process.terminationStatus != 0 {
+      let errorData = pipe.fileHandleForReading.readDataToEndOfFile()
+      throw RuntimeError.Generic(
+        "creating OCI archive failed: \(String(data: errorData, encoding: .utf8) ?? "unknown error")"
+      )
+    }
+  }
+}

Sources/tart/OCI/Registry.swift

@@ -110,7 +110,7 @@ struct TokenResponse: Decodable, Authentication {
   }
 }
 
-class Registry {
+class Registry: BlobStorage {
   private let baseURL: URL
   let namespace: String
   let credentialsProviders: [CredentialsProvider]

Sources/tart/Root.swift

@@ -27,6 +27,7 @@ struct Root: AsyncParsableCommand {
       Export.self,
       Prune.self,
       Rename.self,
+      Save.self,
       Stop.self,
       Delete.self,
       FQN.self,

Sources/tart/VMDirectory+OCIArchive.swift

@@ -0,0 +1,48 @@
+import Foundation
+
+extension VMDirectory {
+  func saveToArchive(path: String, concurrency: UInt, labels: [String: String] = [:], tag: String? = nil) async throws {
+    let archive = try OCIArchiveWriter()
+
+    var layers = [OCIManifestLayer]()
+
+    let config = try VMConfig(fromURL: configURL)
+    var labels = labels
+    labels[diskFormatLabel] = config.diskFormat.rawValue
+    let configJSON = try JSONEncoder().encode(config)
+    defaultLogger.appendNewLine("saving config...")
+    let configDigest = try await archive.pushBlob(fromData: configJSON, chunkSizeMb: 0, digest: nil)
+    layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest))
+
+    let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64
+    defaultLogger.appendNewLine("saving disk... this will take a while...")
+    let progress = Progress(totalUnitCount: diskSize)
+    ProgressObserver(progress).log(defaultLogger)
+
+    layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: archive, chunkSizeMb: 0, concurrency: concurrency, progress: progress))
+
+    defaultLogger.appendNewLine("saving NVRAM...")
+    let nvram = try FileHandle(forReadingFrom: nvramURL).readToEnd()!
+    let nvramDigest = try await archive.pushBlob(fromData: nvram, chunkSizeMb: 0, digest: nil)
+    layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest))
+
+    let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels)
+    let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON()
+    let ociConfigDigest = try await archive.pushBlob(fromData: ociConfigJSON, chunkSizeMb: 0, digest: nil)
+
+    let manifest = OCIManifest(
+      config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
+      layers: layers,
+      uncompressedDiskSize: UInt64(diskSize),
+      uploadDate: Date()
+    )
+
+    let tagRef = tag ?? "latest"
+    defaultLogger.appendNewLine("saving manifest...")
+    _ = try await archive.pushManifest(reference: tagRef, manifest: manifest)
+
+    try archive.finalize(path: path, tag: tagRef)
+
+    defaultLogger.appendNewLine("saved to \(path)")
+  }
+}