From 4ce8a115f703e04cda07519ce879817f0d7bed3a Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 12 Aug 2026 12:10:28 -0700 Subject: [PATCH] Add OCI transport and base clone with DiskImageKit (#1304) * Add OCI transport and base clone with DiskImageKit * Address stacked OCI pull review feedback * Stream file digest hashing * Lock frozen overlays during push --- Sources/tart/Commands/Clone.swift | 57 ++- Sources/tart/Commands/Get.swift | 3 +- Sources/tart/Commands/Import.swift | 5 + Sources/tart/Commands/List.swift | 4 +- Sources/tart/Commands/Push.swift | 11 +- Sources/tart/Commands/Run.swift | 61 ++- Sources/tart/Commands/Set.swift | 8 + Sources/tart/ContentStore.swift | 52 ++- Sources/tart/DiskImageStack.swift | 114 +++-- Sources/tart/OCI/Digest.swift | 52 ++- Sources/tart/OCI/Layerizer/Disk.swift | 2 +- Sources/tart/OCI/Layerizer/DiskV2.swift | 11 +- Sources/tart/VM.swift | 35 +- Sources/tart/VMDirectory+Archive.swift | 4 + Sources/tart/VMDirectory+DiskImageStack.swift | 125 ++++++ Sources/tart/VMDirectory+OCI.swift | 280 +++++++++++-- Sources/tart/VMDirectory.swift | 102 ++++- Sources/tart/VMStorageOCI.swift | 314 +++++++++++++- Tests/TartTests/CommandBehaviorTests.swift | 135 ++++++ Tests/TartTests/ContentStoreTests.swift | 104 +++++ Tests/TartTests/DigestTests.swift | 31 ++ Tests/TartTests/DiskImageStackTests.swift | 89 ++-- Tests/TartTests/LayerizerTests.swift | 9 +- Tests/TartTests/OCIManifestTests.swift | 2 +- .../VMDirectoryDiskImageStackTests.swift | 185 +++++++++ Tests/TartTests/VMDirectoryLayoutTests.swift | 81 +++- Tests/TartTests/VMStorageOCITests.swift | 391 ++++++++++++++++++ 27 files changed, 2054 insertions(+), 213 deletions(-) create mode 100644 Sources/tart/VMDirectory+DiskImageStack.swift create mode 100644 Tests/TartTests/CommandBehaviorTests.swift create mode 100644 Tests/TartTests/VMDirectoryDiskImageStackTests.swift create mode 100644 Tests/TartTests/VMStorageOCITests.swift diff --git a/Sources/tart/Commands/Clone.swift b/Sources/tart/Commands/Clone.swift index b6497e2..62cc75c 100644 --- a/Sources/tart/Commands/Clone.swift +++ b/Sources/tart/Commands/Clone.swift @@ -31,6 +31,9 @@ struct Clone: AsyncParsableCommand { @Flag(help: .hidden) var deduplicate: Bool = false + @Flag(help: "create a stacked disk that uses the source image as an immutable base") + var stacked: Bool = false + @Option(help: ArgumentHelp("limit automatic pruning to n gigabytes", valueName: "n")) var pruneLimit: UInt = 100 @@ -47,8 +50,15 @@ struct Clone: AsyncParsableCommand { func run() async throws { let ociStorage = try VMStorageOCI() let localStorage = try VMStorageLocal() + let remoteName = try? RemoteName(sourceName) - if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) { + if stacked { + guard remoteName != nil else { + throw ValidationError("--stacked requires a remote image") + } + } + + if let remoteName, try !ociStorage.hasUsableCachedImageForClone(remoteName, requireManifest: stacked) { // Pull the VM in case it's OCI-based and doesn't exist locally yet let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure) try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate) @@ -66,9 +76,28 @@ struct Clone: AsyncParsableCommand { let lock = try FileLock(lockURL: Config().tartHomeDir) try lock.lock() + let sourceState = try sourceVM.state() let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress()) - && sourceVM.state() != .Suspended - try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC) + && sourceState != .Suspended + + if stacked { + guard sourceVM.isStandalone else { + throw ValidationError("--stacked cannot use an image that already has a stacked disk") + } + guard try VMConfig(fromURL: sourceVM.configURL).os == .darwin else { + throw ValidationError("--stacked currently supports only macOS images") + } + try sourceVM.cloneAsStackedBase(to: tmpVMDir, generateMAC: generateMAC) + } else if sourceVM.isStackedCachedImage { + try sourceVM.cloneStacked(to: tmpVMDir, copyWritableOverlay: false, generateMAC: generateMAC) + } else if sourceVM.isStackedVM { + guard sourceState == .Stopped else { + throw RuntimeError.VMConfigurationError("VM \"\(sourceName)\" must be stopped before cloning") + } + try sourceVM.cloneStacked(to: tmpVMDir, copyWritableOverlay: true, generateMAC: generateMAC) + } else { + try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC) + } try localStorage.move(newName, from: tmpVMDir) @@ -78,11 +107,23 @@ struct Clone: AsyncParsableCommand { // is not actually claiming new space until the VM is started and it writes something to disk. // // So, once we clone the VM let's try to claim the rest of space for the VM to run without errors. - let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes() - // Avoid reclaiming an excessive amount of disk space. - let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024) - if reclaimBytes > 0 { - try Prune.reclaimIfNeeded(UInt64(reclaimBytes), sourceVM) + if sourceVM.isStandalone { + let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes() + // Avoid reclaiming an excessive amount of disk space. + let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024) + if reclaimBytes > 0 { + try Prune.reclaimIfNeeded(UInt64(reclaimBytes), sourceVM) + } + } else if sourceVM.isStackedVM || sourceVM.isStackedCachedImage { + let clonedVM = try localStorage.open(newName) + // A stacked clone owns only its writable overlay locally, but that + // overlay may grow to the full guest-visible disk block layout at + // runtime. Reclaim against the clone so it is not pruned itself. + let unallocatedBytes = try clonedVM.diskSizeBytes() - clonedVM.allocatedSizeBytes() + let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024) + if reclaimBytes > 0 { + try Prune.reclaimIfNeeded(UInt64(reclaimBytes), clonedVM) + } } }, onCancel: { try? FileManager.default.removeItem(at: tmpVMDir.baseURL) diff --git a/Sources/tart/Commands/Get.swift b/Sources/tart/Commands/Get.swift index c006a6d..594f0a4 100644 --- a/Sources/tart/Commands/Get.swift +++ b/Sources/tart/Commands/Get.swift @@ -31,7 +31,7 @@ struct Get: AsyncParsableCommand { OS: vmConfig.os, CPU: vmConfig.cpuCount, Memory: memorySizeInMb, - Disk: HumanReadableByteCount(try vmDir.sizeBytes()) { $0 / 1000 / 1000 / 1000 }, + Disk: HumanReadableByteCount(try vmDir.diskSizeBytes()) { $0 / 1000 / 1000 / 1000 }, DiskFormat: vmConfig.diskFormat.rawValue, Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { String(format: "%.3f", Float($0) / 1000 / 1000 / 1000) @@ -40,7 +40,6 @@ struct Get: AsyncParsableCommand { Running: try vmDir.running(), State: try vmDir.state().rawValue ) - print(format.renderSingle(info)) } } diff --git a/Sources/tart/Commands/Import.swift b/Sources/tart/Commands/Import.swift index edb0253..7fc955c 100644 --- a/Sources/tart/Commands/Import.swift +++ b/Sources/tart/Commands/Import.swift @@ -31,6 +31,11 @@ struct Import: AsyncParsableCommand { print("importing...") try tmpVMDir.importFromArchive(path: path) + if tmpVMDir.isStackedVM || tmpVMDir.isStackedCachedImage { + try? FileManager.default.removeItem(at: tmpVMDir.baseURL) + throw RuntimeError.ImportFailed("importing stacked VMs is not supported yet") + } + try await withTaskCancellationHandler(operation: { // Acquire a global lock let lock = try FileLock(lockURL: Config().tartHomeDir) diff --git a/Sources/tart/Commands/List.swift b/Sources/tart/Commands/List.swift index 86311e5..28e67bf 100644 --- a/Sources/tart/Commands/List.swift +++ b/Sources/tart/Commands/List.swift @@ -42,7 +42,7 @@ struct List: AsyncParsableCommand { try VMInfo( Source: "local", Name: name, - Disk: HumanReadableByteCount(try vmDir.sizeBytes()) { $0 / 1000 / 1000 / 1000 }, + Disk: HumanReadableByteCount(try vmDir.diskSizeBytes()) { $0 / 1000 / 1000 / 1000 }, Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { $0 / 1000 / 1000 / 1000 }, Accessed: formatAccessDate(try vmDir.accessDate()), Running: vmDir.running(), @@ -56,7 +56,7 @@ struct List: AsyncParsableCommand { try VMInfo( Source: "OCI", Name: name, - Disk: HumanReadableByteCount(try vmDir.sizeBytes()) { $0 / 1000 / 1000 / 1000 }, + Disk: HumanReadableByteCount(try vmDir.diskSizeBytes()) { $0 / 1000 / 1000 / 1000 }, Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { $0 / 1000 / 1000 / 1000 }, Accessed: formatAccessDate(try vmDir.accessDate()), Running: vmDir.running(), diff --git a/Sources/tart/Commands/Push.swift b/Sources/tart/Commands/Push.swift index ce67bdd..532681d 100644 --- a/Sources/tart/Commands/Push.swift +++ b/Sources/tart/Commands/Push.swift @@ -69,7 +69,7 @@ struct Push: AsyncParsableCommand { let references = remoteNamesForRegistry.map{ $0.reference.value } let pushedRemoteName: RemoteName - // If we're pushing a local OCI VM, check if points to an already existing registry manifest + // If we're pushing a cached remote image, check if it points to an existing registry manifest // and if so, only upload manifests (without config, disk and NVRAM) to the user-specified references if let remoteName = try? RemoteName(localName) { pushedRemoteName = try await lightweightPushToRegistry( @@ -78,17 +78,18 @@ struct Push: AsyncParsableCommand { references: references ) } else { - pushedRemoteName = try await localVMDir.pushToRegistry( + let pushedImage = try await localVMDir.pushToRegistry( registry: registry, references: references, chunkSizeMb: chunkSize, concurrency: concurrency, labels: parseLabels() ) + pushedRemoteName = pushedImage.name + // Populate the local cache (if requested) if populateCache { - let expectedPushedVMDir = try ociStorage.create(pushedRemoteName) - try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false) + try ociStorage.populate(pushedImage.name, from: localVMDir, manifest: pushedImage.manifest) } } @@ -102,7 +103,7 @@ struct Push: AsyncParsableCommand { } func lightweightPushToRegistry(registry: Registry, remoteName: RemoteName, references: [String]) async throws -> RemoteName { - // Is the local OCI VM already present in the registry? + // Is the cached remote image already present in the registry? let digest = try VMStorageOCI().digest(remoteName) let (remoteManifest, _) = try await registry.pullManifest(reference: digest) diff --git a/Sources/tart/Commands/Run.swift b/Sources/tart/Commands/Run.swift index ad22bf1..e39a716 100644 --- a/Sources/tart/Commands/Run.swift +++ b/Sources/tart/Commands/Run.swift @@ -455,10 +455,15 @@ struct Run: AsyncParsableCommand { let provisioning = try provisioningOpts.map { try GuestProvisioningOptions($0) } #endif + // Keep these values alive while the VM runs. Some additional disks own a + // lock that protects their temporary backing files from Config.gc(). + let additionalDisks = try additionalDisks() + defer { withExtendedLifetime(additionalDisks) {} } + vm = try VM( vmDir: vmDir, network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(), - additionalStorageDevices: try additionalDiskAttachments(), + additionalStorageDevices: additionalDisks.map(\.configuration), directorySharingDevices: directoryShares() + rosettaDirectoryShare(), serialPorts: serialPorts, suspendable: suspendable, @@ -727,9 +732,9 @@ struct Run: AsyncParsableCommand { } } - func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] { + func additionalDisks() throws -> [AdditionalDisk] { try disk.map { - try AdditionalDisk(parseFrom: $0).configuration + try AdditionalDisk(parseFrom: $0) } } @@ -952,14 +957,32 @@ struct VMView: NSViewRepresentable { struct AdditionalDisk { let configuration: VZStorageDeviceConfiguration + // Retained for as long as the additional disk is attached, so Config.gc() + // cannot remove a temporary backing file or stacked-disk directory. + private let temporaryDiskLock: FileLock? init(parseFrom: String) throws { let (diskPath, readOnly, syncModeRaw, cachingModeRaw) = Self.parseOptions(parseFrom) - self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw, cachingModeRaw: cachingModeRaw) + self = try Self.craft( + diskPath, + readOnly: readOnly, + syncModeRaw: syncModeRaw, + cachingModeRaw: cachingModeRaw + ) } - static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String, cachingModeRaw: String) throws -> VZStorageDeviceConfiguration { + private init(configuration: VZStorageDeviceConfiguration, temporaryDiskLock: FileLock? = nil) { + self.configuration = configuration + self.temporaryDiskLock = temporaryDiskLock + } + + private static func craft( + _ diskPath: String, + readOnly diskReadOnly: Bool, + syncModeRaw: String, + cachingModeRaw: String + ) throws -> AdditionalDisk { let diskURL = URL(string: diskPath) if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) { @@ -974,7 +997,7 @@ struct AdditionalDisk { synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw) ) - return VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment) + return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment)) } // Expand the tilde (~) since at this point we're dealing with a local path, @@ -1005,13 +1028,33 @@ struct AdditionalDisk { let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true), readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)) - return VZVirtioBlockDeviceConfiguration(attachment: blockAttachment) + return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: blockAttachment)) } // Support remote VM names in --disk command-line argument if let remoteName = try? RemoteName(diskPath) { let vmDir = try VMStorageOCI().open(remoteName) + if vmDir.isStackedCachedImage { + // A cached stacked image has no writable top overlay. Create one in a + // disposable directory for this additional-disk attachment. + let temporaryVMDir = try VMDirectory.temporary() + try FileManager.default.copyItem(at: vmDir.configURL, to: temporaryVMDir.configURL) + try FileManager.default.copyItem(at: vmDir.nvramURL, to: temporaryVMDir.nvramURL) + try FileManager.default.copyItem(at: vmDir.manifestURL, to: temporaryVMDir.manifestURL) + let lock = try FileLock(lockURL: temporaryVMDir.baseURL) + try lock.lock() + let stack = try temporaryVMDir.diskImageStack() + try stack.createWritableOverlay() + let attachment = try stack.makeAttachment( + readOnly: diskReadOnly, + cachingMode: try VZDiskImageCachingMode(cachingModeRaw) ?? .automatic, + synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw) + ) + + return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: attachment), temporaryDiskLock: lock) + } + // Unfortunately, VZDiskImageStorageDeviceAttachment does not support // FileHandle, so we can't easily clone the disk, open it and unlink(2) // to simplify the garbage collection, so use an intermediate directory. @@ -1024,7 +1067,7 @@ struct AdditionalDisk { let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: clonedDiskURL, readOnly: diskReadOnly) - return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment) + return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment), temporaryDiskLock: lock) } // Error out if the disk is locked by the host (e.g. it was mounted in Finder), @@ -1040,7 +1083,7 @@ struct AdditionalDisk { synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw) ) - return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment) + return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)) } static func parseOptions(_ parseFrom: String) -> (String, Bool, String, String) { diff --git a/Sources/tart/Commands/Set.swift b/Sources/tart/Commands/Set.swift index 384fda8..39025f8 100644 --- a/Sources/tart/Commands/Set.swift +++ b/Sources/tart/Commands/Set.swift @@ -39,6 +39,14 @@ struct Set: AsyncParsableCommand { func run() async throws { let vmDir = try VMStorageLocal().open(name) + + // Replacing disk.img would leave a stacked VM with both disk.img and + // overlay.asif, which is not a supported local layout. Reject before + // saving any other requested configuration changes. + if disk != nil, vmDir.isStackedVM { + throw ValidationError("--disk is not supported for VMs with a stacked disk") + } + var vmConfig = try VMConfig(fromURL: vmDir.configURL) if let cpu = cpu { diff --git a/Sources/tart/ContentStore.swift b/Sources/tart/ContentStore.swift index 4402ae9..7608e61 100644 --- a/Sources/tart/ContentStore.swift +++ b/Sources/tart/ContentStore.swift @@ -39,15 +39,48 @@ struct ContentStore { return targetURL.deletingLastPathComponent().appendingPathComponent(".\(UUID().uuidString).tmp") } - /// Returns a validated cache hit. Corrupt files are treated as misses so a - /// later pull can safely rebuild them. - func existingContentURL(for contentDigest: String) throws -> URL? { + /// Returns a stable staging path so an interrupted registry pull can resume + /// reconstructing this content entry on a later attempt. + func resumableContentURL(for contentDigest: String) throws -> URL { + let targetURL = try contentURL(for: contentDigest) + + return targetURL.deletingLastPathComponent().appendingPathComponent(".\(targetURL.lastPathComponent).partial") + } + + /// Returns a stable lock file for serializing reconstruction of one content + /// entry. The file is intentionally retained; flock state lives on the file + /// descriptor and disappears when the owning process exits. + func lockURL(for contentDigest: String) throws -> URL { + let targetURL = try contentURL(for: contentDigest) + + let lockURL = targetURL.deletingLastPathComponent().appendingPathComponent(".\(targetURL.lastPathComponent).lock") + if !FileManager.default.fileExists(atPath: lockURL.path) { + _ = FileManager.default.createFile(atPath: lockURL.path, contents: nil) + } + + return lockURL + } + + /// Returns a digest-addressed entry without rereading it. Pull verifies + /// content hashes before accepting a cache hit; clone only needs a cheap + /// structural check, like Tart's existing disk.img path. + func contentURLIfPresent(for contentDigest: String) throws -> URL? { let url = try contentURL(for: contentDigest) guard FileManager.default.fileExists(atPath: url.path) else { return nil } + return url + } + + /// Returns a validated cache hit. Corrupt files are treated as misses so a + /// later pull can safely rebuild them. + func existingContentURL(for contentDigest: String) throws -> URL? { + guard let url = try contentURLIfPresent(for: contentDigest) else { + return nil + } + guard try Digest.hash(url) == contentDigest else { return nil } @@ -57,7 +90,8 @@ struct ContentStore { /// Move a fully reconstructed temporary file into the cache after verifying /// its semantic identity. The caller should create the temporary file with - /// `temporaryContentURL(for:)` so rename stays on the same filesystem. + /// temporaryContentURL(for:) or resumableContentURL(for:) so rename stays on + /// the same filesystem. func install(_ temporaryURL: URL, contentDigest: String) throws -> URL { let actualDigest = try Digest.hash(temporaryURL) guard actualDigest == contentDigest else { @@ -65,14 +99,20 @@ struct ContentStore { } let targetURL = try contentURL(for: contentDigest) + let lock = try FileLock(lockURL: baseURL) + try lock.lock() + defer { try? lock.unlock() } if let existingURL = try existingContentURL(for: contentDigest) { try? FileManager.default.removeItem(at: temporaryURL) return existingURL } - try? FileManager.default.removeItem(at: targetURL) - try FileManager.default.moveItem(at: temporaryURL, to: targetURL) + if FileManager.default.fileExists(atPath: targetURL.path) { + _ = try FileManager.default.replaceItemAt(targetURL, withItemAt: temporaryURL) + } else { + try FileManager.default.moveItem(at: temporaryURL, to: targetURL) + } return targetURL } diff --git a/Sources/tart/DiskImageStack.swift b/Sources/tart/DiskImageStack.swift index 27543a6..2a60e76 100644 --- a/Sources/tart/DiskImageStack.swift +++ b/Sources/tart/DiskImageStack.swift @@ -5,20 +5,17 @@ import Virtualization import DiskImageKit #endif -/// One immutable complete disk file used by a stacked disk. -/// -/// This is a reconstructed base disk or published ASIF overlay, not an OCI -/// layer or an individual Tart disk chunk. -struct DiskImageFile { - let url: URL - let contentDigest: String +/// The logical block layout exposed by a disk image. +struct DiskImageBlockLayout { + let blockSize: UInt64 + let blockCount: UInt64 } enum DiskImageStackError: Error, Equatable, CustomStringConvertible { case unavailable case writableOverlayAlreadyExists(URL) case writableOverlayMissing(URL) - case invalidGeometry(String) + case invalidBlockLayout(String) case invalidDiskImage(URL, String) var description: String { @@ -29,7 +26,7 @@ enum DiskImageStackError: Error, Equatable, CustomStringConvertible { "writable overlay already exists: \(url.path)" case .writableOverlayMissing(let url): "writable overlay is missing: \(url.path)" - case .invalidGeometry(let reason): + case .invalidBlockLayout(let reason): reason case .invalidDiskImage(let url, let reason): "\(reason): \(url.path)" @@ -38,16 +35,66 @@ enum DiskImageStackError: Error, Equatable, CustomStringConvertible { } struct DiskImageStack { - /// DiskImageKit-ready paths and geometry after Tart disk chunks have been + /// DiskImageKit-ready paths and block layout after Tart disk chunks have been /// reconstructed into complete immutable files. The writable overlay stays /// private to one VM. - let base: DiskImageFile + let baseURL: URL let baseFormat: DiskImageFormat - let overlays: [DiskImageFile] + let immutableOverlayURLs: [URL] let writableOverlayURL: URL let blockSize: UInt64 let blockCount: UInt64 + /// Reads a disk image's current block layout without resolving or validating a + /// whole stack. This is used for the VM's private writable overlay, whose + /// size may be newer than the pinned immutable parent manifest. + static func diskImageBlockLayout(at url: URL) throws -> DiskImageBlockLayout { + #if canImport(DiskImageKit) + if #available(macOS 27.0, *) { + let image = try DiskImage(opening: .open(url: url, mode: .readOnly)) + return DiskImageBlockLayout( + blockSize: UInt64(image.blockSize.rawValue), + blockCount: UInt64(image.blockCount) + ) + } + #endif + + throw DiskImageStackError.unavailable + } + + static func baseBlockLayout( + at url: URL, + expectedFormat: DiskImageFormat + ) throws -> DiskImageBlockLayout { + #if canImport(DiskImageKit) + if #available(macOS 27.0, *) { + let image = try DiskImage(opening: .open(url: url, mode: .readOnly)) + let matchesFormat = switch expectedFormat { + case .raw: + image.format == .raw + case .asif: + image.format == .asif + } + guard matchesFormat else { + throw DiskImageStackError.invalidDiskImage(url, "base disk format does not match") + } + guard image.layerType == nil, image.parentUUID == nil else { + throw DiskImageStackError.invalidDiskImage(url, "base disk must not be an overlay") + } + if expectedFormat == .asif && image.layerUUID == nil { + throw DiskImageStackError.invalidDiskImage(url, "ASIF base disk is missing a UUID") + } + + return DiskImageBlockLayout( + blockSize: UInt64(image.blockSize.rawValue), + blockCount: UInt64(image.blockCount) + ) + } + #endif + + throw DiskImageStackError.unavailable + } + func createWritableOverlay() throws { #if canImport(DiskImageKit) if #available(macOS 27.0, *) { @@ -68,12 +115,14 @@ struct DiskImageStack { } func makeAttachment( + readOnly: Bool = false, cachingMode: VZDiskImageCachingMode = .automatic, synchronizationMode: VZDiskImageSynchronizationMode = .full - ) throws -> VZDiskImageStorageDeviceAttachment { + ) throws -> VZStorageDeviceAttachment { #if canImport(DiskImageKit) if #available(macOS 27.0, *) { return try attachmentWithDiskImageKit( + readOnly: readOnly, cachingMode: cachingMode, synchronizationMode: synchronizationMode ) @@ -108,6 +157,7 @@ struct DiskImageStack { @available(macOS 27.0, *) private func attachmentWithDiskImageKit( + readOnly: Bool, cachingMode: VZDiskImageCachingMode, synchronizationMode: VZDiskImageSynchronizationMode ) throws -> VZDiskImageStorageDeviceAttachment { @@ -118,7 +168,7 @@ struct DiskImageStack { let parent = try validatedParentImage() let writableOverlay = try openOverlay( at: writableOverlayURL, - mode: .readWrite + mode: readOnly ? .readOnly : .readWrite ) let stackedImage = try append(writableOverlay, to: parent, at: writableOverlayURL) try validateAppendedOverlay(stackedImage, at: writableOverlayURL) @@ -133,7 +183,7 @@ struct DiskImageStack { @available(macOS 27.0, *) private func growWritableOverlayWithDiskImageKit(toBlockCount blockCount: UInt64) throws { guard blockCount > 0, let desiredBlockCount = Int(exactly: blockCount) else { - throw DiskImageStackError.invalidGeometry("invalid stacked disk block count \(blockCount)") + throw DiskImageStackError.invalidBlockLayout("invalid stacked disk block count \(blockCount)") } let parent = try validatedParentImage() @@ -160,31 +210,29 @@ struct DiskImageStack { private func validatedParentImage() throws -> DiskImage { let expectedBlockSize = try diskImageBlockSize(blockSize) guard blockCount > 0, let expectedBlockCount = Int(exactly: blockCount) else { - throw DiskImageStackError.invalidGeometry("invalid stacked disk block count \(blockCount)") + throw DiskImageStackError.invalidBlockLayout("invalid stacked disk block count \(blockCount)") } - try verifyContentDigest(base) - let baseImage = try DiskImage(opening: .open(url: base.url, mode: .readOnly)) - try validateBase(baseImage, at: base.url, expectedFormat: baseFormat) + let baseImage = try DiskImage(opening: .open(url: baseURL, mode: .readOnly)) + try validateBase(baseImage, at: baseURL, expectedFormat: baseFormat) var image = baseImage - for overlay in overlays { + for overlayURL in immutableOverlayURLs { let openedOverlay = try openOverlay( - at: overlay.url, - expectedDigest: overlay.contentDigest, + at: overlayURL, mode: .readOnly ) - let stackedImage = try append(openedOverlay, to: image, at: overlay.url) - try validateAppendedOverlay(stackedImage, at: overlay.url) + let stackedImage = try append(openedOverlay, to: image, at: overlayURL) + try validateAppendedOverlay(stackedImage, at: overlayURL) image = stackedImage } guard image.blockSize == expectedBlockSize else { - throw DiskImageStackError.invalidGeometry("immutable disk stack does not match manifest block size") + throw DiskImageStackError.invalidBlockLayout("immutable disk stack does not match manifest block size") } guard image.blockCount == expectedBlockCount else { - throw DiskImageStackError.invalidGeometry("immutable disk stack does not match manifest block count") + throw DiskImageStackError.invalidBlockLayout("immutable disk stack does not match manifest block count") } return image @@ -216,13 +264,8 @@ struct DiskImageStack { @available(macOS 27.0, *) private func openOverlay( at url: URL, - expectedDigest: String? = nil, mode: OpenConfiguration.Mode ) throws -> DiskImage { - if let expectedDigest { - try verifyContentDigest(DiskImageFile(url: url, contentDigest: expectedDigest)) - } - let image = try DiskImage(opening: .open(url: url, mode: mode)) guard image.format == .asif else { throw DiskImageStackError.invalidDiskImage(url, "overlay must use ASIF format") @@ -247,17 +290,10 @@ struct DiskImageStack { } } - @available(macOS 27.0, *) - private func verifyContentDigest(_ diskImage: DiskImageFile) throws { - guard try Digest.hash(diskImage.url) == diskImage.contentDigest else { - throw DiskImageStackError.invalidDiskImage(diskImage.url, "disk image content digest does not match") - } - } - @available(macOS 27.0, *) private func diskImageBlockSize(_ value: UInt64) throws -> DiskImage.BlockSize { guard let intValue = Int(exactly: value), let blockSize = DiskImage.BlockSize(rawValue: intValue) else { - throw DiskImageStackError.invalidGeometry("unsupported stacked disk block size \(value)") + throw DiskImageStackError.invalidBlockLayout("unsupported stacked disk block size \(value)") } return blockSize diff --git a/Sources/tart/OCI/Digest.swift b/Sources/tart/OCI/Digest.swift index 51c4201..5455008 100644 --- a/Sources/tart/OCI/Digest.swift +++ b/Sources/tart/OCI/Digest.swift @@ -7,6 +7,8 @@ enum DigestError: Error { } class Digest { + private static let fileBufferSize = 4 * 1024 * 1024 + var hash: SHA256 = SHA256() func update(_ data: Data) { @@ -22,7 +24,10 @@ class Digest { } static func hash(_ url: URL) throws -> String { - hash(try Data(contentsOf: url)) + let file = try FileHandle(forReadingFrom: url) + defer { try? file.close() } + + return try hashContents(from: file) } static func hash(_ url: URL, offset: UInt64, size: UInt64) throws -> String { @@ -36,20 +41,53 @@ class Digest { throw DigestError.InvalidOffset } - if (offset + size) > fileSize { + if size > fileSize - offset { throw DigestError.InvalidSize } - // Read a chunk of size ``size`` at offset ``offset`` - // and calculate it's digest + // Read the requested range incrementally and calculate its digest. let fh = try FileHandle(forReadingFrom: url) - defer { try! fh.close() } + defer { try? fh.close() } try fh.seek(toOffset: offset) - let data = try fh.read(upToCount: Int(size))! + return try hashContents(from: fh, size: size) + } - return hash(data) + /// Streams a file into SHA-256 while keeping Foundation's temporary read + /// buffers scoped to one chunk. + private static func hashContents(from file: FileHandle, size: UInt64? = nil) throws -> String { + let digest = Digest() + var remaining = size + + while remaining.map({ $0 > 0 }) ?? true { + let didRead = try autoreleasepool { () throws -> Bool in + let count = remaining.map { + Int(min(UInt64(fileBufferSize), $0)) + } ?? fileBufferSize + + guard let data = try file.read(upToCount: count), !data.isEmpty else { + if remaining != nil { + throw DigestError.InvalidSize + } + + return false + } + + digest.update(data) + if let bytesRemaining = remaining { + remaining = bytesRemaining - UInt64(data.count) + } + + return true + } + + if !didRead { + break + } + } + + return digest.finalize() } } diff --git a/Sources/tart/OCI/Layerizer/Disk.swift b/Sources/tart/OCI/Layerizer/Disk.swift index 051f543..c03b852 100644 --- a/Sources/tart/OCI/Layerizer/Disk.swift +++ b/Sources/tart/OCI/Layerizer/Disk.swift @@ -1,6 +1,6 @@ import Foundation protocol Disk { - static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] + static func push(diskURL: URL, mediaType: String, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws } diff --git a/Sources/tart/OCI/Layerizer/DiskV2.swift b/Sources/tart/OCI/Layerizer/DiskV2.swift index 2239985..b599240 100644 --- a/Sources/tart/OCI/Layerizer/DiskV2.swift +++ b/Sources/tart/OCI/Layerizer/DiskV2.swift @@ -22,7 +22,14 @@ class DiskV2: Disk { private static let holeGranularityBytes = 4 * 1024 * 1024 private static let zeroChunk = Data(count: holeGranularityBytes) - static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] { + static func push( + diskURL: URL, + mediaType: String, + registry: Registry, + chunkSizeMb: Int, + concurrency: UInt, + progress: Progress + ) async throws -> [OCIManifestLayer] { var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = [] // Open the disk file @@ -63,7 +70,7 @@ class DiskV2: Disk { progress.completedUnitCount += Int64(data.count) return (index, OCIManifestLayer( - mediaType: diskV2MediaType, + mediaType: mediaType, size: compressedData.count, digest: compressedDataDigest, uncompressedSize: UInt64(data.count), diff --git a/Sources/tart/VM.swift b/Sources/tart/VM.swift index 77ef457..a2fb948 100644 --- a/Sources/tart/VM.swift +++ b/Sources/tart/VM.swift @@ -64,7 +64,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject { // Initialize the virtual machine and its configuration self.network = network - configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, + configuration = try Self.craftConfiguration(vmDir: vmDir, nvramURL: vmDir.nvramURL, vmConfig: config, network: network, additionalStorageDevices: additionalStorageDevices, directorySharingDevices: directorySharingDevices, @@ -196,7 +196,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject { // Initialize the virtual machine and its configuration self.network = network - configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL, + configuration = try Self.craftConfiguration(vmDir: vmDir, + nvramURL: vmDir.nvramURL, vmConfig: config, network: network, additionalStorageDevices: additionalStorageDevices, directorySharingDevices: directorySharingDevices, @@ -312,7 +313,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject { } static func craftConfiguration( - diskURL: URL, + vmDir: VMDirectory, nvramURL: URL, vmConfig: VMConfig, network: Network = NetworkShared(), @@ -404,15 +405,25 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject { } // Storage - let attachment = try VZDiskImageStorageDeviceAttachment( - url: diskURL, - readOnly: false, - // When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1] - // - // [1]: https://github.com/cirruslabs/tart/pull/675 - cachingMode: caching ?? (vmConfig.os == .linux ? .cached : .automatic), - synchronizationMode: sync - ) + // When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1] + // + // [1]: https://github.com/cirruslabs/tart/pull/675 + let cachingMode = caching ?? (vmConfig.os == .linux ? .cached : .automatic) + let attachment: VZStorageDeviceAttachment + if vmDir.isStackedVM { + attachment = try vmDir.diskImageStack().makeAttachment( + readOnly: false, + cachingMode: cachingMode, + synchronizationMode: sync + ) + } else { + attachment = try VZDiskImageStorageDeviceAttachment( + url: vmDir.diskURL, + readOnly: false, + cachingMode: cachingMode, + synchronizationMode: sync + ) + } var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)] devices.append(contentsOf: additionalStorageDevices) diff --git a/Sources/tart/VMDirectory+Archive.swift b/Sources/tart/VMDirectory+Archive.swift index dc62aac..969a84b 100644 --- a/Sources/tart/VMDirectory+Archive.swift +++ b/Sources/tart/VMDirectory+Archive.swift @@ -10,6 +10,10 @@ fileprivate let permissions = FilePermissions(rawValue: 0o644) // [2]: https://developer.apple.com/documentation/compression/algorithm/lzfse extension VMDirectory { func exportToArchive(path: String) throws { + guard !isStackedVM && !isStackedCachedImage else { + throw RuntimeError.ExportFailed("exporting stacked VMs is not supported yet") + } + guard let fileStream = ArchiveByteStream.fileStream( path: FilePath(path), mode: .writeOnly, diff --git a/Sources/tart/VMDirectory+DiskImageStack.swift b/Sources/tart/VMDirectory+DiskImageStack.swift new file mode 100644 index 0000000..027672f --- /dev/null +++ b/Sources/tart/VMDirectory+DiskImageStack.swift @@ -0,0 +1,125 @@ +import Foundation + +extension VMDirectory { + func diskImageStack(contentStore providedStore: ContentStore? = nil) throws -> DiskImageStack { + let manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL)) + let base: TartDiskFileGroup + let overlays: [TartDiskFileGroup] + + switch try manifest.tartDiskRepresentation() { + case .flat(let pinnedBase) where pinnedBase.contentDigest != nil: + base = pinnedBase + overlays = [] + case .stacked(let stackedBase, let stackedOverlays): + base = stackedBase + overlays = stackedOverlays + default: + throw RuntimeError.VMConfigurationError("VM is missing its disk image metadata") + } + guard let blockSize = manifest.diskBlockSize(), + let blockCount = manifest.diskBlockCount() else { + throw DiskImageStackError.invalidBlockLayout("disk image metadata is missing block layout") + } + + let contentStore = try providedStore ?? ContentStore() + let baseURL = try diskImageURL(for: base, contentStore: contentStore) + let immutableOverlayURLs = try overlays.map { try diskImageURL(for: $0, contentStore: contentStore) } + let config = try VMConfig(fromURL: configURL) + + return DiskImageStack( + baseURL: baseURL, + baseFormat: config.diskFormat, + immutableOverlayURLs: immutableOverlayURLs, + writableOverlayURL: overlayURL, + blockSize: blockSize, + blockCount: blockCount + ) + } + + func cloneStacked( + to destination: VMDirectory, + copyWritableOverlay: Bool, + generateMAC: Bool, + contentStore: ContentStore? = nil + ) throws { + try FileManager.default.copyItem(at: configURL, to: destination.configURL) + try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL) + try FileManager.default.copyItem(at: manifestURL, to: destination.manifestURL) + + if copyWritableOverlay { + try FileManager.default.copyItem(at: overlayURL, to: destination.overlayURL) + } else { + try destination.diskImageStack(contentStore: contentStore).createWritableOverlay() + } + + if generateMAC { + try destination.regenerateMACAddress() + } + } + + func cloneAsStackedBase( + to destination: VMDirectory, + generateMAC: Bool, + contentStore providedStore: ContentStore? = nil + ) throws { + let config = try VMConfig(fromURL: configURL) + let blockLayout = try DiskImageStack.baseBlockLayout(at: diskURL, expectedFormat: config.diskFormat) + let contentDigest = try Digest.hash(diskURL) + let contentStore = try providedStore ?? ContentStore() + + if try contentStore.existingContentURL(for: contentDigest) == nil { + let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest) + do { + try FileManager.default.copyItem(at: diskURL, to: temporaryURL) + _ = try contentStore.install(temporaryURL, contentDigest: contentDigest) + } catch { + try? FileManager.default.removeItem(at: temporaryURL) + throw error + } + } + + var manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL)) + guard case .flat = try manifest.tartDiskRepresentation() else { + throw RuntimeError.VMConfigurationError("--stacked cannot use an image that already has a stacked disk") + } + + guard let firstDiskIndex = manifest.layers.firstIndex(where: { $0.mediaType == diskV2MediaType }) else { + throw OCIManifestValidationError.invalidLayout("manifest must contain at least one disk chunk") + } + + var baseAnnotations = manifest.layers[firstDiskIndex].annotations ?? [:] + baseAnnotations[diskFileContentDigestAnnotation] = contentDigest + manifest.layers[firstDiskIndex].annotations = baseAnnotations + let diskSize = blockLayout.blockSize.multipliedReportingOverflow(by: blockLayout.blockCount) + guard !diskSize.overflow else { + throw DiskImageStackError.invalidBlockLayout("stacked disk block layout overflows UInt64") + } + var annotations = manifest.annotations ?? [:] + annotations[diskBlockSizeAnnotation] = String(blockLayout.blockSize) + annotations[uncompressedDiskSizeAnnotation] = String(diskSize.partialValue) + manifest.annotations = annotations + + try FileManager.default.copyItem(at: configURL, to: destination.configURL) + try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL) + try manifest.toJSON().write(to: destination.manifestURL) + try destination.diskImageStack(contentStore: contentStore).createWritableOverlay() + + if generateMAC { + try destination.regenerateMACAddress() + } + } + + private func diskImageURL(for group: TartDiskFileGroup, contentStore: ContentStore) throws -> URL { + guard let contentDigest = group.contentDigest else { + throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest") + } + // Pull/install verifies immutable content before publishing it. Clone and + // run use the trusted content-addressed entry without rereading a possibly + // very large disk file, matching Tart's existing disk.img behavior. + guard let url = try contentStore.contentURLIfPresent(for: contentDigest) else { + throw RuntimeError.VMMissingFiles("VM is missing cached disk content \(contentDigest)") + } + + return url + } +} diff --git a/Sources/tart/VMDirectory+OCI.swift b/Sources/tart/VMDirectory+OCI.swift index 9610416..ed0ec64 100644 --- a/Sources/tart/VMDirectory+OCI.swift +++ b/Sources/tart/VMDirectory+OCI.swift @@ -6,7 +6,6 @@ let legacyDiskV1MediaType = "application/vnd.cirruslabs.tart.disk.v1" enum OCIError: Error { case ShouldBeExactlyOneLayer - case ShouldBeAtLeastOneLayer case FailedToCreateVmFile case LayerIsMissingUncompressedSizeAnnotation case LayerIsMissingUncompressedDigestAnnotation @@ -14,7 +13,7 @@ enum OCIError: Error { extension VMDirectory { func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws { - // Pull VM's config file layer and re-serialize it into a config file + // Pull VM's config file layer and store it as the local config file. let configLayers = manifest.layers.filter { $0.mediaType == configMediaType } @@ -30,17 +29,22 @@ extension VMDirectory { } try configFile.close() - // Pull VM's disk layers and decompress them into a disk file + // Pull VM's disk chunks and decompress them into complete disk files. if manifest.layers.contains(where: { $0.mediaType == legacyDiskV1MediaType }) { throw RuntimeError.Generic("Pulling OCI images with legacy disk media type \(legacyDiskV1MediaType) is no longer supported, please re-push the image using a current Tart version") } - let layers = manifest.layers.filter { $0.mediaType == diskV2MediaType } - if layers.isEmpty { - throw OCIError.ShouldBeAtLeastOneLayer + let diskRepresentation = try manifest.tartDiskRepresentation() + let diskChunks: [OCIManifestLayer] + + switch diskRepresentation { + case .flat(let base): + diskChunks = base.chunks + case .stacked(let base, let overlays): + diskChunks = base.chunks + overlays.flatMap(\.chunks) } - let diskCompressedSize = layers.map { Int64($0.size) }.reduce(0, +) + let diskCompressedSize = diskChunks.map { Int64($0.size) }.reduce(0, +) OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute( key: "compressed_disk_size_bytes", value: .int(Int(diskCompressedSize)) @@ -53,19 +57,42 @@ extension VMDirectory { ProgressObserver(progress).log(defaultLogger) do { - try await DiskV2.pull(registry: registry, diskLayers: layers, diskURL: diskURL, - concurrency: concurrency, progress: progress, - localLayerCache: localLayerCache, - deduplicate: deduplicate) + switch diskRepresentation { + case .flat(let base): + try await DiskV2.pull(registry: registry, diskLayers: base.chunks, diskURL: diskURL, + concurrency: concurrency, progress: progress, + localLayerCache: localLayerCache, + deduplicate: deduplicate) + + if deduplicate, let llc = localLayerCache { + // set custom attribute to remember deduplicated bytes + diskURL.setDeduplicatedBytes(llc.deduplicatedBytes) + } + case .stacked(let base, let overlays): + // The deterministic resumable directory may contain a partial + // disk.img from an interrupted pull while this tag was standalone. A + // cached stacked image must not retain that file or it is mistaken for + // a standalone VM after the pull is moved into cache. + if FileManager.default.fileExists(atPath: diskURL.path) { + try FileManager.default.removeItem(at: diskURL) + } + + let contentStore = try ContentStore() + + for group in [base] + overlays { + _ = try await pullDiskFile( + registry: registry, + group: group, + contentStore: contentStore, + concurrency: concurrency, + progress: progress + ) + } + } } catch let error where error is FilterError { throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)") } - if deduplicate, let llc = localLayerCache { - // set custom attribute to remember deduplicated bytes - diskURL.setDeduplicatedBytes(llc.deduplicatedBytes) - } - // Pull VM's NVRAM file layer and store it in an NVRAM file defaultLogger.appendNewLine("pulling NVRAM...") @@ -83,12 +110,47 @@ extension VMDirectory { try nvram.write(contentsOf: data) } try nvram.close() - - // Serialize VM's manifest to enable better deduplication on subsequent "tart pull"'s - try manifest.toJSON().write(to: manifestURL) } - func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, concurrency: UInt, labels: [String: String] = [:]) async throws -> RemoteName { + /// Reconstructs one complete immutable base disk or published ASIF overlay + /// from its Tart disk chunks, unless the shared content store already has a + /// verified copy. + private func pullDiskFile( + registry: Registry, + group: TartDiskFileGroup, + contentStore: ContentStore, + concurrency: UInt, + progress: Progress + ) async throws -> URL { + guard let contentDigest = group.contentDigest else { + throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest") + } + + // Pulls for the same semantic disk file share a stable resumable path so + // DiskV2 can resume after a transient failure. Serialize writers before + // rechecking the final entry to avoid racing on that shared path. + let lock = try FileLock(lockURL: contentStore.lockURL(for: contentDigest)) + try lock.lock() + defer { try? lock.unlock() } + + if let existingURL = try contentStore.existingContentURL(for: contentDigest) { + progress.completedUnitCount += group.chunks.reduce(0) { $0 + Int64($1.size) } + return existingURL + } + + let resumableURL = try contentStore.resumableContentURL(for: contentDigest) + try await DiskV2.pull( + registry: registry, + diskLayers: group.chunks, + diskURL: resumableURL, + concurrency: concurrency, + progress: progress + ) + + return try contentStore.install(resumableURL, contentDigest: contentDigest) + } + + func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, concurrency: UInt, labels: [String: String] = [:]) async throws -> (name: RemoteName, manifest: OCIManifest) { var layers = Array() // Read VM's config and push it as blob @@ -102,14 +164,12 @@ extension VMDirectory { let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb) layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest)) - // Compress the disk file as multiple chunks and push them as disk layers - let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64 - - defaultLogger.appendNewLine("pushing disk... this will take a while...") - let progress = Progress(totalUnitCount: diskSize) - ProgressObserver(progress).log(defaultLogger) - - layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress)) + let (diskLayers, diskAnnotations) = try await pushDiskLayers( + registry: registry, + chunkSizeMb: chunkSizeMb, + concurrency: concurrency + ) + layers.append(contentsOf: diskLayers) // Read VM's NVRAM and push it as blob defaultLogger.appendNewLine("pushing NVRAM...") @@ -122,12 +182,13 @@ extension VMDirectory { let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels) let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON() let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb) - let manifest = OCIManifest( + var manifest = OCIManifest( config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest), - layers: layers, - uncompressedDiskSize: UInt64(diskSize), - uploadDate: Date() + layers: layers ) + var annotations = diskAnnotations + annotations[uploadTimeAnnotation] = Date().toISO() + manifest.annotations = annotations // Manifest for reference in references { @@ -137,7 +198,160 @@ extension VMDirectory { } let pushedReference = Reference(digest: try manifest.digest()) - return RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference) + let name = RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference) + return (name, manifest) + } + + /// Builds the disk portion of the manifest. Registry transport is shared + /// for standalone and stacked VMs; only their local disk representation + /// determines which descriptors need to be uploaded or reused. + private func pushDiskLayers( + registry: Registry, + chunkSizeMb: Int, + concurrency: UInt + ) async throws -> ([OCIManifestLayer], [String: String]) { + guard isStackedVM else { + let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64 + defaultLogger.appendNewLine("pushing disk... this will take a while...") + let progress = Progress(totalUnitCount: diskSize) + ProgressObserver(progress).log(defaultLogger) + + let layers = try await DiskV2.push( + diskURL: diskURL, + mediaType: diskV2MediaType, + registry: registry, + chunkSizeMb: chunkSizeMb, + concurrency: concurrency, + progress: progress + ) + return (layers, [uncompressedDiskSizeAnnotation: String(diskSize)]) + } + + let localManifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL)) + let inheritedGroups: [TartDiskFileGroup] + switch try localManifest.tartDiskRepresentation() { + case .flat(let base) where base.contentDigest != nil: + inheritedGroups = [base] + case .stacked(let base, let overlays): + inheritedGroups = [base] + overlays + default: + throw RuntimeError.VMConfigurationError("stacked VM is missing a pinned disk stack") + } + + let contentStore = try ContentStore() + var layers: [OCIManifestLayer] = [] + for group in inheritedGroups { + layers.append(contentsOf: try await descriptorsForCachedDiskFile( + group, + contentStore: contentStore, + registry: registry, + chunkSizeMb: chunkSizeMb, + concurrency: concurrency + )) + } + + // Keep the snapshot out of startup GC while this potentially long push + // hashes, uploads, and inspects it. + let frozenOverlayDirectory = try VMDirectory.temporary() + let frozenOverlayLock = try FileLock(lockURL: frozenOverlayDirectory.baseURL) + try frozenOverlayLock.lock() + defer { + try? frozenOverlayLock.unlock() + try? FileManager.default.removeItem(at: frozenOverlayDirectory.baseURL) + } + + let frozenOverlayURL = frozenOverlayDirectory.baseURL.appendingPathComponent("overlay.asif") + try FileManager.default.copyItem(at: overlayURL, to: frozenOverlayURL) + + let overlaySize = try FileManager.default.attributesOfItem(atPath: frozenOverlayURL.path)[.size] as! Int64 + defaultLogger.appendNewLine("pushing overlay...") + let progress = Progress(totalUnitCount: overlaySize) + ProgressObserver(progress).log(defaultLogger) + let contentDigest = try Digest.hash(frozenOverlayURL) + let chunks = try await DiskV2.push( + diskURL: frozenOverlayURL, + mediaType: asifOverlayMediaType, + registry: registry, + chunkSizeMb: chunkSizeMb, + concurrency: concurrency, + progress: progress + ) + layers.append(contentsOf: annotatedChunks(chunks, kind: .asifOverlay, contentDigest: contentDigest)) + + let blockLayout = try DiskImageStack.diskImageBlockLayout(at: frozenOverlayURL) + let diskSize = blockLayout.blockSize.multipliedReportingOverflow(by: blockLayout.blockCount) + guard !diskSize.overflow else { + throw DiskImageStackError.invalidBlockLayout("stacked disk block layout overflows UInt64") + } + + var annotations = localManifest.annotations ?? [:] + annotations[diskBlockSizeAnnotation] = String(blockLayout.blockSize) + annotations[uncompressedDiskSizeAnnotation] = String(diskSize.partialValue) + + return (layers, annotations) + } + + /// Returns transport descriptors for an immutable disk file. If the + /// target registry lacks the original blobs, recreate them from the local + /// content store. + private func descriptorsForCachedDiskFile( + _ group: TartDiskFileGroup, + contentStore: ContentStore, + registry: Registry, + chunkSizeMb: Int, + concurrency: UInt + ) async throws -> [OCIManifestLayer] { + guard let contentDigest = group.contentDigest else { + throw RuntimeError.VMConfigurationError("stacked VM is missing a pinned disk file digest") + } + + var allChunksExist = true + for chunk in group.chunks { + if try await !registry.blobExists(chunk.digest) { + allChunksExist = false + break + } + } + if allChunksExist { + return group.chunks + } + + guard let contentURL = try contentStore.existingContentURL(for: contentDigest) else { + throw RuntimeError.VMMissingFiles("stacked VM is missing cached disk content \(contentDigest)") + } + let contentSize = try FileManager.default.attributesOfItem(atPath: contentURL.path)[.size] as! Int64 + let progress = Progress(totalUnitCount: contentSize) + let mediaType = group.kind == .base ? diskV2MediaType : asifOverlayMediaType + let chunks = try await DiskV2.push( + diskURL: contentURL, + mediaType: mediaType, + registry: registry, + chunkSizeMb: chunkSizeMb, + concurrency: concurrency, + progress: progress + ) + + return annotatedChunks(chunks, kind: group.kind, contentDigest: contentDigest) + } + + private func annotatedChunks( + _ chunks: [OCIManifestLayer], + kind: TartDiskFileGroup.Kind, + contentDigest: String + ) -> [OCIManifestLayer] { + guard !chunks.isEmpty else { + return chunks + } + + var chunks = chunks + var annotations = chunks[0].annotations ?? [:] + annotations[diskFileContentDigestAnnotation] = contentDigest + if kind == .asifOverlay { + annotations[diskFileChunkCountAnnotation] = String(chunks.count) + } + chunks[0].annotations = annotations + + return chunks } } diff --git a/Sources/tart/VMDirectory.swift b/Sources/tart/VMDirectory.swift index 443a838..89a4922 100644 --- a/Sources/tart/VMDirectory.swift +++ b/Sources/tart/VMDirectory.swift @@ -142,6 +142,24 @@ struct VMDirectory: Prunable { layout?.isRunnable == true } + var isStandalone: Bool { + layout == .standalone + } + + var isStackedVM: Bool { + layout == .stackedLocal + } + + var isStackedCachedImage: Bool { + layout == .stackedOCIRecord + } + + /// Shapes that may live in the remote-image cache. A cached stacked image + /// has no writable overlay and is intentionally not runnable as a local VM. + var isCachedImage: Bool { + layout == .standalone || layout == .stackedOCIRecord + } + func initialize(overwrite: Bool = false) throws { if !overwrite && initialized { throw RuntimeError.VMDirectoryAlreadyInitialized("VM directory is already initialized, preventing overwrite") @@ -172,6 +190,20 @@ struct VMDirectory: Prunable { } } + func validateCachedImage(userFriendlyName: String) throws { + if !FileManager.default.fileExists(atPath: baseURL.path) { + throw RuntimeError.VMDoesNotExist(name: userFriendlyName) + } + + if !isCachedImage { + throw RuntimeError.VMMissingFiles( + "cached image is missing files for a supported layout: " + + "standalone requires \(configURL.lastPathComponent), \(diskURL.lastPathComponent) and \(nvramURL.lastPathComponent); " + + "stacked requires \(configURL.lastPathComponent), \(manifestURL.lastPathComponent) and \(nvramURL.lastPathComponent)" + ) + } + } + func clone(to: VMDirectory, generateMAC: Bool) throws { try FileManager.default.copyItem(at: configURL, to: to.configURL) try FileManager.default.copyItem(at: nvramURL, to: to.nvramURL) @@ -198,7 +230,26 @@ struct VMDirectory: Prunable { try vmConfig.save(toURL: configURL) } - func resizeDisk(_ sizeGB: UInt16, format: DiskImageFormat = .raw) throws { + func resizeDisk( + _ sizeGB: UInt16, + format: DiskImageFormat = .raw, + contentStore: ContentStore? = nil + ) throws { + if isStackedVM { + guard try state() == .Stopped else { + throw RuntimeError.VMConfigurationError("VM \"\(name)\" must be stopped before resizing its disk") + } + + let stack = try diskImageStack(contentStore: contentStore) + let desiredSizeBytes = UInt64(sizeGB) * 1000 * 1000 * 1000 + guard desiredSizeBytes.isMultiple(of: stack.blockSize) else { + throw RuntimeError.InvalidDiskSize("new disk size must align to the stacked disk block size") + } + + try stack.growWritableOverlay(toBlockCount: desiredSizeBytes / stack.blockSize) + return + } + let diskExists = FileManager.default.fileExists(atPath: diskURL.path) if diskExists { @@ -332,7 +383,7 @@ struct VMDirectory: Prunable { } func allocatedSizeBytes() throws -> Int { - try configURL.allocatedSizeBytes() + diskURL.allocatedSizeBytes() + nvramURL.allocatedSizeBytes() + try configURL.allocatedSizeBytes() + localDiskStorageAllocatedSizeBytes() + nvramURL.allocatedSizeBytes() } func allocatedSizeGB() throws -> Int { @@ -340,7 +391,7 @@ struct VMDirectory: Prunable { } func deduplicatedSizeBytes() throws -> Int { - try configURL.deduplicatedSizeBytes() + diskURL.deduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes() + try configURL.deduplicatedSizeBytes() + localDiskStorageDeduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes() } func deduplicatedSizeGB() throws -> Int { @@ -348,7 +399,7 @@ struct VMDirectory: Prunable { } func sizeBytes() throws -> Int { - try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes() + try configURL.sizeBytes() + localDiskStorageSizeBytes() + nvramURL.sizeBytes() } func sizeGB() throws -> Int { @@ -356,6 +407,30 @@ struct VMDirectory: Prunable { } func diskSizeBytes() throws -> Int { + if isStackedVM { + let blockLayout = try DiskImageStack.diskImageBlockLayout(at: overlayURL) + let product = blockLayout.blockSize.multipliedReportingOverflow(by: blockLayout.blockCount) + guard !product.overflow, let diskSizeBytes = Int(exactly: product.partialValue) else { + throw RuntimeError.VMConfigurationError("VM has invalid stacked disk block layout") + } + + return diskSizeBytes + } + + if isStackedCachedImage { + let manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL)) + guard let blockSize = manifest.diskBlockSize(), + let blockCount = manifest.diskBlockCount() else { + throw RuntimeError.VMConfigurationError("VM has invalid stacked disk block layout") + } + let product = blockSize.multipliedReportingOverflow(by: blockCount) + guard !product.overflow, let diskSizeBytes = Int(exactly: product.partialValue) else { + throw RuntimeError.VMConfigurationError("VM has invalid stacked disk block layout") + } + + return diskSizeBytes + } + let vmConfig = try VMConfig(fromURL: configURL) return switch vmConfig.diskFormat { @@ -377,4 +452,23 @@ struct VMDirectory: Prunable { func isExplicitlyPulled() -> Bool { FileManager.default.fileExists(atPath: explicitlyPulledMark.path) } + + private var localDiskStorageURL: URL { + isStackedVM ? overlayURL : diskURL + } + + // Cached stacked images own no disk file in their VM directory. Their + // immutable disk content lives in the shared content store and must not be + // charged to every cached image that references it. + private func localDiskStorageAllocatedSizeBytes() throws -> Int { + isStackedCachedImage ? 0 : try localDiskStorageURL.allocatedSizeBytes() + } + + private func localDiskStorageDeduplicatedSizeBytes() throws -> Int { + isStackedCachedImage ? 0 : try localDiskStorageURL.deduplicatedSizeBytes() + } + + private func localDiskStorageSizeBytes() throws -> Int { + isStackedCachedImage ? 0 : try localDiskStorageURL.sizeBytes() + } } diff --git a/Sources/tart/VMStorageOCI.swift b/Sources/tart/VMStorageOCI.swift index 4240b4e..4848f77 100644 --- a/Sources/tart/VMStorageOCI.swift +++ b/Sources/tart/VMStorageOCI.swift @@ -18,7 +18,109 @@ class VMStorageOCI: PrunableStorage { } func exists(_ name: RemoteName) -> Bool { - VMDirectory(baseURL: vmURL(name)).initialized + VMDirectory(baseURL: vmURL(name)).isCachedImage + } + + /// Whether clone can use a cached image without pulling. Standalone images keep + /// Tart's existing structural check. Stacked cached images cheaply require every + /// immutable file with its expected length; explicit pull remains the path + /// that hashes content and repairs same-sized corruption. + func hasUsableCachedImageForClone(_ name: RemoteName, requireManifest: Bool = false) throws -> Bool { + guard exists(name) else { + return false + } + + let vmDir = VMDirectory(baseURL: vmURL(name)) + if requireManifest && !FileManager.default.fileExists(atPath: vmDir.manifestURL.path) { + return false + } + guard vmDir.isStackedCachedImage else { + return true + } + + let manifest = try OCIManifest(fromJSON: Data(contentsOf: vmDir.manifestURL)) + guard case .stacked(let base, let overlays) = try manifest.tartDiskRepresentation() else { + return true + } + + let contentStore = try ContentStore() + for group in [base] + overlays { + guard let contentDigest = group.contentDigest, + let contentURL = try contentStore.contentURLIfPresent(for: contentDigest) else { + return false + } + + var expectedSize: UInt64 = 0 + for chunk in group.chunks { + guard let uncompressedSize = chunk.uncompressedSize() else { + return false + } + let addition = expectedSize.addingReportingOverflow(uncompressedSize) + guard !addition.overflow else { + return false + } + expectedSize = addition.partialValue + } + + guard let actualSize = UInt64(exactly: try contentURL.sizeBytes()), + actualSize == expectedSize else { + return false + } + } + + return true + } + + /// Whether a cached image is complete enough for `pull` to return without + /// repairing it. Standalone images keep Tart's existing structural cache-hit + /// behavior; stacked cached images additionally need every immutable disk file in + /// the shared content store. + func hasCompleteCachedImage(_ name: RemoteName, manifest: OCIManifest) throws -> Bool { + guard exists(name) else { + return false + } + + guard let missingGroups = try missingStackedDiskFileGroups(for: manifest) else { + return true + } + + return missingGroups.isEmpty + } + + /// The lock-free pull fast path is only useful for a tag that already + /// points at this digest. New or retargeted tags validate once after taking + /// the host lock instead of hashing a large stack twice. + func hasCompleteLinkedImage(_ name: RemoteName, digestName: RemoteName, manifest: OCIManifest) throws -> Bool { + guard exists(name), linked(from: name, to: digestName) else { + return false + } + + return try hasCompleteCachedImage(digestName, manifest: manifest) + } + + /// Bytes that this pull may need to materialize locally. For stacked images + /// this is the sum of only the missing complete disk files, not the final + /// guest-visible disk block layout. + func requiredDiskStorageBytes(for manifest: OCIManifest) throws -> UInt64? { + guard let missingGroups = try missingStackedDiskFileGroups(for: manifest) else { + return manifest.uncompressedDiskSize() + } + + var total: UInt64 = 0 + for group in missingGroups { + for chunk in group.chunks { + guard let uncompressedSize = chunk.uncompressedSize() else { + throw OCIManifestValidationError.invalidDiskMetadata("disk chunks need uncompressed size and content digest") + } + let addition = total.addingReportingOverflow(uncompressedSize) + guard !addition.overflow else { + throw RuntimeError.PullFailed("stacked disk storage size overflows UInt64") + } + total = addition.partialValue + } + } + + return total } func digest(_ name: RemoteName) throws -> String { @@ -34,7 +136,7 @@ class VMStorageOCI: PrunableStorage { func open(_ name: RemoteName, _ accessDate: Date = Date()) throws -> VMDirectory { let vmDir = VMDirectory(baseURL: vmURL(name)) - try vmDir.validate(userFriendlyName: name.description) + try vmDir.validateCachedImage(userFriendlyName: name.description) try vmDir.baseURL.updateAccessDate(accessDate) @@ -44,11 +146,55 @@ class VMStorageOCI: PrunableStorage { func create(_ name: RemoteName, overwrite: Bool = false) throws -> VMDirectory { let vmDir = VMDirectory(baseURL: vmURL(name)) + if !overwrite && vmDir.isCachedImage { + throw RuntimeError.VMDirectoryAlreadyInitialized("VM directory is already initialized, preventing overwrite") + } + try vmDir.initialize(overwrite: overwrite) return vmDir } + /// Materialize the digest-addressed cached image for an image Tart just + /// pushed, without routing its own local data back through the registry. + func populate(_ name: RemoteName, from source: VMDirectory, manifest: OCIManifest) throws { + if try hasCompleteCachedImage(name, manifest: manifest) { + return + } + + let vmDir = try create(name, overwrite: exists(name)) + + if source.isStackedVM { + guard case .stacked(_, let overlays) = try manifest.tartDiskRepresentation(), + let contentDigest = overlays.last?.contentDigest else { + throw RuntimeError.VMConfigurationError("pushed image is missing its writable ASIF overlay") + } + + // The pushed top overlay becomes immutable in the cached image. Keep a + // semantic copy so later clones do not need to fetch it back. + let contentStore = try ContentStore() + if try contentStore.existingContentURL(for: contentDigest) == nil { + let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest) + do { + try FileManager.default.copyItem(at: source.overlayURL, to: temporaryURL) + _ = try contentStore.install(temporaryURL, contentDigest: contentDigest) + } catch { + try? FileManager.default.removeItem(at: temporaryURL) + throw error + } + } + + try FileManager.default.copyItem(at: source.configURL, to: vmDir.configURL) + try FileManager.default.copyItem(at: source.nvramURL, to: vmDir.nvramURL) + } else { + try source.clone(to: vmDir, generateMAC: false) + } + + // Keep the exact manifest Tart submitted so tag links and later pushes + // refer to the same digest-addressed cached image. + try manifest.toJSON().write(to: vmDir.manifestURL) + } + func move(_ name: RemoteName, from: VMDirectory) throws{ let targetURL = vmURL(name) @@ -84,7 +230,7 @@ class VMStorageOCI: PrunableStorage { } let vmDir = VMDirectory(baseURL: foundURL.resolvingSymlinksInPath()) - if !vmDir.initialized { + if !vmDir.isCachedImage { continue } @@ -113,7 +259,7 @@ class VMStorageOCI: PrunableStorage { for case let foundURL as URL in enumerator { let vmDir = VMDirectory(baseURL: foundURL) - if !vmDir.initialized { + if !vmDir.isCachedImage { continue } @@ -141,7 +287,9 @@ class VMStorageOCI: PrunableStorage { } func prunables() throws -> [Prunable] { - try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir } + try list().filter { (_, vmDir, isSymlink) in + !isSymlink && vmDir.isStandalone + }.map { (_, vmDir, _) in vmDir } } func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws { @@ -157,7 +305,7 @@ class VMStorageOCI: PrunableStorage { let digestName = RemoteName(host: name.host, namespace: name.namespace, reference: Reference(digest: Digest.hash(manifestData))) - if exists(name) && exists(digestName) && linked(from: name, to: digestName) { + if try hasCompleteLinkedImage(name, digestName: digestName, manifest: manifest) { // optimistically check if we need to do anything at all before locking defaultLogger.appendNewLine("\(digestName) image is already cached and linked!") return @@ -181,11 +329,13 @@ class VMStorageOCI: PrunableStorage { throw CancellationError() } - if !exists(digestName) { + if try !hasCompleteCachedImage(digestName, manifest: manifest) { let span = OTel.shared.tracer.spanBuilder(spanName: "pull").setActive(true).startSpan() defer { span.end() } let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description) + let digestVMDir = VMDirectory(baseURL: vmURL(digestName)) + let preserveExplicitlyPulledMark = digestVMDir.isExplicitlyPulled() // Open an existing VM directory corresponding to this name, if any, // marking it as outdated to speed up the garbage collection process @@ -195,22 +345,41 @@ class VMStorageOCI: PrunableStorage { let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL) try tmpVMDirLock.lock() + // A previously pulled standalone image already has the complete base + // disk locally as disk.img. Promote that file into the content store + // before sizing or pulling so a stacked child only fetches overlays. + try reuseStandaloneDiskForStackedBaseIfPossible(manifest) + // Try to reclaim some cache space if we know the VM size in advance - if let uncompressedDiskSize = manifest.uncompressedDiskSize() { - OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute( - key: "oci.image-uncompressed-disk-size-bytes", - value: .int(Int(uncompressedDiskSize)) - ) + if let requiredDiskStorageBytes = try requiredDiskStorageBytes(for: manifest) { + if let telemetryValue = Int(exactly: requiredDiskStorageBytes) { + OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute( + key: "oci.image-required-disk-storage-bytes", + value: .int(telemetryValue) + ) + } let otherVMFilesSize: UInt64 = 128 * 1024 * 1024 + let requiredStorage = requiredDiskStorageBytes.addingReportingOverflow(otherVMFilesSize) + guard !requiredStorage.overflow else { + throw RuntimeError.PullFailed("required pull storage size overflows UInt64") + } - try Prune.reclaimIfNeeded(uncompressedDiskSize + otherVMFilesSize) + try Prune.reclaimIfNeeded(requiredStorage.partialValue) } try await withTaskCancellationHandler(operation: { try await retry(maxAttempts: 5) { - // Choose the best base image which has the most deduplication ratio - let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry) + // Existing standalone images can still reuse another complete local disk. + // Stacked images reconstruct their immutable files through the + // shared content store instead of materializing disk.img. + let localLayerCache: LocalLayerCache? + switch try manifest.tartDiskRepresentation() { + case .flat: + localLayerCache = try await chooseLocalLayerCache(name, manifest, registry) + case .stacked: + localLayerCache = nil + } if let llc = localLayerCache { let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file) @@ -232,6 +401,14 @@ class VMStorageOCI: PrunableStorage { return .throw } + + // Preserve the exact manifest bytes received from the registry. Its + // digest identifies this cached image and stacked VMs pin it. + try manifestData.write(to: tmpVMDir.manifestURL) + if preserveExplicitlyPulledMark { + tmpVMDir.markExplicitlyPulled() + } + try move(digestName, from: tmpVMDir) }, onCancel: { try? FileManager.default.removeItem(at: tmpVMDir.baseURL) @@ -253,6 +430,82 @@ class VMStorageOCI: PrunableStorage { _ = try VMStorageOCI().open(name) } + /// Returns `nil` for standalone images and the missing immutable disk-file groups + /// for stacked images. `ContentStore.existingContentURL()` intentionally + /// validates the digest so corrupt entries are repaired by a normal pull. + private func missingStackedDiskFileGroups(for manifest: OCIManifest) throws -> [TartDiskFileGroup]? { + guard case .stacked(let base, let overlays) = try manifest.tartDiskRepresentation() else { + return nil + } + + let contentStore = try ContentStore() + var missingGroups: [TartDiskFileGroup] = [] + for group in [base] + overlays { + guard let contentDigest = group.contentDigest else { + throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest") + } + if try contentStore.existingContentURL(for: contentDigest) == nil { + missingGroups.append(group) + } + } + + return missingGroups + } + + /// Seed a stacked image's immutable base from an already pulled standalone + /// OCI record when both manifests describe the same transport chunks. The + /// content store still verifies the whole-file digest before publishing it. + func reuseStandaloneDiskForStackedBaseIfPossible(_ manifest: OCIManifest) throws { + guard case .stacked(let base, _) = try manifest.tartDiskRepresentation(), + let contentDigest = base.contentDigest else { + return + } + + let contentStore = try ContentStore() + // Content-store entries are verified when installed. Avoid hashing a + // potentially large prewarmed base again on every stacked pull. + guard try contentStore.contentURLIfPresent(for: contentDigest) == nil else { + return + } + + for (_, vmDir, isSymlink) in try list() where !isSymlink && vmDir.isStandalone { + guard let manifestData = try? Data(contentsOf: vmDir.manifestURL), + let candidateManifest = try? OCIManifest(fromJSON: manifestData), + case .flat(let candidateBase) = try? candidateManifest.tartDiskRepresentation(), + diskChunksMatch(candidateBase.chunks, base.chunks) else { + continue + } + + let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest) + do { + try FileManager.default.copyItem(at: vmDir.diskURL, to: temporaryURL) + _ = try contentStore.install(temporaryURL, contentDigest: contentDigest) + return + } catch ContentStoreError.contentDigestMismatch { + try? FileManager.default.removeItem(at: temporaryURL) + } catch { + try? FileManager.default.removeItem(at: temporaryURL) + throw error + } + } + } + + /// Compare the OCI transport identity while ignoring stacked-only + /// whole-file annotations added to the first base chunk. + private func diskChunksMatch(_ left: [OCIManifestLayer], _ right: [OCIManifestLayer]) -> Bool { + guard left.count == right.count else { + return false + } + + return zip(left, right).allSatisfy { left, right in + left.mediaType == right.mediaType && + left.size == right.size && + left.digest == right.digest && + left.uncompressedSize() == right.uncompressedSize() && + left.uncompressedContentDigest() == right.uncompressedContentDigest() + } + } + func linked(from: RemoteName, to: RemoteName) -> Bool { do { let resolvedFrom = try FileManager.default.destinationOfSymbolicLink(atPath: vmURL(from).path) @@ -280,10 +533,16 @@ class VMStorageOCI: PrunableStorage { } // Load OCI VM images and their manifests (if present) - var candidates: [(name: String, vmDir: VMDirectory, manifest: OCIManifest, deduplicatedBytes: UInt64)] = [] + var candidates: [( + name: String, + vmDir: VMDirectory, + manifest: OCIManifest, + manifestDigest: String, + deduplicatedBytes: UInt64 + )] = [] for (name, vmDir, isSymlink) in try list() { - if isSymlink { + if isSymlink || !vmDir.isStandalone { continue } @@ -295,7 +554,13 @@ class VMStorageOCI: PrunableStorage { continue } - candidates.append((name, vmDir, manifest, calculateDeduplicatedBytes(manifest))) + candidates.append(( + name, + vmDir, + manifest, + Digest.hash(manifestJSON), + calculateDeduplicatedBytes(manifest) + )) } // Previously we haven't stored the OCI VM image manifests, but still fetched the VM image manifest if @@ -305,10 +570,17 @@ class VMStorageOCI: PrunableStorage { // with the registry if we haven't already retrieved the manifest for that OCI VM image. if name.reference.type == .Tag, let vmDir = try? open(name), + vmDir.isStandalone, let digest = try? digest(name), - try !candidates.contains(where: {try $0.manifest.digest() == digest}), - let (manifest, _) = try? await registry.pullManifest(reference: digest) { - candidates.append((name.description, vmDir, manifest, calculateDeduplicatedBytes(manifest))) + !candidates.contains(where: { $0.manifestDigest == digest }), + let (manifest, manifestData) = try? await registry.pullManifest(reference: digest) { + candidates.append(( + name.description, + vmDir, + manifest, + Digest.hash(manifestData), + calculateDeduplicatedBytes(manifest) + )) } // Now, find the best match based on how many bytes we'll deduplicate diff --git a/Tests/TartTests/CommandBehaviorTests.swift b/Tests/TartTests/CommandBehaviorTests.swift new file mode 100644 index 0000000..67f9dc7 --- /dev/null +++ b/Tests/TartTests/CommandBehaviorTests.swift @@ -0,0 +1,135 @@ +import Foundation +import ArgumentParser +import XCTest +@testable import tart + +final class CommandBehaviorTests: XCTestCase { + func testSetDiskRejectsStackedVMBeforeSavingConfig() async throws { + try await withTemporaryTartHome { + let vmDir = try VMStorageLocal().create("stacked") + let originalConfig = config() + try originalConfig.save(toURL: vmDir.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data())) + XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.manifestURL.path, contents: Data())) + XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.overlayURL.path, contents: Data())) + + let replacementURL = try temporaryDirectory().appendingPathComponent("replacement.img") + XCTAssertTrue(FileManager.default.createFile(atPath: replacementURL.path, contents: Data("replacement".utf8))) + + let command = try Set.parseAsRoot([ + "stacked", + "--cpu", "4", + "--disk", replacementURL.path, + ]) as! Set + + do { + try await command.run() + XCTFail("expected stacked disk replacement to be rejected") + } catch let error as ValidationError { + XCTAssertEqual(error.message, "--disk is not supported for VMs with a stacked disk") + } + + XCTAssertEqual(try VMConfig(fromURL: vmDir.configURL).cpuCount, originalConfig.cpuCount) + XCTAssertFalse(FileManager.default.fileExists(atPath: vmDir.diskURL.path)) + } + } + + func testRemoteAdditionalDiskRetainsTemporaryBackingFileLock() throws { + try withTemporaryTartHome { + let storage = try VMStorageOCI() + let name = try RemoteName("example.com/org/image:latest") + let cachedImage = try storage.create(name) + try config().save(toURL: cachedImage.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: cachedImage.nvramURL.path, contents: Data())) + XCTAssertTrue(FileManager.default.createFile( + atPath: cachedImage.diskURL.path, + contents: Data(repeating: 0, count: 4096) + )) + + do { + let additionalDisk = try AdditionalDisk(parseFrom: name.description) + let entriesBeforeGC = try temporaryEntries() + XCTAssertEqual(entriesBeforeGC.count, 1) + + try Config().gc() + XCTAssertEqual(try temporaryEntries(), entriesBeforeGC) + + withExtendedLifetime(additionalDisk) {} + } + + try Config().gc() + XCTAssertTrue(try temporaryEntries().isEmpty) + } + } + + func testGarbageCollectionPreservesLockedTemporaryDirectory() throws { + try withTemporaryTartHome { + let temporaryVMDir = try VMDirectory.temporary() + let lock = try FileLock(lockURL: temporaryVMDir.baseURL) + try lock.lock() + XCTAssertTrue(FileManager.default.createFile( + atPath: temporaryVMDir.overlayURL.path, + contents: Data("overlay".utf8) + )) + + try Config().gc() + XCTAssertTrue(FileManager.default.fileExists(atPath: temporaryVMDir.overlayURL.path)) + + try lock.unlock() + try Config().gc() + XCTAssertFalse(FileManager.default.fileExists(atPath: temporaryVMDir.baseURL.path)) + } + } + + private func config() -> VMConfig { + VMConfig( + platform: Linux(), + cpuCountMin: 2, + memorySizeMin: 512 * 1024 * 1024, + diskFormat: .raw + ) + } + + private func temporaryEntries() throws -> [URL] { + try FileManager.default.contentsOfDirectory( + at: Config().tartTmpDir, + includingPropertiesForKeys: nil + ) + } + + private func withTemporaryTartHome(_ body: () throws -> Void) throws { + let home = try temporaryDirectory() + let previousHome = ProcessInfo.processInfo.environment["TART_HOME"] + setenv("TART_HOME", home.path, 1) + defer { restoreEnvironment("TART_HOME", to: previousHome) } + + try body() + } + + private func withTemporaryTartHome(_ body: () async throws -> Void) async throws { + let home = try temporaryDirectory() + let previousHome = ProcessInfo.processInfo.environment["TART_HOME"] + setenv("TART_HOME", home.path, 1) + defer { restoreEnvironment("TART_HOME", to: previousHome) } + + try await body() + } + + private func temporaryDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: false) + addTeardownBlock { + try? FileManager.default.removeItem(at: url) + } + + return url + } + + private func restoreEnvironment(_ name: String, to value: String?) { + if let value { + setenv(name, value, 1) + } else { + unsetenv(name) + } + } +} diff --git a/Tests/TartTests/ContentStoreTests.swift b/Tests/TartTests/ContentStoreTests.swift index 66f64bd..95224cc 100644 --- a/Tests/TartTests/ContentStoreTests.swift +++ b/Tests/TartTests/ContentStoreTests.swift @@ -33,6 +33,63 @@ final class ContentStoreTests: XCTestCase { XCTAssertNil(try store.existingContentURL(for: expectedDigest)) } + func testResumableAndLockURLsAreStablePerDigest() throws { + let store = try temporaryStore() + let firstDigest = Digest.hash(Data("first".utf8)) + let secondDigest = Digest.hash(Data("second".utf8)) + + XCTAssertEqual( + try store.resumableContentURL(for: firstDigest), + try store.resumableContentURL(for: firstDigest) + ) + XCTAssertNotEqual( + try store.resumableContentURL(for: firstDigest), + try store.resumableContentURL(for: secondDigest) + ) + XCTAssertEqual( + try store.lockURL(for: firstDigest), + try store.lockURL(for: firstDigest) + ) + XCTAssertTrue(FileManager.default.fileExists(atPath: try store.lockURL(for: firstDigest).path)) + } + + func testInstallReplacesCorruptEntry() throws { + let store = try temporaryStore() + let data = Data("expected".utf8) + let digest = Digest.hash(data) + let contentURL = try store.contentURL(for: digest) + try FileManager.default.createDirectory(at: contentURL.deletingLastPathComponent(), withIntermediateDirectories: true) + try Data("corrupt".utf8).write(to: contentURL) + let temporaryURL = try store.temporaryContentURL(for: digest) + try data.write(to: temporaryURL) + + XCTAssertEqual(try store.install(temporaryURL, contentDigest: digest), contentURL) + XCTAssertEqual(try Digest.hash(contentURL), digest) + } + + func testInstallPreservesExistingValidEntry() throws { + let store = try temporaryStore() + let data = Data("expected".utf8) + let digest = Digest.hash(data) + let firstTemporaryURL = try store.temporaryContentURL(for: digest) + try data.write(to: firstTemporaryURL) + let installedURL = try store.install(firstTemporaryURL, contentDigest: digest) + let secondTemporaryURL = try store.temporaryContentURL(for: digest) + try data.write(to: secondTemporaryURL) + + XCTAssertEqual(try store.install(secondTemporaryURL, contentDigest: digest), installedURL) + XCTAssertFalse(FileManager.default.fileExists(atPath: secondTemporaryURL.path)) + XCTAssertEqual(try Digest.hash(installedURL), digest) + } + + func testConcurrentInstallsAcceptDigestValidWinner() throws { + try assertConcurrentInstalls(seedCorruptEntry: false) + } + + func testConcurrentInstallsRepairCorruptEntry() throws { + try assertConcurrentInstalls(seedCorruptEntry: true) + } + func testInstallRejectsWrongContentDigest() throws { let store = try temporaryStore() let expectedDigest = Digest.hash(Data("expected".utf8)) @@ -64,4 +121,51 @@ final class ContentStoreTests: XCTestCase { return try ContentStore(baseURL: url) } + + private func assertConcurrentInstalls(seedCorruptEntry: Bool) throws { + let store = try temporaryStore() + let data = Data("expected".utf8) + let digest = Digest.hash(data) + let contentURL = try store.contentURL(for: digest) + try FileManager.default.createDirectory(at: contentURL.deletingLastPathComponent(), withIntermediateDirectories: true) + if seedCorruptEntry { + try Data("corrupt".utf8).write(to: contentURL) + } + + let temporaryURLs = try (0..<16).map { _ in + let url = try store.temporaryContentURL(for: digest) + try data.write(to: url) + return url + } + let errors = ErrorCollector() + + DispatchQueue.concurrentPerform(iterations: temporaryURLs.count) { index in + do { + _ = try store.install(temporaryURLs[index], contentDigest: digest) + } catch { + errors.append(error) + } + } + + XCTAssertTrue(errors.values.isEmpty, "unexpected install errors: \(errors.values)") + XCTAssertEqual(try Digest.hash(contentURL), digest) + XCTAssertTrue(temporaryURLs.allSatisfy { !FileManager.default.fileExists(atPath: $0.path) }) + } + + private final class ErrorCollector: @unchecked Sendable { + private let lock = NSLock() + private var errors: [Error] = [] + + var values: [Error] { + lock.lock() + defer { lock.unlock() } + return errors + } + + func append(_ error: Error) { + lock.lock() + defer { lock.unlock() } + errors.append(error) + } + } } diff --git a/Tests/TartTests/DigestTests.swift b/Tests/TartTests/DigestTests.swift index 1c6fa56..7807b4e 100644 --- a/Tests/TartTests/DigestTests.swift +++ b/Tests/TartTests/DigestTests.swift @@ -1,3 +1,4 @@ +import Foundation import XCTest @testable import tart @@ -21,4 +22,34 @@ final class DigestTests: XCTestCase { XCTAssertEqual(Digest.hash(data), "sha256:d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592") } + + func testFileAndRangeHashingMatchDataHashing() throws { + let prefix = Data(repeating: 0x61, count: 4 * 1024 * 1024 + 17) + let range = Data("range".utf8) + let suffix = Data(repeating: 0x62, count: 23) + let data = prefix + range + suffix + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try data.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + XCTAssertEqual(try Digest.hash(url), Digest.hash(data)) + XCTAssertEqual(try Digest.hash(url, offset: UInt64(prefix.count), size: UInt64(range.count)), Digest.hash(range)) + } + + func testRangeHashingRejectsOutOfBoundsRanges() throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try Data("range".utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + XCTAssertThrowsError(try Digest.hash(url, offset: 6, size: 0)) { error in + guard case DigestError.InvalidOffset = error else { + return XCTFail("unexpected error: \(error)") + } + } + XCTAssertThrowsError(try Digest.hash(url, offset: 1, size: UInt64.max)) { error in + guard case DigestError.InvalidSize = error else { + return XCTFail("unexpected error: \(error)") + } + } + } } diff --git a/Tests/TartTests/DiskImageStackTests.swift b/Tests/TartTests/DiskImageStackTests.swift index 70a5654..6f648eb 100644 --- a/Tests/TartTests/DiskImageStackTests.swift +++ b/Tests/TartTests/DiskImageStackTests.swift @@ -45,6 +45,13 @@ import XCTest _ = try fixture.disk.makeAttachment() } + func testAttachesStackReadOnly() throws { + let fixture = try Fixture(baseFormat: .raw) + try fixture.disk.createWritableOverlay() + + _ = try fixture.disk.makeAttachment(readOnly: true) + } + func testRejectsMissingWritableOverlayWhenAttaching() throws { let fixture = try Fixture(baseFormat: .raw) @@ -74,49 +81,15 @@ import XCTest } } - func testRejectsWrongContentDigest() throws { - let fixture = try Fixture(baseFormat: .raw) - fixture.disk = DiskImageStack( - base: DiskImageFile(url: fixture.disk.base.url, contentDigest: "sha256:wrong"), - baseFormat: fixture.disk.baseFormat, - overlays: fixture.disk.overlays, - writableOverlayURL: fixture.disk.writableOverlayURL, - blockSize: fixture.disk.blockSize, - blockCount: fixture.disk.blockCount - ) - - assertThrows(.invalidDiskImage(fixture.disk.base.url, "disk image content digest does not match")) { - try fixture.disk.createWritableOverlay() - } - } - - func testRejectsWrongOverlayContentDigest() throws { - let fixture = try Fixture(baseFormat: .asif, publishedOverlayCount: 1) - fixture.disk = DiskImageStack( - base: fixture.disk.base, - baseFormat: fixture.disk.baseFormat, - overlays: [ - DiskImageFile(url: fixture.disk.overlays[0].url, contentDigest: "sha256:wrong"), - ], - writableOverlayURL: fixture.disk.writableOverlayURL, - blockSize: fixture.disk.blockSize, - blockCount: fixture.disk.blockCount - ) - - assertThrows(.invalidDiskImage(fixture.disk.overlays[0].url, "disk image content digest does not match")) { - try fixture.disk.createWritableOverlay() - } - } - func testRejectsNonASIFPublishedOverlay() throws { let fixture = try Fixture(baseFormat: .raw) let overlayURL = fixture.directory.appendingPathComponent("published-raw.img") _ = try DiskImage(creating: .raw(url: overlayURL, blockCount: 8)) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: fixture.disk.baseFormat, - overlays: [ - DiskImageFile(url: overlayURL, contentDigest: try Digest.hash(overlayURL)), + immutableOverlayURLs: [ + overlayURL, ], writableOverlayURL: fixture.disk.writableOverlayURL, blockSize: fixture.disk.blockSize, @@ -131,15 +104,15 @@ import XCTest func testRejectsWrongBaseFormat() throws { let fixture = try Fixture(baseFormat: .raw) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: .asif, - overlays: fixture.disk.overlays, + immutableOverlayURLs: fixture.disk.immutableOverlayURLs, writableOverlayURL: fixture.disk.writableOverlayURL, blockSize: fixture.disk.blockSize, blockCount: fixture.disk.blockCount ) - assertThrows(.invalidDiskImage(fixture.disk.base.url, "base disk format does not match")) { + assertThrows(.invalidDiskImage(fixture.disk.baseURL, "base disk format does not match")) { try fixture.disk.createWritableOverlay() } } @@ -147,15 +120,15 @@ import XCTest func testRejectsBlockSizeMismatch() throws { let fixture = try Fixture(baseFormat: .raw) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: fixture.disk.baseFormat, - overlays: fixture.disk.overlays, + immutableOverlayURLs: fixture.disk.immutableOverlayURLs, writableOverlayURL: fixture.disk.writableOverlayURL, blockSize: 4096, blockCount: fixture.disk.blockCount ) - assertThrows(.invalidGeometry("immutable disk stack does not match manifest block size")) { + assertThrows(.invalidBlockLayout("immutable disk stack does not match manifest block size")) { try fixture.disk.createWritableOverlay() } } @@ -163,15 +136,15 @@ import XCTest func testRejectsUnsupportedBlockSize() throws { let fixture = try Fixture(baseFormat: .raw) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: fixture.disk.baseFormat, - overlays: fixture.disk.overlays, + immutableOverlayURLs: fixture.disk.immutableOverlayURLs, writableOverlayURL: fixture.disk.writableOverlayURL, blockSize: 123, blockCount: fixture.disk.blockCount ) - assertThrows(.invalidGeometry("unsupported stacked disk block size 123")) { + assertThrows(.invalidBlockLayout("unsupported stacked disk block size 123")) { try fixture.disk.createWritableOverlay() } } @@ -179,15 +152,15 @@ import XCTest func testRejectsManifestBlockCountMismatch() throws { let fixture = try Fixture(baseFormat: .raw) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: fixture.disk.baseFormat, - overlays: fixture.disk.overlays, + immutableOverlayURLs: fixture.disk.immutableOverlayURLs, writableOverlayURL: fixture.disk.writableOverlayURL, blockSize: fixture.disk.blockSize, blockCount: fixture.disk.blockCount + 1 ) - assertThrows(.invalidGeometry("immutable disk stack does not match manifest block count")) { + assertThrows(.invalidBlockLayout("immutable disk stack does not match manifest block count")) { try fixture.disk.createWritableOverlay() } } @@ -199,9 +172,9 @@ import XCTest let copiedURL = fixture.directory.appendingPathComponent("copied-overlay.asif") try fixture.disk.copyWritableOverlay(to: copiedURL) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: fixture.disk.baseFormat, - overlays: fixture.disk.overlays, + immutableOverlayURLs: fixture.disk.immutableOverlayURLs, writableOverlayURL: copiedURL, blockSize: fixture.disk.blockSize, blockCount: fixture.disk.blockCount @@ -216,15 +189,15 @@ import XCTest let fixture = try Fixture(baseFormat: .asif) let other = try Fixture(baseFormat: .asif, publishedOverlayCount: 1) fixture.disk = DiskImageStack( - base: fixture.disk.base, + baseURL: fixture.disk.baseURL, baseFormat: fixture.disk.baseFormat, - overlays: other.disk.overlays, + immutableOverlayURLs: other.disk.immutableOverlayURLs, writableOverlayURL: fixture.disk.writableOverlayURL, blockSize: fixture.disk.blockSize, blockCount: fixture.disk.blockCount ) - assertThrows(.invalidDiskImage(other.disk.overlays[0].url, "ASIF overlay is incompatible with its parent")) { + assertThrows(.invalidDiskImage(other.disk.immutableOverlayURLs[0], "ASIF overlay is incompatible with its parent")) { try fixture.disk.createWritableOverlay() } } @@ -263,19 +236,19 @@ import XCTest _ = try DiskImage(creating: .asif(url: baseURL, blockCount: 8, blockSize: .bytes512)) } - var overlays: [DiskImageFile] = [] + var immutableOverlayURLs: [URL] = [] var image = try DiskImage(opening: .open(url: baseURL, mode: .readOnly)) for index in 0.. VMDirectory { + let vmDir = try temporaryVMDirectory() + let config = VMConfig( + platform: Linux(), + cpuCountMin: 2, + memorySizeMin: 512 * 1024 * 1024, + diskFormat: diskFormat + ) + try config.save(toURL: vmDir.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data())) + switch diskFormat { + case .raw: + _ = try DiskImage(creating: .raw(url: vmDir.diskURL, blockCount: 8)) + case .asif: + _ = try DiskImage(creating: .asif(url: vmDir.diskURL, blockCount: 8, blockSize: .bytes512)) + } + + let diskChunk = OCIManifestLayer( + mediaType: diskV2MediaType, + size: 1, + digest: "sha256:transport", + uncompressedSize: 4096, + uncompressedContentDigest: "sha256:chunk" + ) + let manifest = OCIManifest( + config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"), + layers: [ + OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"), + diskChunk, + OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"), + ] + ) + try manifest.toJSON().write(to: vmDir.manifestURL) + + return vmDir + } + + private func temporaryContentStore() throws -> ContentStore { + let url = try temporaryDirectory() + return try ContentStore(baseURL: url) + } + + private func temporaryVMDirectory() throws -> VMDirectory { + VMDirectory(baseURL: try temporaryDirectory()) + } + + private func temporaryDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: false) + addTeardownBlock { + try? FileManager.default.removeItem(at: url) + } + + return url + } + } +#endif diff --git a/Tests/TartTests/VMDirectoryLayoutTests.swift b/Tests/TartTests/VMDirectoryLayoutTests.swift index 3e130d3..364a5fd 100644 --- a/Tests/TartTests/VMDirectoryLayoutTests.swift +++ b/Tests/TartTests/VMDirectoryLayoutTests.swift @@ -13,9 +13,11 @@ final class VMDirectoryLayoutTests: XCTestCase { XCTAssertEqual(vmDir.layout, .standalone) XCTAssertTrue(vmDir.initialized) + XCTAssertTrue(vmDir.isCachedImage) + XCTAssertNoThrow(try vmDir.validateCachedImage(userFriendlyName: "standalone")) } - func testStackedLocalLayout() throws { + func testStackedVMLayout() throws { let vmDir = try temporaryVMDirectory() try touch(vmDir.configURL) @@ -25,9 +27,10 @@ final class VMDirectoryLayoutTests: XCTestCase { XCTAssertEqual(vmDir.layout, .stackedLocal) XCTAssertTrue(vmDir.initialized) + XCTAssertFalse(vmDir.isCachedImage) } - func testStackedOCIRecordLayout() throws { + func testStackedCachedImageLayout() throws { let vmDir = try temporaryVMDirectory() try touch(vmDir.configURL) @@ -36,6 +39,8 @@ final class VMDirectoryLayoutTests: XCTestCase { XCTAssertEqual(vmDir.layout, .stackedOCIRecord) XCTAssertFalse(vmDir.initialized) + XCTAssertTrue(vmDir.isCachedImage) + XCTAssertNoThrow(try vmDir.validateCachedImage(userFriendlyName: "stacked")) } func testAmbiguousDiskAndOverlayIsNotInitialized() throws { @@ -49,6 +54,52 @@ final class VMDirectoryLayoutTests: XCTestCase { XCTAssertNil(vmDir.layout) XCTAssertFalse(vmDir.initialized) + XCTAssertFalse(vmDir.isCachedImage) + } + + func testStackedVMAccountingUsesOverlay() throws { + let vmDir = try temporaryVMDirectory() + + try Data("config".utf8).write(to: vmDir.configURL) + try Data("nvram".utf8).write(to: vmDir.nvramURL) + try Data("overlay".utf8).write(to: vmDir.overlayURL) + try stackedManifest(blockSize: 512, blockCount: 8).toJSON().write(to: vmDir.manifestURL) + + XCTAssertEqual( + try vmDir.sizeBytes(), + try vmDir.configURL.sizeBytes() + vmDir.overlayURL.sizeBytes() + vmDir.nvramURL.sizeBytes() + ) + XCTAssertEqual( + try vmDir.allocatedSizeBytes(), + try vmDir.configURL.allocatedSizeBytes() + vmDir.overlayURL.allocatedSizeBytes() + vmDir.nvramURL.allocatedSizeBytes() + ) + } + + func testStackedExportIsRejected() throws { + let vmDir = try temporaryVMDirectory() + try touch(vmDir.configURL) + try touch(vmDir.nvramURL) + try touch(vmDir.manifestURL) + try touch(vmDir.overlayURL) + let archiveURL = vmDir.baseURL.appendingPathComponent("export.tvm") + + XCTAssertThrowsError(try vmDir.exportToArchive(path: archiveURL.path)) { error in + guard case RuntimeError.ExportFailed(let message) = error else { + return XCTFail("unexpected error: \(error)") + } + XCTAssertEqual(message, "exporting stacked VMs is not supported yet") + } + XCTAssertFalse(FileManager.default.fileExists(atPath: archiveURL.path)) + + try FileManager.default.removeItem(at: vmDir.overlayURL) + XCTAssertTrue(vmDir.isStackedCachedImage) + XCTAssertThrowsError(try vmDir.exportToArchive(path: archiveURL.path)) { error in + guard case RuntimeError.ExportFailed(let message) = error else { + return XCTFail("unexpected error: \(error)") + } + XCTAssertEqual(message, "exporting stacked VMs is not supported yet") + } + XCTAssertFalse(FileManager.default.fileExists(atPath: archiveURL.path)) } private func temporaryVMDirectory() throws -> VMDirectory { @@ -64,4 +115,30 @@ final class VMDirectoryLayoutTests: XCTestCase { private func touch(_ url: URL) throws { XCTAssertTrue(FileManager.default.createFile(atPath: url.path, contents: Data())) } + + private func stackedManifest(blockSize: UInt64, blockCount: UInt64) -> OCIManifest { + var disk = OCIManifestLayer( + mediaType: diskV2MediaType, + size: 1, + digest: "sha256:transport", + uncompressedSize: blockSize * blockCount, + uncompressedContentDigest: "sha256:chunk" + ) + disk.annotations?[diskFileContentDigestAnnotation] = "sha256:base" + + var manifest = OCIManifest( + config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"), + layers: [ + OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"), + disk, + OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"), + ] + ) + manifest.annotations = [ + uncompressedDiskSizeAnnotation: String(blockSize * blockCount), + diskBlockSizeAnnotation: String(blockSize), + ] + + return manifest + } } diff --git a/Tests/TartTests/VMStorageOCITests.swift b/Tests/TartTests/VMStorageOCITests.swift new file mode 100644 index 0000000..5a4a8fc --- /dev/null +++ b/Tests/TartTests/VMStorageOCITests.swift @@ -0,0 +1,391 @@ +import Foundation +import XCTest +@testable import tart + +#if canImport(DiskImageKit) + import DiskImageKit +#endif + +final class VMStorageOCITests: XCTestCase { + func testPopulateStandalonePushedImageCachesDiskAndManifest() throws { + try withTemporaryTartHome { + let source = try standaloneSource(diskData: Data("disk".utf8)) + let manifest = try flatManifest() + let name = try digestName(for: manifest) + let storage = try VMStorageOCI() + + try storage.populate(name, from: source, manifest: manifest) + + let cached = try storage.open(name) + XCTAssertTrue(cached.isStandalone) + XCTAssertEqual(try Data(contentsOf: cached.diskURL), Data("disk".utf8)) + XCTAssertEqual(try OCIManifest(fromJSON: Data(contentsOf: cached.manifestURL)), manifest) + } + } + + func testStackedCloneRequiresManifestForLegacyStandaloneCachedImage() throws { + try withTemporaryTartHome { + let manifest = try flatManifest() + let name = try digestName(for: manifest) + let storage = try VMStorageOCI() + let record = try storage.create(name) + try config().save(toURL: record.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data())) + XCTAssertTrue(FileManager.default.createFile(atPath: record.diskURL.path, contents: Data())) + + XCTAssertTrue(try storage.hasUsableCachedImageForClone(name)) + XCTAssertFalse(try storage.hasUsableCachedImageForClone(name, requireManifest: true)) + } + } + + func testCloneCacheCheckRejectsMissingOrWrongSizedStackedContent() throws { + try withTemporaryTartHome { + let baseData = Data("base".utf8) + let overlayData = Data("overlay".utf8) + let baseDigest = Digest.hash(baseData) + let overlayDigest = Digest.hash(overlayData) + let manifest = try stackedManifest( + baseContentDigest: baseDigest, + overlayContentDigest: overlayDigest, + baseUncompressedSize: UInt64(baseData.count), + overlayUncompressedSize: UInt64(overlayData.count) + ) + let name = try digestName(for: manifest) + let storage = try VMStorageOCI() + let record = try storage.create(name) + try config().save(toURL: record.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data())) + try manifest.toJSON().write(to: record.manifestURL) + + XCTAssertFalse(try storage.hasUsableCachedImageForClone(name)) + + let contentStore = try ContentStore() + try installContent(baseData, contentDigest: baseDigest, into: contentStore) + try installContent(overlayData, contentDigest: overlayDigest, into: contentStore) + XCTAssertTrue(try storage.hasUsableCachedImageForClone(name)) + + try Data("bad".utf8).write(to: try contentStore.contentURL(for: overlayDigest)) + XCTAssertFalse(try storage.hasUsableCachedImageForClone(name)) + } + } + + func testListIncludesStackedCachedImage() throws { + try withTemporaryTartHome { + let manifest = try stackedManifest() + let name = try digestName(for: manifest) + let storage = try VMStorageOCI() + let record = try storage.create(name) + try config().save(toURL: record.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data())) + try manifest.toJSON().write(to: record.manifestURL) + + XCTAssertTrue(try storage.list().contains { $0.0 == name.description }) + XCTAssertEqual(try record.diskSizeBytes(), 4096) + XCTAssertNoThrow(try record.allocatedSizeBytes()) + } + } + + func testStackedCacheHitRequiresVerifiedContentAndSizesMissingFiles() throws { + try withTemporaryTartHome { + let baseData = Data("base".utf8) + let overlayData = Data("overlay".utf8) + let baseDigest = Digest.hash(baseData) + let overlayDigest = Digest.hash(overlayData) + let manifest = try stackedManifest( + baseContentDigest: baseDigest, + overlayContentDigest: overlayDigest, + baseUncompressedSize: 10, + overlayUncompressedSize: 20 + ) + let name = try digestName(for: manifest) + let storage = try VMStorageOCI() + let record = try storage.create(name) + try config().save(toURL: record.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data())) + try manifest.toJSON().write(to: record.manifestURL) + + XCTAssertFalse(try storage.hasCompleteCachedImage(name, manifest: manifest)) + XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 30) + + let contentStore = try ContentStore() + try installContent(baseData, contentDigest: baseDigest, into: contentStore) + XCTAssertFalse(try storage.hasCompleteCachedImage(name, manifest: manifest)) + XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 20) + + try installContent(overlayData, contentDigest: overlayDigest, into: contentStore) + XCTAssertTrue(try storage.hasCompleteCachedImage(name, manifest: manifest)) + XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 0) + + let overlayURL = try contentStore.contentURL(for: overlayDigest) + try Data("corrupt".utf8).write(to: overlayURL) + XCTAssertFalse(try storage.hasCompleteCachedImage(name, manifest: manifest)) + XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 20) + } + } + + func testStackedPullReusesPreviouslyPulledStandaloneDisk() throws { + try withTemporaryTartHome { + let diskData = Data([0]) + let contentDigest = Digest.hash(diskData) + let flatManifest = try flatManifest() + let flatName = try digestName(for: flatManifest) + let storage = try VMStorageOCI() + let flatRecord = try storage.create(flatName) + try config().save(toURL: flatRecord.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: flatRecord.nvramURL.path, contents: Data())) + try diskData.write(to: flatRecord.diskURL) + try flatManifest.toJSON().write(to: flatRecord.manifestURL) + + let stackedManifest = try stackedManifest(baseContentDigest: contentDigest) + XCTAssertNil(try ContentStore().existingContentURL(for: contentDigest)) + + try storage.reuseStandaloneDiskForStackedBaseIfPossible(stackedManifest) + + let reusedURL = try XCTUnwrap(try ContentStore().existingContentURL(for: contentDigest)) + XCTAssertEqual(try Data(contentsOf: reusedURL), diskData) + } + } + + func testStackedPullDoesNotRehashInstalledBaseBeforeReuse() throws { + try withTemporaryTartHome { + let contentDigest = Digest.hash(Data("base".utf8)) + let manifest = try stackedManifest(baseContentDigest: contentDigest) + let contentURL = try ContentStore().contentURL(for: contentDigest) + + // Hashing this path would throw. Once an entry is published, this + // fast path must trust its presence and let normal pull validation + // repair unusable content later. + try FileManager.default.createDirectory(at: contentURL, withIntermediateDirectories: false) + + XCTAssertNoThrow(try VMStorageOCI().reuseStandaloneDiskForStackedBaseIfPossible(manifest)) + } + } + + func testNewTagDoesNotValidateCachedStackBeforeLock() throws { + try withTemporaryTartHome { + let baseDigest = Digest.hash(Data("base".utf8)) + let overlayDigest = Digest.hash(Data("overlay".utf8)) + let manifest = try stackedManifest( + baseContentDigest: baseDigest, + overlayContentDigest: overlayDigest + ) + let digestName = try digestName(for: manifest) + let tagName = RemoteName( + host: digestName.host, + namespace: digestName.namespace, + reference: Reference(tag: "latest") + ) + let storage = try VMStorageOCI() + let record = try storage.create(digestName) + try config().save(toURL: record.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data())) + try manifest.toJSON().write(to: record.manifestURL) + + // Hashing this directory as a disk file throws. A new tag must skip + // validation until after it has taken the host lock. + let contentURL = try ContentStore().contentURL(for: baseDigest) + try FileManager.default.createDirectory(at: contentURL, withIntermediateDirectories: false) + XCTAssertFalse(try storage.hasCompleteLinkedImage(tagName, digestName: digestName, manifest: manifest)) + } + } + + func testStandaloneLayerCacheIgnoresStackedCachedImages() async throws { + try await withTemporaryTartHome { + var targetManifest = try flatManifest() + var stackedCandidateManifest = try stackedManifest() + let sharedDiskSize = 2 * 1024 * 1024 * 1024 + targetManifest.layers[1].size = sharedDiskSize + stackedCandidateManifest.layers[1] = targetManifest.layers[1] + + let candidateName = try digestName(for: stackedCandidateManifest) + let storage = try VMStorageOCI() + let candidate = try storage.create(candidateName) + try config().save(toURL: candidate.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: candidate.nvramURL.path, contents: Data())) + try stackedCandidateManifest.toJSON().write(to: candidate.manifestURL) + + let targetName = RemoteName( + host: "example.com", + namespace: "org/target", + reference: Reference(digest: try targetManifest.digest()) + ) + let registry = try Registry(host: targetName.host, namespace: targetName.namespace) + + let layerCache = try await storage.chooseLocalLayerCache(targetName, targetManifest, registry) + XCTAssertNil(layerCache) + } + } + + #if canImport(DiskImageKit) + @available(macOS 27.0, *) + func testPopulateStackedPushedImageCachesImmutableTopOverlay() throws { + if #unavailable(macOS 27.0) { + throw XCTSkip("DiskImageKit tests require macOS 27 or newer") + } + + try withTemporaryTartHome { + let source = try diskImageSource() + let stacked = try temporaryVMDirectory() + try source.cloneAsStackedBase(to: stacked, generateMAC: false) + + var manifest = try OCIManifest(fromJSON: Data(contentsOf: stacked.manifestURL)) + let contentDigest = try Digest.hash(stacked.overlayURL) + var overlay = OCIManifestLayer( + mediaType: asifOverlayMediaType, + size: 1, + digest: "sha256:overlay-transport", + uncompressedSize: 1, + uncompressedContentDigest: "sha256:overlay-chunk" + ) + overlay.annotations?[diskFileContentDigestAnnotation] = contentDigest + overlay.annotations?[diskFileChunkCountAnnotation] = "1" + manifest.layers.insert(overlay, at: manifest.layers.count - 1) + + let name = try digestName(for: manifest) + let storage = try VMStorageOCI() + try storage.populate(name, from: stacked, manifest: manifest) + + let cached = try storage.open(name) + XCTAssertTrue(cached.isStackedCachedImage) + XCTAssertFalse(FileManager.default.fileExists(atPath: cached.overlayURL.path)) + XCTAssertEqual(try OCIManifest(fromJSON: Data(contentsOf: cached.manifestURL)), manifest) + + let cachedContent = try XCTUnwrap(try ContentStore().existingContentURL(for: contentDigest)) + XCTAssertEqual(try Digest.hash(cachedContent), contentDigest) + } + } + #endif + + private func standaloneSource(diskData: Data) throws -> VMDirectory { + let vmDir = try temporaryVMDirectory() + try config().save(toURL: vmDir.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data())) + try diskData.write(to: vmDir.diskURL) + + return vmDir + } + + #if canImport(DiskImageKit) + @available(macOS 27.0, *) + private func diskImageSource() throws -> VMDirectory { + let vmDir = try temporaryVMDirectory() + try config().save(toURL: vmDir.configURL) + XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data())) + _ = try DiskImage(creating: .raw(url: vmDir.diskURL, blockCount: 8)) + try flatManifest().toJSON().write(to: vmDir.manifestURL) + + return vmDir + } + #endif + + private func config() -> VMConfig { + VMConfig( + platform: Linux(), + cpuCountMin: 2, + memorySizeMin: 512 * 1024 * 1024, + diskFormat: .raw + ) + } + + private func flatManifest() throws -> OCIManifest { + let disk = OCIManifestLayer( + mediaType: diskV2MediaType, + size: 1, + digest: "sha256:disk-transport", + uncompressedSize: 1, + uncompressedContentDigest: "sha256:disk-chunk" + ) + + return OCIManifest( + config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"), + layers: [ + OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"), + disk, + OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"), + ] + ) + } + + private func stackedManifest( + baseContentDigest: String = "sha256:base", + overlayContentDigest: String = "sha256:overlay", + baseUncompressedSize: UInt64 = 1, + overlayUncompressedSize: UInt64 = 1 + ) throws -> OCIManifest { + var manifest = try flatManifest() + manifest.annotations?[diskBlockSizeAnnotation] = "512" + manifest.annotations?[uncompressedDiskSizeAnnotation] = "4096" + manifest.layers[1].annotations?[diskFileContentDigestAnnotation] = baseContentDigest + manifest.layers[1].annotations?[uncompressedSizeAnnotation] = String(baseUncompressedSize) + var overlay = OCIManifestLayer( + mediaType: asifOverlayMediaType, + size: 1, + digest: "sha256:overlay-transport", + uncompressedSize: overlayUncompressedSize, + uncompressedContentDigest: "sha256:overlay-chunk" + ) + overlay.annotations?[diskFileContentDigestAnnotation] = overlayContentDigest + overlay.annotations?[diskFileChunkCountAnnotation] = "1" + manifest.layers.insert(overlay, at: manifest.layers.count - 1) + + return manifest + } + + private func installContent(_ data: Data, contentDigest: String, into contentStore: ContentStore) throws { + let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest) + try data.write(to: temporaryURL) + _ = try contentStore.install(temporaryURL, contentDigest: contentDigest) + } + + private func digestName(for manifest: OCIManifest) throws -> RemoteName { + RemoteName( + host: "example.com", + namespace: "org/image", + reference: Reference(digest: try manifest.digest()) + ) + } + + private func withTemporaryTartHome(_ body: () throws -> Void) throws { + let home = try temporaryDirectory() + let previousHome = ProcessInfo.processInfo.environment["TART_HOME"] + setenv("TART_HOME", home.path, 1) + defer { + if let previousHome { + setenv("TART_HOME", previousHome, 1) + } else { + unsetenv("TART_HOME") + } + } + + try body() + } + + private func withTemporaryTartHome(_ body: () async throws -> Void) async throws { + let home = try temporaryDirectory() + let previousHome = ProcessInfo.processInfo.environment["TART_HOME"] + setenv("TART_HOME", home.path, 1) + defer { + if let previousHome { + setenv("TART_HOME", previousHome, 1) + } else { + unsetenv("TART_HOME") + } + } + + try await body() + } + + private func temporaryVMDirectory() throws -> VMDirectory { + VMDirectory(baseURL: try temporaryDirectory()) + } + + private func temporaryDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: false) + addTeardownBlock { + try? FileManager.default.removeItem(at: url) + } + + return url + } +}