mirror of https://github.com/cirruslabs/tart.git
Add OCI transport and base clone with DiskImageKit (#1304)
* Add OCI transport and base clone with DiskImageKit * Address stacked OCI pull review feedback * Stream file digest hashing * Lock frozen overlays during push
This commit is contained in:
parent
f87b57bbc5
commit
4ce8a115f7
|
|
@ -31,6 +31,9 @@ struct Clone: AsyncParsableCommand {
|
||||||
@Flag(help: .hidden)
|
@Flag(help: .hidden)
|
||||||
var deduplicate: Bool = false
|
var deduplicate: Bool = false
|
||||||
|
|
||||||
|
@Flag(help: "create a stacked disk that uses the source image as an immutable base")
|
||||||
|
var stacked: Bool = false
|
||||||
|
|
||||||
@Option(help: ArgumentHelp("limit automatic pruning to n gigabytes", valueName: "n"))
|
@Option(help: ArgumentHelp("limit automatic pruning to n gigabytes", valueName: "n"))
|
||||||
var pruneLimit: UInt = 100
|
var pruneLimit: UInt = 100
|
||||||
|
|
||||||
|
|
@ -47,8 +50,15 @@ struct Clone: AsyncParsableCommand {
|
||||||
func run() async throws {
|
func run() async throws {
|
||||||
let ociStorage = try VMStorageOCI()
|
let ociStorage = try VMStorageOCI()
|
||||||
let localStorage = try VMStorageLocal()
|
let localStorage = try VMStorageLocal()
|
||||||
|
let remoteName = try? RemoteName(sourceName)
|
||||||
|
|
||||||
if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
|
if stacked {
|
||||||
|
guard remoteName != nil else {
|
||||||
|
throw ValidationError("--stacked requires a remote image")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let remoteName, try !ociStorage.hasUsableCachedImageForClone(remoteName, requireManifest: stacked) {
|
||||||
// Pull the VM in case it's OCI-based and doesn't exist locally yet
|
// Pull the VM in case it's OCI-based and doesn't exist locally yet
|
||||||
let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
|
let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
|
||||||
try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
|
try await ociStorage.pull(remoteName, registry: registry, concurrency: concurrency, deduplicate: deduplicate)
|
||||||
|
|
@ -66,9 +76,28 @@ struct Clone: AsyncParsableCommand {
|
||||||
let lock = try FileLock(lockURL: Config().tartHomeDir)
|
let lock = try FileLock(lockURL: Config().tartHomeDir)
|
||||||
try lock.lock()
|
try lock.lock()
|
||||||
|
|
||||||
|
let sourceState = try sourceVM.state()
|
||||||
let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
|
let generateMAC = try localStorage.hasVMsWithMACAddress(macAddress: sourceVM.macAddress())
|
||||||
&& sourceVM.state() != .Suspended
|
&& sourceState != .Suspended
|
||||||
try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)
|
|
||||||
|
if stacked {
|
||||||
|
guard sourceVM.isStandalone else {
|
||||||
|
throw ValidationError("--stacked cannot use an image that already has a stacked disk")
|
||||||
|
}
|
||||||
|
guard try VMConfig(fromURL: sourceVM.configURL).os == .darwin else {
|
||||||
|
throw ValidationError("--stacked currently supports only macOS images")
|
||||||
|
}
|
||||||
|
try sourceVM.cloneAsStackedBase(to: tmpVMDir, generateMAC: generateMAC)
|
||||||
|
} else if sourceVM.isStackedCachedImage {
|
||||||
|
try sourceVM.cloneStacked(to: tmpVMDir, copyWritableOverlay: false, generateMAC: generateMAC)
|
||||||
|
} else if sourceVM.isStackedVM {
|
||||||
|
guard sourceState == .Stopped else {
|
||||||
|
throw RuntimeError.VMConfigurationError("VM \"\(sourceName)\" must be stopped before cloning")
|
||||||
|
}
|
||||||
|
try sourceVM.cloneStacked(to: tmpVMDir, copyWritableOverlay: true, generateMAC: generateMAC)
|
||||||
|
} else {
|
||||||
|
try sourceVM.clone(to: tmpVMDir, generateMAC: generateMAC)
|
||||||
|
}
|
||||||
|
|
||||||
try localStorage.move(newName, from: tmpVMDir)
|
try localStorage.move(newName, from: tmpVMDir)
|
||||||
|
|
||||||
|
|
@ -78,11 +107,23 @@ struct Clone: AsyncParsableCommand {
|
||||||
// is not actually claiming new space until the VM is started and it writes something to disk.
|
// is not actually claiming new space until the VM is started and it writes something to disk.
|
||||||
//
|
//
|
||||||
// So, once we clone the VM let's try to claim the rest of space for the VM to run without errors.
|
// So, once we clone the VM let's try to claim the rest of space for the VM to run without errors.
|
||||||
let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes()
|
if sourceVM.isStandalone {
|
||||||
// Avoid reclaiming an excessive amount of disk space.
|
let unallocatedBytes = try sourceVM.sizeBytes() - sourceVM.allocatedSizeBytes()
|
||||||
let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024)
|
// Avoid reclaiming an excessive amount of disk space.
|
||||||
if reclaimBytes > 0 {
|
let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024)
|
||||||
try Prune.reclaimIfNeeded(UInt64(reclaimBytes), sourceVM)
|
if reclaimBytes > 0 {
|
||||||
|
try Prune.reclaimIfNeeded(UInt64(reclaimBytes), sourceVM)
|
||||||
|
}
|
||||||
|
} else if sourceVM.isStackedVM || sourceVM.isStackedCachedImage {
|
||||||
|
let clonedVM = try localStorage.open(newName)
|
||||||
|
// A stacked clone owns only its writable overlay locally, but that
|
||||||
|
// overlay may grow to the full guest-visible disk block layout at
|
||||||
|
// runtime. Reclaim against the clone so it is not pruned itself.
|
||||||
|
let unallocatedBytes = try clonedVM.diskSizeBytes() - clonedVM.allocatedSizeBytes()
|
||||||
|
let reclaimBytes = min(unallocatedBytes, Int(pruneLimit) * 1024 * 1024 * 1024)
|
||||||
|
if reclaimBytes > 0 {
|
||||||
|
try Prune.reclaimIfNeeded(UInt64(reclaimBytes), clonedVM)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}, onCancel: {
|
}, onCancel: {
|
||||||
try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
|
try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ struct Get: AsyncParsableCommand {
|
||||||
OS: vmConfig.os,
|
OS: vmConfig.os,
|
||||||
CPU: vmConfig.cpuCount,
|
CPU: vmConfig.cpuCount,
|
||||||
Memory: memorySizeInMb,
|
Memory: memorySizeInMb,
|
||||||
Disk: HumanReadableByteCount(try vmDir.sizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
Disk: HumanReadableByteCount(try vmDir.diskSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
||||||
DiskFormat: vmConfig.diskFormat.rawValue,
|
DiskFormat: vmConfig.diskFormat.rawValue,
|
||||||
Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) {
|
Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) {
|
||||||
String(format: "%.3f", Float($0) / 1000 / 1000 / 1000)
|
String(format: "%.3f", Float($0) / 1000 / 1000 / 1000)
|
||||||
|
|
@ -40,7 +40,6 @@ struct Get: AsyncParsableCommand {
|
||||||
Running: try vmDir.running(),
|
Running: try vmDir.running(),
|
||||||
State: try vmDir.state().rawValue
|
State: try vmDir.state().rawValue
|
||||||
)
|
)
|
||||||
|
|
||||||
print(format.renderSingle(info))
|
print(format.renderSingle(info))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,11 @@ struct Import: AsyncParsableCommand {
|
||||||
print("importing...")
|
print("importing...")
|
||||||
try tmpVMDir.importFromArchive(path: path)
|
try tmpVMDir.importFromArchive(path: path)
|
||||||
|
|
||||||
|
if tmpVMDir.isStackedVM || tmpVMDir.isStackedCachedImage {
|
||||||
|
try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
|
||||||
|
throw RuntimeError.ImportFailed("importing stacked VMs is not supported yet")
|
||||||
|
}
|
||||||
|
|
||||||
try await withTaskCancellationHandler(operation: {
|
try await withTaskCancellationHandler(operation: {
|
||||||
// Acquire a global lock
|
// Acquire a global lock
|
||||||
let lock = try FileLock(lockURL: Config().tartHomeDir)
|
let lock = try FileLock(lockURL: Config().tartHomeDir)
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@ struct List: AsyncParsableCommand {
|
||||||
try VMInfo(
|
try VMInfo(
|
||||||
Source: "local",
|
Source: "local",
|
||||||
Name: name,
|
Name: name,
|
||||||
Disk: HumanReadableByteCount(try vmDir.sizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
Disk: HumanReadableByteCount(try vmDir.diskSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
||||||
Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
||||||
Accessed: formatAccessDate(try vmDir.accessDate()),
|
Accessed: formatAccessDate(try vmDir.accessDate()),
|
||||||
Running: vmDir.running(),
|
Running: vmDir.running(),
|
||||||
|
|
@ -56,7 +56,7 @@ struct List: AsyncParsableCommand {
|
||||||
try VMInfo(
|
try VMInfo(
|
||||||
Source: "OCI",
|
Source: "OCI",
|
||||||
Name: name,
|
Name: name,
|
||||||
Disk: HumanReadableByteCount(try vmDir.sizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
Disk: HumanReadableByteCount(try vmDir.diskSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
||||||
Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
Size: HumanReadableByteCount(try vmDir.allocatedSizeBytes()) { $0 / 1000 / 1000 / 1000 },
|
||||||
Accessed: formatAccessDate(try vmDir.accessDate()),
|
Accessed: formatAccessDate(try vmDir.accessDate()),
|
||||||
Running: vmDir.running(),
|
Running: vmDir.running(),
|
||||||
|
|
|
||||||
|
|
@ -69,7 +69,7 @@ struct Push: AsyncParsableCommand {
|
||||||
let references = remoteNamesForRegistry.map{ $0.reference.value }
|
let references = remoteNamesForRegistry.map{ $0.reference.value }
|
||||||
|
|
||||||
let pushedRemoteName: RemoteName
|
let pushedRemoteName: RemoteName
|
||||||
// If we're pushing a local OCI VM, check if points to an already existing registry manifest
|
// If we're pushing a cached remote image, check if it points to an existing registry manifest
|
||||||
// and if so, only upload manifests (without config, disk and NVRAM) to the user-specified references
|
// and if so, only upload manifests (without config, disk and NVRAM) to the user-specified references
|
||||||
if let remoteName = try? RemoteName(localName) {
|
if let remoteName = try? RemoteName(localName) {
|
||||||
pushedRemoteName = try await lightweightPushToRegistry(
|
pushedRemoteName = try await lightweightPushToRegistry(
|
||||||
|
|
@ -78,17 +78,18 @@ struct Push: AsyncParsableCommand {
|
||||||
references: references
|
references: references
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
pushedRemoteName = try await localVMDir.pushToRegistry(
|
let pushedImage = try await localVMDir.pushToRegistry(
|
||||||
registry: registry,
|
registry: registry,
|
||||||
references: references,
|
references: references,
|
||||||
chunkSizeMb: chunkSize,
|
chunkSizeMb: chunkSize,
|
||||||
concurrency: concurrency,
|
concurrency: concurrency,
|
||||||
labels: parseLabels()
|
labels: parseLabels()
|
||||||
)
|
)
|
||||||
|
pushedRemoteName = pushedImage.name
|
||||||
|
|
||||||
// Populate the local cache (if requested)
|
// Populate the local cache (if requested)
|
||||||
if populateCache {
|
if populateCache {
|
||||||
let expectedPushedVMDir = try ociStorage.create(pushedRemoteName)
|
try ociStorage.populate(pushedImage.name, from: localVMDir, manifest: pushedImage.manifest)
|
||||||
try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -102,7 +103,7 @@ struct Push: AsyncParsableCommand {
|
||||||
}
|
}
|
||||||
|
|
||||||
func lightweightPushToRegistry(registry: Registry, remoteName: RemoteName, references: [String]) async throws -> RemoteName {
|
func lightweightPushToRegistry(registry: Registry, remoteName: RemoteName, references: [String]) async throws -> RemoteName {
|
||||||
// Is the local OCI VM already present in the registry?
|
// Is the cached remote image already present in the registry?
|
||||||
let digest = try VMStorageOCI().digest(remoteName)
|
let digest = try VMStorageOCI().digest(remoteName)
|
||||||
|
|
||||||
let (remoteManifest, _) = try await registry.pullManifest(reference: digest)
|
let (remoteManifest, _) = try await registry.pullManifest(reference: digest)
|
||||||
|
|
|
||||||
|
|
@ -455,10 +455,15 @@ struct Run: AsyncParsableCommand {
|
||||||
let provisioning = try provisioningOpts.map { try GuestProvisioningOptions($0) }
|
let provisioning = try provisioningOpts.map { try GuestProvisioningOptions($0) }
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
// Keep these values alive while the VM runs. Some additional disks own a
|
||||||
|
// lock that protects their temporary backing files from Config.gc().
|
||||||
|
let additionalDisks = try additionalDisks()
|
||||||
|
defer { withExtendedLifetime(additionalDisks) {} }
|
||||||
|
|
||||||
vm = try VM(
|
vm = try VM(
|
||||||
vmDir: vmDir,
|
vmDir: vmDir,
|
||||||
network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
|
network: userSpecifiedNetwork(vmDir: vmDir) ?? NetworkShared(),
|
||||||
additionalStorageDevices: try additionalDiskAttachments(),
|
additionalStorageDevices: additionalDisks.map(\.configuration),
|
||||||
directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
|
directorySharingDevices: directoryShares() + rosettaDirectoryShare(),
|
||||||
serialPorts: serialPorts,
|
serialPorts: serialPorts,
|
||||||
suspendable: suspendable,
|
suspendable: suspendable,
|
||||||
|
|
@ -727,9 +732,9 @@ struct Run: AsyncParsableCommand {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func additionalDiskAttachments() throws -> [VZStorageDeviceConfiguration] {
|
func additionalDisks() throws -> [AdditionalDisk] {
|
||||||
try disk.map {
|
try disk.map {
|
||||||
try AdditionalDisk(parseFrom: $0).configuration
|
try AdditionalDisk(parseFrom: $0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -952,14 +957,32 @@ struct VMView: NSViewRepresentable {
|
||||||
|
|
||||||
struct AdditionalDisk {
|
struct AdditionalDisk {
|
||||||
let configuration: VZStorageDeviceConfiguration
|
let configuration: VZStorageDeviceConfiguration
|
||||||
|
// Retained for as long as the additional disk is attached, so Config.gc()
|
||||||
|
// cannot remove a temporary backing file or stacked-disk directory.
|
||||||
|
private let temporaryDiskLock: FileLock?
|
||||||
|
|
||||||
init(parseFrom: String) throws {
|
init(parseFrom: String) throws {
|
||||||
let (diskPath, readOnly, syncModeRaw, cachingModeRaw) = Self.parseOptions(parseFrom)
|
let (diskPath, readOnly, syncModeRaw, cachingModeRaw) = Self.parseOptions(parseFrom)
|
||||||
|
|
||||||
self.configuration = try Self.craft(diskPath, readOnly: readOnly, syncModeRaw: syncModeRaw, cachingModeRaw: cachingModeRaw)
|
self = try Self.craft(
|
||||||
|
diskPath,
|
||||||
|
readOnly: readOnly,
|
||||||
|
syncModeRaw: syncModeRaw,
|
||||||
|
cachingModeRaw: cachingModeRaw
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
static func craft(_ diskPath: String, readOnly diskReadOnly: Bool, syncModeRaw: String, cachingModeRaw: String) throws -> VZStorageDeviceConfiguration {
|
private init(configuration: VZStorageDeviceConfiguration, temporaryDiskLock: FileLock? = nil) {
|
||||||
|
self.configuration = configuration
|
||||||
|
self.temporaryDiskLock = temporaryDiskLock
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func craft(
|
||||||
|
_ diskPath: String,
|
||||||
|
readOnly diskReadOnly: Bool,
|
||||||
|
syncModeRaw: String,
|
||||||
|
cachingModeRaw: String
|
||||||
|
) throws -> AdditionalDisk {
|
||||||
let diskURL = URL(string: diskPath)
|
let diskURL = URL(string: diskPath)
|
||||||
|
|
||||||
if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
|
if (["nbd", "nbds", "nbd+unix", "nbds+unix"].contains(diskURL?.scheme)) {
|
||||||
|
|
@ -974,7 +997,7 @@ struct AdditionalDisk {
|
||||||
synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)
|
synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw)
|
||||||
)
|
)
|
||||||
|
|
||||||
return VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment)
|
return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: nbdAttachment))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Expand the tilde (~) since at this point we're dealing with a local path,
|
// Expand the tilde (~) since at this point we're dealing with a local path,
|
||||||
|
|
@ -1005,13 +1028,33 @@ struct AdditionalDisk {
|
||||||
let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
|
let blockAttachment = try VZDiskBlockDeviceStorageDeviceAttachment(fileHandle: FileHandle(fileDescriptor: fd, closeOnDealloc: true),
|
||||||
readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw))
|
readOnly: diskReadOnly, synchronizationMode: try VZDiskSynchronizationMode(syncModeRaw))
|
||||||
|
|
||||||
return VZVirtioBlockDeviceConfiguration(attachment: blockAttachment)
|
return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: blockAttachment))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Support remote VM names in --disk command-line argument
|
// Support remote VM names in --disk command-line argument
|
||||||
if let remoteName = try? RemoteName(diskPath) {
|
if let remoteName = try? RemoteName(diskPath) {
|
||||||
let vmDir = try VMStorageOCI().open(remoteName)
|
let vmDir = try VMStorageOCI().open(remoteName)
|
||||||
|
|
||||||
|
if vmDir.isStackedCachedImage {
|
||||||
|
// A cached stacked image has no writable top overlay. Create one in a
|
||||||
|
// disposable directory for this additional-disk attachment.
|
||||||
|
let temporaryVMDir = try VMDirectory.temporary()
|
||||||
|
try FileManager.default.copyItem(at: vmDir.configURL, to: temporaryVMDir.configURL)
|
||||||
|
try FileManager.default.copyItem(at: vmDir.nvramURL, to: temporaryVMDir.nvramURL)
|
||||||
|
try FileManager.default.copyItem(at: vmDir.manifestURL, to: temporaryVMDir.manifestURL)
|
||||||
|
let lock = try FileLock(lockURL: temporaryVMDir.baseURL)
|
||||||
|
try lock.lock()
|
||||||
|
let stack = try temporaryVMDir.diskImageStack()
|
||||||
|
try stack.createWritableOverlay()
|
||||||
|
let attachment = try stack.makeAttachment(
|
||||||
|
readOnly: diskReadOnly,
|
||||||
|
cachingMode: try VZDiskImageCachingMode(cachingModeRaw) ?? .automatic,
|
||||||
|
synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
|
||||||
|
)
|
||||||
|
|
||||||
|
return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: attachment), temporaryDiskLock: lock)
|
||||||
|
}
|
||||||
|
|
||||||
// Unfortunately, VZDiskImageStorageDeviceAttachment does not support
|
// Unfortunately, VZDiskImageStorageDeviceAttachment does not support
|
||||||
// FileHandle, so we can't easily clone the disk, open it and unlink(2)
|
// FileHandle, so we can't easily clone the disk, open it and unlink(2)
|
||||||
// to simplify the garbage collection, so use an intermediate directory.
|
// to simplify the garbage collection, so use an intermediate directory.
|
||||||
|
|
@ -1024,7 +1067,7 @@ struct AdditionalDisk {
|
||||||
|
|
||||||
let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: clonedDiskURL, readOnly: diskReadOnly)
|
let diskImageAttachment = try VZDiskImageStorageDeviceAttachment(url: clonedDiskURL, readOnly: diskReadOnly)
|
||||||
|
|
||||||
return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
|
return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment), temporaryDiskLock: lock)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Error out if the disk is locked by the host (e.g. it was mounted in Finder),
|
// Error out if the disk is locked by the host (e.g. it was mounted in Finder),
|
||||||
|
|
@ -1040,7 +1083,7 @@ struct AdditionalDisk {
|
||||||
synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
|
synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
|
||||||
)
|
)
|
||||||
|
|
||||||
return VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment)
|
return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: diskImageAttachment))
|
||||||
}
|
}
|
||||||
|
|
||||||
static func parseOptions(_ parseFrom: String) -> (String, Bool, String, String) {
|
static func parseOptions(_ parseFrom: String) -> (String, Bool, String, String) {
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,14 @@ struct Set: AsyncParsableCommand {
|
||||||
|
|
||||||
func run() async throws {
|
func run() async throws {
|
||||||
let vmDir = try VMStorageLocal().open(name)
|
let vmDir = try VMStorageLocal().open(name)
|
||||||
|
|
||||||
|
// Replacing disk.img would leave a stacked VM with both disk.img and
|
||||||
|
// overlay.asif, which is not a supported local layout. Reject before
|
||||||
|
// saving any other requested configuration changes.
|
||||||
|
if disk != nil, vmDir.isStackedVM {
|
||||||
|
throw ValidationError("--disk is not supported for VMs with a stacked disk")
|
||||||
|
}
|
||||||
|
|
||||||
var vmConfig = try VMConfig(fromURL: vmDir.configURL)
|
var vmConfig = try VMConfig(fromURL: vmDir.configURL)
|
||||||
|
|
||||||
if let cpu = cpu {
|
if let cpu = cpu {
|
||||||
|
|
|
||||||
|
|
@ -39,15 +39,48 @@ struct ContentStore {
|
||||||
return targetURL.deletingLastPathComponent().appendingPathComponent(".\(UUID().uuidString).tmp")
|
return targetURL.deletingLastPathComponent().appendingPathComponent(".\(UUID().uuidString).tmp")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns a validated cache hit. Corrupt files are treated as misses so a
|
/// Returns a stable staging path so an interrupted registry pull can resume
|
||||||
/// later pull can safely rebuild them.
|
/// reconstructing this content entry on a later attempt.
|
||||||
func existingContentURL(for contentDigest: String) throws -> URL? {
|
func resumableContentURL(for contentDigest: String) throws -> URL {
|
||||||
|
let targetURL = try contentURL(for: contentDigest)
|
||||||
|
|
||||||
|
return targetURL.deletingLastPathComponent().appendingPathComponent(".\(targetURL.lastPathComponent).partial")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a stable lock file for serializing reconstruction of one content
|
||||||
|
/// entry. The file is intentionally retained; flock state lives on the file
|
||||||
|
/// descriptor and disappears when the owning process exits.
|
||||||
|
func lockURL(for contentDigest: String) throws -> URL {
|
||||||
|
let targetURL = try contentURL(for: contentDigest)
|
||||||
|
|
||||||
|
let lockURL = targetURL.deletingLastPathComponent().appendingPathComponent(".\(targetURL.lastPathComponent).lock")
|
||||||
|
if !FileManager.default.fileExists(atPath: lockURL.path) {
|
||||||
|
_ = FileManager.default.createFile(atPath: lockURL.path, contents: nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
return lockURL
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a digest-addressed entry without rereading it. Pull verifies
|
||||||
|
/// content hashes before accepting a cache hit; clone only needs a cheap
|
||||||
|
/// structural check, like Tart's existing disk.img path.
|
||||||
|
func contentURLIfPresent(for contentDigest: String) throws -> URL? {
|
||||||
let url = try contentURL(for: contentDigest)
|
let url = try contentURL(for: contentDigest)
|
||||||
|
|
||||||
guard FileManager.default.fileExists(atPath: url.path) else {
|
guard FileManager.default.fileExists(atPath: url.path) else {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a validated cache hit. Corrupt files are treated as misses so a
|
||||||
|
/// later pull can safely rebuild them.
|
||||||
|
func existingContentURL(for contentDigest: String) throws -> URL? {
|
||||||
|
guard let url = try contentURLIfPresent(for: contentDigest) else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
guard try Digest.hash(url) == contentDigest else {
|
guard try Digest.hash(url) == contentDigest else {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
@ -57,7 +90,8 @@ struct ContentStore {
|
||||||
|
|
||||||
/// Move a fully reconstructed temporary file into the cache after verifying
|
/// Move a fully reconstructed temporary file into the cache after verifying
|
||||||
/// its semantic identity. The caller should create the temporary file with
|
/// its semantic identity. The caller should create the temporary file with
|
||||||
/// `temporaryContentURL(for:)` so rename stays on the same filesystem.
|
/// temporaryContentURL(for:) or resumableContentURL(for:) so rename stays on
|
||||||
|
/// the same filesystem.
|
||||||
func install(_ temporaryURL: URL, contentDigest: String) throws -> URL {
|
func install(_ temporaryURL: URL, contentDigest: String) throws -> URL {
|
||||||
let actualDigest = try Digest.hash(temporaryURL)
|
let actualDigest = try Digest.hash(temporaryURL)
|
||||||
guard actualDigest == contentDigest else {
|
guard actualDigest == contentDigest else {
|
||||||
|
|
@ -65,14 +99,20 @@ struct ContentStore {
|
||||||
}
|
}
|
||||||
|
|
||||||
let targetURL = try contentURL(for: contentDigest)
|
let targetURL = try contentURL(for: contentDigest)
|
||||||
|
let lock = try FileLock(lockURL: baseURL)
|
||||||
|
try lock.lock()
|
||||||
|
defer { try? lock.unlock() }
|
||||||
|
|
||||||
if let existingURL = try existingContentURL(for: contentDigest) {
|
if let existingURL = try existingContentURL(for: contentDigest) {
|
||||||
try? FileManager.default.removeItem(at: temporaryURL)
|
try? FileManager.default.removeItem(at: temporaryURL)
|
||||||
return existingURL
|
return existingURL
|
||||||
}
|
}
|
||||||
|
|
||||||
try? FileManager.default.removeItem(at: targetURL)
|
if FileManager.default.fileExists(atPath: targetURL.path) {
|
||||||
try FileManager.default.moveItem(at: temporaryURL, to: targetURL)
|
_ = try FileManager.default.replaceItemAt(targetURL, withItemAt: temporaryURL)
|
||||||
|
} else {
|
||||||
|
try FileManager.default.moveItem(at: temporaryURL, to: targetURL)
|
||||||
|
}
|
||||||
|
|
||||||
return targetURL
|
return targetURL
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,20 +5,17 @@ import Virtualization
|
||||||
import DiskImageKit
|
import DiskImageKit
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/// One immutable complete disk file used by a stacked disk.
|
/// The logical block layout exposed by a disk image.
|
||||||
///
|
struct DiskImageBlockLayout {
|
||||||
/// This is a reconstructed base disk or published ASIF overlay, not an OCI
|
let blockSize: UInt64
|
||||||
/// layer or an individual Tart disk chunk.
|
let blockCount: UInt64
|
||||||
struct DiskImageFile {
|
|
||||||
let url: URL
|
|
||||||
let contentDigest: String
|
|
||||||
}
|
}
|
||||||
|
|
||||||
enum DiskImageStackError: Error, Equatable, CustomStringConvertible {
|
enum DiskImageStackError: Error, Equatable, CustomStringConvertible {
|
||||||
case unavailable
|
case unavailable
|
||||||
case writableOverlayAlreadyExists(URL)
|
case writableOverlayAlreadyExists(URL)
|
||||||
case writableOverlayMissing(URL)
|
case writableOverlayMissing(URL)
|
||||||
case invalidGeometry(String)
|
case invalidBlockLayout(String)
|
||||||
case invalidDiskImage(URL, String)
|
case invalidDiskImage(URL, String)
|
||||||
|
|
||||||
var description: String {
|
var description: String {
|
||||||
|
|
@ -29,7 +26,7 @@ enum DiskImageStackError: Error, Equatable, CustomStringConvertible {
|
||||||
"writable overlay already exists: \(url.path)"
|
"writable overlay already exists: \(url.path)"
|
||||||
case .writableOverlayMissing(let url):
|
case .writableOverlayMissing(let url):
|
||||||
"writable overlay is missing: \(url.path)"
|
"writable overlay is missing: \(url.path)"
|
||||||
case .invalidGeometry(let reason):
|
case .invalidBlockLayout(let reason):
|
||||||
reason
|
reason
|
||||||
case .invalidDiskImage(let url, let reason):
|
case .invalidDiskImage(let url, let reason):
|
||||||
"\(reason): \(url.path)"
|
"\(reason): \(url.path)"
|
||||||
|
|
@ -38,16 +35,66 @@ enum DiskImageStackError: Error, Equatable, CustomStringConvertible {
|
||||||
}
|
}
|
||||||
|
|
||||||
struct DiskImageStack {
|
struct DiskImageStack {
|
||||||
/// DiskImageKit-ready paths and geometry after Tart disk chunks have been
|
/// DiskImageKit-ready paths and block layout after Tart disk chunks have been
|
||||||
/// reconstructed into complete immutable files. The writable overlay stays
|
/// reconstructed into complete immutable files. The writable overlay stays
|
||||||
/// private to one VM.
|
/// private to one VM.
|
||||||
let base: DiskImageFile
|
let baseURL: URL
|
||||||
let baseFormat: DiskImageFormat
|
let baseFormat: DiskImageFormat
|
||||||
let overlays: [DiskImageFile]
|
let immutableOverlayURLs: [URL]
|
||||||
let writableOverlayURL: URL
|
let writableOverlayURL: URL
|
||||||
let blockSize: UInt64
|
let blockSize: UInt64
|
||||||
let blockCount: UInt64
|
let blockCount: UInt64
|
||||||
|
|
||||||
|
/// Reads a disk image's current block layout without resolving or validating a
|
||||||
|
/// whole stack. This is used for the VM's private writable overlay, whose
|
||||||
|
/// size may be newer than the pinned immutable parent manifest.
|
||||||
|
static func diskImageBlockLayout(at url: URL) throws -> DiskImageBlockLayout {
|
||||||
|
#if canImport(DiskImageKit)
|
||||||
|
if #available(macOS 27.0, *) {
|
||||||
|
let image = try DiskImage(opening: .open(url: url, mode: .readOnly))
|
||||||
|
return DiskImageBlockLayout(
|
||||||
|
blockSize: UInt64(image.blockSize.rawValue),
|
||||||
|
blockCount: UInt64(image.blockCount)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
throw DiskImageStackError.unavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
static func baseBlockLayout(
|
||||||
|
at url: URL,
|
||||||
|
expectedFormat: DiskImageFormat
|
||||||
|
) throws -> DiskImageBlockLayout {
|
||||||
|
#if canImport(DiskImageKit)
|
||||||
|
if #available(macOS 27.0, *) {
|
||||||
|
let image = try DiskImage(opening: .open(url: url, mode: .readOnly))
|
||||||
|
let matchesFormat = switch expectedFormat {
|
||||||
|
case .raw:
|
||||||
|
image.format == .raw
|
||||||
|
case .asif:
|
||||||
|
image.format == .asif
|
||||||
|
}
|
||||||
|
guard matchesFormat else {
|
||||||
|
throw DiskImageStackError.invalidDiskImage(url, "base disk format does not match")
|
||||||
|
}
|
||||||
|
guard image.layerType == nil, image.parentUUID == nil else {
|
||||||
|
throw DiskImageStackError.invalidDiskImage(url, "base disk must not be an overlay")
|
||||||
|
}
|
||||||
|
if expectedFormat == .asif && image.layerUUID == nil {
|
||||||
|
throw DiskImageStackError.invalidDiskImage(url, "ASIF base disk is missing a UUID")
|
||||||
|
}
|
||||||
|
|
||||||
|
return DiskImageBlockLayout(
|
||||||
|
blockSize: UInt64(image.blockSize.rawValue),
|
||||||
|
blockCount: UInt64(image.blockCount)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
throw DiskImageStackError.unavailable
|
||||||
|
}
|
||||||
|
|
||||||
func createWritableOverlay() throws {
|
func createWritableOverlay() throws {
|
||||||
#if canImport(DiskImageKit)
|
#if canImport(DiskImageKit)
|
||||||
if #available(macOS 27.0, *) {
|
if #available(macOS 27.0, *) {
|
||||||
|
|
@ -68,12 +115,14 @@ struct DiskImageStack {
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeAttachment(
|
func makeAttachment(
|
||||||
|
readOnly: Bool = false,
|
||||||
cachingMode: VZDiskImageCachingMode = .automatic,
|
cachingMode: VZDiskImageCachingMode = .automatic,
|
||||||
synchronizationMode: VZDiskImageSynchronizationMode = .full
|
synchronizationMode: VZDiskImageSynchronizationMode = .full
|
||||||
) throws -> VZDiskImageStorageDeviceAttachment {
|
) throws -> VZStorageDeviceAttachment {
|
||||||
#if canImport(DiskImageKit)
|
#if canImport(DiskImageKit)
|
||||||
if #available(macOS 27.0, *) {
|
if #available(macOS 27.0, *) {
|
||||||
return try attachmentWithDiskImageKit(
|
return try attachmentWithDiskImageKit(
|
||||||
|
readOnly: readOnly,
|
||||||
cachingMode: cachingMode,
|
cachingMode: cachingMode,
|
||||||
synchronizationMode: synchronizationMode
|
synchronizationMode: synchronizationMode
|
||||||
)
|
)
|
||||||
|
|
@ -108,6 +157,7 @@ struct DiskImageStack {
|
||||||
|
|
||||||
@available(macOS 27.0, *)
|
@available(macOS 27.0, *)
|
||||||
private func attachmentWithDiskImageKit(
|
private func attachmentWithDiskImageKit(
|
||||||
|
readOnly: Bool,
|
||||||
cachingMode: VZDiskImageCachingMode,
|
cachingMode: VZDiskImageCachingMode,
|
||||||
synchronizationMode: VZDiskImageSynchronizationMode
|
synchronizationMode: VZDiskImageSynchronizationMode
|
||||||
) throws -> VZDiskImageStorageDeviceAttachment {
|
) throws -> VZDiskImageStorageDeviceAttachment {
|
||||||
|
|
@ -118,7 +168,7 @@ struct DiskImageStack {
|
||||||
let parent = try validatedParentImage()
|
let parent = try validatedParentImage()
|
||||||
let writableOverlay = try openOverlay(
|
let writableOverlay = try openOverlay(
|
||||||
at: writableOverlayURL,
|
at: writableOverlayURL,
|
||||||
mode: .readWrite
|
mode: readOnly ? .readOnly : .readWrite
|
||||||
)
|
)
|
||||||
let stackedImage = try append(writableOverlay, to: parent, at: writableOverlayURL)
|
let stackedImage = try append(writableOverlay, to: parent, at: writableOverlayURL)
|
||||||
try validateAppendedOverlay(stackedImage, at: writableOverlayURL)
|
try validateAppendedOverlay(stackedImage, at: writableOverlayURL)
|
||||||
|
|
@ -133,7 +183,7 @@ struct DiskImageStack {
|
||||||
@available(macOS 27.0, *)
|
@available(macOS 27.0, *)
|
||||||
private func growWritableOverlayWithDiskImageKit(toBlockCount blockCount: UInt64) throws {
|
private func growWritableOverlayWithDiskImageKit(toBlockCount blockCount: UInt64) throws {
|
||||||
guard blockCount > 0, let desiredBlockCount = Int(exactly: blockCount) else {
|
guard blockCount > 0, let desiredBlockCount = Int(exactly: blockCount) else {
|
||||||
throw DiskImageStackError.invalidGeometry("invalid stacked disk block count \(blockCount)")
|
throw DiskImageStackError.invalidBlockLayout("invalid stacked disk block count \(blockCount)")
|
||||||
}
|
}
|
||||||
|
|
||||||
let parent = try validatedParentImage()
|
let parent = try validatedParentImage()
|
||||||
|
|
@ -160,31 +210,29 @@ struct DiskImageStack {
|
||||||
private func validatedParentImage() throws -> DiskImage {
|
private func validatedParentImage() throws -> DiskImage {
|
||||||
let expectedBlockSize = try diskImageBlockSize(blockSize)
|
let expectedBlockSize = try diskImageBlockSize(blockSize)
|
||||||
guard blockCount > 0, let expectedBlockCount = Int(exactly: blockCount) else {
|
guard blockCount > 0, let expectedBlockCount = Int(exactly: blockCount) else {
|
||||||
throw DiskImageStackError.invalidGeometry("invalid stacked disk block count \(blockCount)")
|
throw DiskImageStackError.invalidBlockLayout("invalid stacked disk block count \(blockCount)")
|
||||||
}
|
}
|
||||||
|
|
||||||
try verifyContentDigest(base)
|
let baseImage = try DiskImage(opening: .open(url: baseURL, mode: .readOnly))
|
||||||
let baseImage = try DiskImage(opening: .open(url: base.url, mode: .readOnly))
|
try validateBase(baseImage, at: baseURL, expectedFormat: baseFormat)
|
||||||
try validateBase(baseImage, at: base.url, expectedFormat: baseFormat)
|
|
||||||
|
|
||||||
var image = baseImage
|
var image = baseImage
|
||||||
|
|
||||||
for overlay in overlays {
|
for overlayURL in immutableOverlayURLs {
|
||||||
let openedOverlay = try openOverlay(
|
let openedOverlay = try openOverlay(
|
||||||
at: overlay.url,
|
at: overlayURL,
|
||||||
expectedDigest: overlay.contentDigest,
|
|
||||||
mode: .readOnly
|
mode: .readOnly
|
||||||
)
|
)
|
||||||
let stackedImage = try append(openedOverlay, to: image, at: overlay.url)
|
let stackedImage = try append(openedOverlay, to: image, at: overlayURL)
|
||||||
try validateAppendedOverlay(stackedImage, at: overlay.url)
|
try validateAppendedOverlay(stackedImage, at: overlayURL)
|
||||||
image = stackedImage
|
image = stackedImage
|
||||||
}
|
}
|
||||||
|
|
||||||
guard image.blockSize == expectedBlockSize else {
|
guard image.blockSize == expectedBlockSize else {
|
||||||
throw DiskImageStackError.invalidGeometry("immutable disk stack does not match manifest block size")
|
throw DiskImageStackError.invalidBlockLayout("immutable disk stack does not match manifest block size")
|
||||||
}
|
}
|
||||||
guard image.blockCount == expectedBlockCount else {
|
guard image.blockCount == expectedBlockCount else {
|
||||||
throw DiskImageStackError.invalidGeometry("immutable disk stack does not match manifest block count")
|
throw DiskImageStackError.invalidBlockLayout("immutable disk stack does not match manifest block count")
|
||||||
}
|
}
|
||||||
|
|
||||||
return image
|
return image
|
||||||
|
|
@ -216,13 +264,8 @@ struct DiskImageStack {
|
||||||
@available(macOS 27.0, *)
|
@available(macOS 27.0, *)
|
||||||
private func openOverlay(
|
private func openOverlay(
|
||||||
at url: URL,
|
at url: URL,
|
||||||
expectedDigest: String? = nil,
|
|
||||||
mode: OpenConfiguration.Mode
|
mode: OpenConfiguration.Mode
|
||||||
) throws -> DiskImage {
|
) throws -> DiskImage {
|
||||||
if let expectedDigest {
|
|
||||||
try verifyContentDigest(DiskImageFile(url: url, contentDigest: expectedDigest))
|
|
||||||
}
|
|
||||||
|
|
||||||
let image = try DiskImage(opening: .open(url: url, mode: mode))
|
let image = try DiskImage(opening: .open(url: url, mode: mode))
|
||||||
guard image.format == .asif else {
|
guard image.format == .asif else {
|
||||||
throw DiskImageStackError.invalidDiskImage(url, "overlay must use ASIF format")
|
throw DiskImageStackError.invalidDiskImage(url, "overlay must use ASIF format")
|
||||||
|
|
@ -247,17 +290,10 @@ struct DiskImageStack {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@available(macOS 27.0, *)
|
|
||||||
private func verifyContentDigest(_ diskImage: DiskImageFile) throws {
|
|
||||||
guard try Digest.hash(diskImage.url) == diskImage.contentDigest else {
|
|
||||||
throw DiskImageStackError.invalidDiskImage(diskImage.url, "disk image content digest does not match")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@available(macOS 27.0, *)
|
@available(macOS 27.0, *)
|
||||||
private func diskImageBlockSize(_ value: UInt64) throws -> DiskImage.BlockSize {
|
private func diskImageBlockSize(_ value: UInt64) throws -> DiskImage.BlockSize {
|
||||||
guard let intValue = Int(exactly: value), let blockSize = DiskImage.BlockSize(rawValue: intValue) else {
|
guard let intValue = Int(exactly: value), let blockSize = DiskImage.BlockSize(rawValue: intValue) else {
|
||||||
throw DiskImageStackError.invalidGeometry("unsupported stacked disk block size \(value)")
|
throw DiskImageStackError.invalidBlockLayout("unsupported stacked disk block size \(value)")
|
||||||
}
|
}
|
||||||
|
|
||||||
return blockSize
|
return blockSize
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,8 @@ enum DigestError: Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
class Digest {
|
class Digest {
|
||||||
|
private static let fileBufferSize = 4 * 1024 * 1024
|
||||||
|
|
||||||
var hash: SHA256 = SHA256()
|
var hash: SHA256 = SHA256()
|
||||||
|
|
||||||
func update(_ data: Data) {
|
func update(_ data: Data) {
|
||||||
|
|
@ -22,7 +24,10 @@ class Digest {
|
||||||
}
|
}
|
||||||
|
|
||||||
static func hash(_ url: URL) throws -> String {
|
static func hash(_ url: URL) throws -> String {
|
||||||
hash(try Data(contentsOf: url))
|
let file = try FileHandle(forReadingFrom: url)
|
||||||
|
defer { try? file.close() }
|
||||||
|
|
||||||
|
return try hashContents(from: file)
|
||||||
}
|
}
|
||||||
|
|
||||||
static func hash(_ url: URL, offset: UInt64, size: UInt64) throws -> String {
|
static func hash(_ url: URL, offset: UInt64, size: UInt64) throws -> String {
|
||||||
|
|
@ -36,20 +41,53 @@ class Digest {
|
||||||
throw DigestError.InvalidOffset
|
throw DigestError.InvalidOffset
|
||||||
}
|
}
|
||||||
|
|
||||||
if (offset + size) > fileSize {
|
if size > fileSize - offset {
|
||||||
throw DigestError.InvalidSize
|
throw DigestError.InvalidSize
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read a chunk of size ``size`` at offset ``offset``
|
// Read the requested range incrementally and calculate its digest.
|
||||||
// and calculate it's digest
|
|
||||||
let fh = try FileHandle(forReadingFrom: url)
|
let fh = try FileHandle(forReadingFrom: url)
|
||||||
defer { try! fh.close() }
|
defer { try? fh.close() }
|
||||||
|
|
||||||
try fh.seek(toOffset: offset)
|
try fh.seek(toOffset: offset)
|
||||||
|
|
||||||
let data = try fh.read(upToCount: Int(size))!
|
return try hashContents(from: fh, size: size)
|
||||||
|
}
|
||||||
|
|
||||||
return hash(data)
|
/// Streams a file into SHA-256 while keeping Foundation's temporary read
|
||||||
|
/// buffers scoped to one chunk.
|
||||||
|
private static func hashContents(from file: FileHandle, size: UInt64? = nil) throws -> String {
|
||||||
|
let digest = Digest()
|
||||||
|
var remaining = size
|
||||||
|
|
||||||
|
while remaining.map({ $0 > 0 }) ?? true {
|
||||||
|
let didRead = try autoreleasepool { () throws -> Bool in
|
||||||
|
let count = remaining.map {
|
||||||
|
Int(min(UInt64(fileBufferSize), $0))
|
||||||
|
} ?? fileBufferSize
|
||||||
|
|
||||||
|
guard let data = try file.read(upToCount: count), !data.isEmpty else {
|
||||||
|
if remaining != nil {
|
||||||
|
throw DigestError.InvalidSize
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
digest.update(data)
|
||||||
|
if let bytesRemaining = remaining {
|
||||||
|
remaining = bytesRemaining - UInt64(data.count)
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if !didRead {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return digest.finalize()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import Foundation
|
import Foundation
|
||||||
|
|
||||||
protocol Disk {
|
protocol Disk {
|
||||||
static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
|
static func push(diskURL: URL, mediaType: String, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer]
|
||||||
static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
|
static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,14 @@ class DiskV2: Disk {
|
||||||
private static let holeGranularityBytes = 4 * 1024 * 1024
|
private static let holeGranularityBytes = 4 * 1024 * 1024
|
||||||
private static let zeroChunk = Data(count: holeGranularityBytes)
|
private static let zeroChunk = Data(count: holeGranularityBytes)
|
||||||
|
|
||||||
static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] {
|
static func push(
|
||||||
|
diskURL: URL,
|
||||||
|
mediaType: String,
|
||||||
|
registry: Registry,
|
||||||
|
chunkSizeMb: Int,
|
||||||
|
concurrency: UInt,
|
||||||
|
progress: Progress
|
||||||
|
) async throws -> [OCIManifestLayer] {
|
||||||
var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []
|
var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = []
|
||||||
|
|
||||||
// Open the disk file
|
// Open the disk file
|
||||||
|
|
@ -63,7 +70,7 @@ class DiskV2: Disk {
|
||||||
progress.completedUnitCount += Int64(data.count)
|
progress.completedUnitCount += Int64(data.count)
|
||||||
|
|
||||||
return (index, OCIManifestLayer(
|
return (index, OCIManifestLayer(
|
||||||
mediaType: diskV2MediaType,
|
mediaType: mediaType,
|
||||||
size: compressedData.count,
|
size: compressedData.count,
|
||||||
digest: compressedDataDigest,
|
digest: compressedDataDigest,
|
||||||
uncompressedSize: UInt64(data.count),
|
uncompressedSize: UInt64(data.count),
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
|
||||||
|
|
||||||
// Initialize the virtual machine and its configuration
|
// Initialize the virtual machine and its configuration
|
||||||
self.network = network
|
self.network = network
|
||||||
configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
|
configuration = try Self.craftConfiguration(vmDir: vmDir,
|
||||||
nvramURL: vmDir.nvramURL, vmConfig: config,
|
nvramURL: vmDir.nvramURL, vmConfig: config,
|
||||||
network: network, additionalStorageDevices: additionalStorageDevices,
|
network: network, additionalStorageDevices: additionalStorageDevices,
|
||||||
directorySharingDevices: directorySharingDevices,
|
directorySharingDevices: directorySharingDevices,
|
||||||
|
|
@ -196,7 +196,8 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
|
||||||
|
|
||||||
// Initialize the virtual machine and its configuration
|
// Initialize the virtual machine and its configuration
|
||||||
self.network = network
|
self.network = network
|
||||||
configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
|
configuration = try Self.craftConfiguration(vmDir: vmDir,
|
||||||
|
nvramURL: vmDir.nvramURL,
|
||||||
vmConfig: config, network: network,
|
vmConfig: config, network: network,
|
||||||
additionalStorageDevices: additionalStorageDevices,
|
additionalStorageDevices: additionalStorageDevices,
|
||||||
directorySharingDevices: directorySharingDevices,
|
directorySharingDevices: directorySharingDevices,
|
||||||
|
|
@ -312,7 +313,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
|
||||||
}
|
}
|
||||||
|
|
||||||
static func craftConfiguration(
|
static func craftConfiguration(
|
||||||
diskURL: URL,
|
vmDir: VMDirectory,
|
||||||
nvramURL: URL,
|
nvramURL: URL,
|
||||||
vmConfig: VMConfig,
|
vmConfig: VMConfig,
|
||||||
network: Network = NetworkShared(),
|
network: Network = NetworkShared(),
|
||||||
|
|
@ -404,15 +405,25 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Storage
|
// Storage
|
||||||
let attachment = try VZDiskImageStorageDeviceAttachment(
|
// When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1]
|
||||||
url: diskURL,
|
//
|
||||||
readOnly: false,
|
// [1]: https://github.com/cirruslabs/tart/pull/675
|
||||||
// When not specified, use "cached" caching mode for Linux VMs to prevent file-system corruption[1]
|
let cachingMode = caching ?? (vmConfig.os == .linux ? .cached : .automatic)
|
||||||
//
|
let attachment: VZStorageDeviceAttachment
|
||||||
// [1]: https://github.com/cirruslabs/tart/pull/675
|
if vmDir.isStackedVM {
|
||||||
cachingMode: caching ?? (vmConfig.os == .linux ? .cached : .automatic),
|
attachment = try vmDir.diskImageStack().makeAttachment(
|
||||||
synchronizationMode: sync
|
readOnly: false,
|
||||||
)
|
cachingMode: cachingMode,
|
||||||
|
synchronizationMode: sync
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
attachment = try VZDiskImageStorageDeviceAttachment(
|
||||||
|
url: vmDir.diskURL,
|
||||||
|
readOnly: false,
|
||||||
|
cachingMode: cachingMode,
|
||||||
|
synchronizationMode: sync
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)]
|
var devices: [VZStorageDeviceConfiguration] = [VZVirtioBlockDeviceConfiguration(attachment: attachment)]
|
||||||
devices.append(contentsOf: additionalStorageDevices)
|
devices.append(contentsOf: additionalStorageDevices)
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,10 @@ fileprivate let permissions = FilePermissions(rawValue: 0o644)
|
||||||
// [2]: https://developer.apple.com/documentation/compression/algorithm/lzfse
|
// [2]: https://developer.apple.com/documentation/compression/algorithm/lzfse
|
||||||
extension VMDirectory {
|
extension VMDirectory {
|
||||||
func exportToArchive(path: String) throws {
|
func exportToArchive(path: String) throws {
|
||||||
|
guard !isStackedVM && !isStackedCachedImage else {
|
||||||
|
throw RuntimeError.ExportFailed("exporting stacked VMs is not supported yet")
|
||||||
|
}
|
||||||
|
|
||||||
guard let fileStream = ArchiveByteStream.fileStream(
|
guard let fileStream = ArchiveByteStream.fileStream(
|
||||||
path: FilePath(path),
|
path: FilePath(path),
|
||||||
mode: .writeOnly,
|
mode: .writeOnly,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,125 @@
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
extension VMDirectory {
|
||||||
|
func diskImageStack(contentStore providedStore: ContentStore? = nil) throws -> DiskImageStack {
|
||||||
|
let manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||||
|
let base: TartDiskFileGroup
|
||||||
|
let overlays: [TartDiskFileGroup]
|
||||||
|
|
||||||
|
switch try manifest.tartDiskRepresentation() {
|
||||||
|
case .flat(let pinnedBase) where pinnedBase.contentDigest != nil:
|
||||||
|
base = pinnedBase
|
||||||
|
overlays = []
|
||||||
|
case .stacked(let stackedBase, let stackedOverlays):
|
||||||
|
base = stackedBase
|
||||||
|
overlays = stackedOverlays
|
||||||
|
default:
|
||||||
|
throw RuntimeError.VMConfigurationError("VM is missing its disk image metadata")
|
||||||
|
}
|
||||||
|
guard let blockSize = manifest.diskBlockSize(),
|
||||||
|
let blockCount = manifest.diskBlockCount() else {
|
||||||
|
throw DiskImageStackError.invalidBlockLayout("disk image metadata is missing block layout")
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentStore = try providedStore ?? ContentStore()
|
||||||
|
let baseURL = try diskImageURL(for: base, contentStore: contentStore)
|
||||||
|
let immutableOverlayURLs = try overlays.map { try diskImageURL(for: $0, contentStore: contentStore) }
|
||||||
|
let config = try VMConfig(fromURL: configURL)
|
||||||
|
|
||||||
|
return DiskImageStack(
|
||||||
|
baseURL: baseURL,
|
||||||
|
baseFormat: config.diskFormat,
|
||||||
|
immutableOverlayURLs: immutableOverlayURLs,
|
||||||
|
writableOverlayURL: overlayURL,
|
||||||
|
blockSize: blockSize,
|
||||||
|
blockCount: blockCount
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneStacked(
|
||||||
|
to destination: VMDirectory,
|
||||||
|
copyWritableOverlay: Bool,
|
||||||
|
generateMAC: Bool,
|
||||||
|
contentStore: ContentStore? = nil
|
||||||
|
) throws {
|
||||||
|
try FileManager.default.copyItem(at: configURL, to: destination.configURL)
|
||||||
|
try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL)
|
||||||
|
try FileManager.default.copyItem(at: manifestURL, to: destination.manifestURL)
|
||||||
|
|
||||||
|
if copyWritableOverlay {
|
||||||
|
try FileManager.default.copyItem(at: overlayURL, to: destination.overlayURL)
|
||||||
|
} else {
|
||||||
|
try destination.diskImageStack(contentStore: contentStore).createWritableOverlay()
|
||||||
|
}
|
||||||
|
|
||||||
|
if generateMAC {
|
||||||
|
try destination.regenerateMACAddress()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneAsStackedBase(
|
||||||
|
to destination: VMDirectory,
|
||||||
|
generateMAC: Bool,
|
||||||
|
contentStore providedStore: ContentStore? = nil
|
||||||
|
) throws {
|
||||||
|
let config = try VMConfig(fromURL: configURL)
|
||||||
|
let blockLayout = try DiskImageStack.baseBlockLayout(at: diskURL, expectedFormat: config.diskFormat)
|
||||||
|
let contentDigest = try Digest.hash(diskURL)
|
||||||
|
let contentStore = try providedStore ?? ContentStore()
|
||||||
|
|
||||||
|
if try contentStore.existingContentURL(for: contentDigest) == nil {
|
||||||
|
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||||
|
do {
|
||||||
|
try FileManager.default.copyItem(at: diskURL, to: temporaryURL)
|
||||||
|
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||||
|
} catch {
|
||||||
|
try? FileManager.default.removeItem(at: temporaryURL)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||||
|
guard case .flat = try manifest.tartDiskRepresentation() else {
|
||||||
|
throw RuntimeError.VMConfigurationError("--stacked cannot use an image that already has a stacked disk")
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let firstDiskIndex = manifest.layers.firstIndex(where: { $0.mediaType == diskV2MediaType }) else {
|
||||||
|
throw OCIManifestValidationError.invalidLayout("manifest must contain at least one disk chunk")
|
||||||
|
}
|
||||||
|
|
||||||
|
var baseAnnotations = manifest.layers[firstDiskIndex].annotations ?? [:]
|
||||||
|
baseAnnotations[diskFileContentDigestAnnotation] = contentDigest
|
||||||
|
manifest.layers[firstDiskIndex].annotations = baseAnnotations
|
||||||
|
let diskSize = blockLayout.blockSize.multipliedReportingOverflow(by: blockLayout.blockCount)
|
||||||
|
guard !diskSize.overflow else {
|
||||||
|
throw DiskImageStackError.invalidBlockLayout("stacked disk block layout overflows UInt64")
|
||||||
|
}
|
||||||
|
var annotations = manifest.annotations ?? [:]
|
||||||
|
annotations[diskBlockSizeAnnotation] = String(blockLayout.blockSize)
|
||||||
|
annotations[uncompressedDiskSizeAnnotation] = String(diskSize.partialValue)
|
||||||
|
manifest.annotations = annotations
|
||||||
|
|
||||||
|
try FileManager.default.copyItem(at: configURL, to: destination.configURL)
|
||||||
|
try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL)
|
||||||
|
try manifest.toJSON().write(to: destination.manifestURL)
|
||||||
|
try destination.diskImageStack(contentStore: contentStore).createWritableOverlay()
|
||||||
|
|
||||||
|
if generateMAC {
|
||||||
|
try destination.regenerateMACAddress()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func diskImageURL(for group: TartDiskFileGroup, contentStore: ContentStore) throws -> URL {
|
||||||
|
guard let contentDigest = group.contentDigest else {
|
||||||
|
throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest")
|
||||||
|
}
|
||||||
|
// Pull/install verifies immutable content before publishing it. Clone and
|
||||||
|
// run use the trusted content-addressed entry without rereading a possibly
|
||||||
|
// very large disk file, matching Tart's existing disk.img behavior.
|
||||||
|
guard let url = try contentStore.contentURLIfPresent(for: contentDigest) else {
|
||||||
|
throw RuntimeError.VMMissingFiles("VM is missing cached disk content \(contentDigest)")
|
||||||
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -6,7 +6,6 @@ let legacyDiskV1MediaType = "application/vnd.cirruslabs.tart.disk.v1"
|
||||||
|
|
||||||
enum OCIError: Error {
|
enum OCIError: Error {
|
||||||
case ShouldBeExactlyOneLayer
|
case ShouldBeExactlyOneLayer
|
||||||
case ShouldBeAtLeastOneLayer
|
|
||||||
case FailedToCreateVmFile
|
case FailedToCreateVmFile
|
||||||
case LayerIsMissingUncompressedSizeAnnotation
|
case LayerIsMissingUncompressedSizeAnnotation
|
||||||
case LayerIsMissingUncompressedDigestAnnotation
|
case LayerIsMissingUncompressedDigestAnnotation
|
||||||
|
|
@ -14,7 +13,7 @@ enum OCIError: Error {
|
||||||
|
|
||||||
extension VMDirectory {
|
extension VMDirectory {
|
||||||
func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
|
func pullFromRegistry(registry: Registry, manifest: OCIManifest, concurrency: UInt, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws {
|
||||||
// Pull VM's config file layer and re-serialize it into a config file
|
// Pull VM's config file layer and store it as the local config file.
|
||||||
let configLayers = manifest.layers.filter {
|
let configLayers = manifest.layers.filter {
|
||||||
$0.mediaType == configMediaType
|
$0.mediaType == configMediaType
|
||||||
}
|
}
|
||||||
|
|
@ -30,17 +29,22 @@ extension VMDirectory {
|
||||||
}
|
}
|
||||||
try configFile.close()
|
try configFile.close()
|
||||||
|
|
||||||
// Pull VM's disk layers and decompress them into a disk file
|
// Pull VM's disk chunks and decompress them into complete disk files.
|
||||||
if manifest.layers.contains(where: { $0.mediaType == legacyDiskV1MediaType }) {
|
if manifest.layers.contains(where: { $0.mediaType == legacyDiskV1MediaType }) {
|
||||||
throw RuntimeError.Generic("Pulling OCI images with legacy disk media type \(legacyDiskV1MediaType) is no longer supported, please re-push the image using a current Tart version")
|
throw RuntimeError.Generic("Pulling OCI images with legacy disk media type \(legacyDiskV1MediaType) is no longer supported, please re-push the image using a current Tart version")
|
||||||
}
|
}
|
||||||
|
|
||||||
let layers = manifest.layers.filter { $0.mediaType == diskV2MediaType }
|
let diskRepresentation = try manifest.tartDiskRepresentation()
|
||||||
if layers.isEmpty {
|
let diskChunks: [OCIManifestLayer]
|
||||||
throw OCIError.ShouldBeAtLeastOneLayer
|
|
||||||
|
switch diskRepresentation {
|
||||||
|
case .flat(let base):
|
||||||
|
diskChunks = base.chunks
|
||||||
|
case .stacked(let base, let overlays):
|
||||||
|
diskChunks = base.chunks + overlays.flatMap(\.chunks)
|
||||||
}
|
}
|
||||||
|
|
||||||
let diskCompressedSize = layers.map { Int64($0.size) }.reduce(0, +)
|
let diskCompressedSize = diskChunks.map { Int64($0.size) }.reduce(0, +)
|
||||||
OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute(
|
OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute(
|
||||||
key: "compressed_disk_size_bytes",
|
key: "compressed_disk_size_bytes",
|
||||||
value: .int(Int(diskCompressedSize))
|
value: .int(Int(diskCompressedSize))
|
||||||
|
|
@ -53,19 +57,42 @@ extension VMDirectory {
|
||||||
ProgressObserver(progress).log(defaultLogger)
|
ProgressObserver(progress).log(defaultLogger)
|
||||||
|
|
||||||
do {
|
do {
|
||||||
try await DiskV2.pull(registry: registry, diskLayers: layers, diskURL: diskURL,
|
switch diskRepresentation {
|
||||||
concurrency: concurrency, progress: progress,
|
case .flat(let base):
|
||||||
localLayerCache: localLayerCache,
|
try await DiskV2.pull(registry: registry, diskLayers: base.chunks, diskURL: diskURL,
|
||||||
deduplicate: deduplicate)
|
concurrency: concurrency, progress: progress,
|
||||||
|
localLayerCache: localLayerCache,
|
||||||
|
deduplicate: deduplicate)
|
||||||
|
|
||||||
|
if deduplicate, let llc = localLayerCache {
|
||||||
|
// set custom attribute to remember deduplicated bytes
|
||||||
|
diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
|
||||||
|
}
|
||||||
|
case .stacked(let base, let overlays):
|
||||||
|
// The deterministic resumable directory may contain a partial
|
||||||
|
// disk.img from an interrupted pull while this tag was standalone. A
|
||||||
|
// cached stacked image must not retain that file or it is mistaken for
|
||||||
|
// a standalone VM after the pull is moved into cache.
|
||||||
|
if FileManager.default.fileExists(atPath: diskURL.path) {
|
||||||
|
try FileManager.default.removeItem(at: diskURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
|
||||||
|
for group in [base] + overlays {
|
||||||
|
_ = try await pullDiskFile(
|
||||||
|
registry: registry,
|
||||||
|
group: group,
|
||||||
|
contentStore: contentStore,
|
||||||
|
concurrency: concurrency,
|
||||||
|
progress: progress
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch let error where error is FilterError {
|
} catch let error where error is FilterError {
|
||||||
throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
|
throw RuntimeError.PullFailed("failed to decompress disk: \(error.localizedDescription)")
|
||||||
}
|
}
|
||||||
|
|
||||||
if deduplicate, let llc = localLayerCache {
|
|
||||||
// set custom attribute to remember deduplicated bytes
|
|
||||||
diskURL.setDeduplicatedBytes(llc.deduplicatedBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pull VM's NVRAM file layer and store it in an NVRAM file
|
// Pull VM's NVRAM file layer and store it in an NVRAM file
|
||||||
defaultLogger.appendNewLine("pulling NVRAM...")
|
defaultLogger.appendNewLine("pulling NVRAM...")
|
||||||
|
|
||||||
|
|
@ -83,12 +110,47 @@ extension VMDirectory {
|
||||||
try nvram.write(contentsOf: data)
|
try nvram.write(contentsOf: data)
|
||||||
}
|
}
|
||||||
try nvram.close()
|
try nvram.close()
|
||||||
|
|
||||||
// Serialize VM's manifest to enable better deduplication on subsequent "tart pull"'s
|
|
||||||
try manifest.toJSON().write(to: manifestURL)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, concurrency: UInt, labels: [String: String] = [:]) async throws -> RemoteName {
|
/// Reconstructs one complete immutable base disk or published ASIF overlay
|
||||||
|
/// from its Tart disk chunks, unless the shared content store already has a
|
||||||
|
/// verified copy.
|
||||||
|
private func pullDiskFile(
|
||||||
|
registry: Registry,
|
||||||
|
group: TartDiskFileGroup,
|
||||||
|
contentStore: ContentStore,
|
||||||
|
concurrency: UInt,
|
||||||
|
progress: Progress
|
||||||
|
) async throws -> URL {
|
||||||
|
guard let contentDigest = group.contentDigest else {
|
||||||
|
throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pulls for the same semantic disk file share a stable resumable path so
|
||||||
|
// DiskV2 can resume after a transient failure. Serialize writers before
|
||||||
|
// rechecking the final entry to avoid racing on that shared path.
|
||||||
|
let lock = try FileLock(lockURL: contentStore.lockURL(for: contentDigest))
|
||||||
|
try lock.lock()
|
||||||
|
defer { try? lock.unlock() }
|
||||||
|
|
||||||
|
if let existingURL = try contentStore.existingContentURL(for: contentDigest) {
|
||||||
|
progress.completedUnitCount += group.chunks.reduce(0) { $0 + Int64($1.size) }
|
||||||
|
return existingURL
|
||||||
|
}
|
||||||
|
|
||||||
|
let resumableURL = try contentStore.resumableContentURL(for: contentDigest)
|
||||||
|
try await DiskV2.pull(
|
||||||
|
registry: registry,
|
||||||
|
diskLayers: group.chunks,
|
||||||
|
diskURL: resumableURL,
|
||||||
|
concurrency: concurrency,
|
||||||
|
progress: progress
|
||||||
|
)
|
||||||
|
|
||||||
|
return try contentStore.install(resumableURL, contentDigest: contentDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pushToRegistry(registry: Registry, references: [String], chunkSizeMb: Int, concurrency: UInt, labels: [String: String] = [:]) async throws -> (name: RemoteName, manifest: OCIManifest) {
|
||||||
var layers = Array<OCIManifestLayer>()
|
var layers = Array<OCIManifestLayer>()
|
||||||
|
|
||||||
// Read VM's config and push it as blob
|
// Read VM's config and push it as blob
|
||||||
|
|
@ -102,14 +164,12 @@ extension VMDirectory {
|
||||||
let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb)
|
let configDigest = try await registry.pushBlob(fromData: configJSON, chunkSizeMb: chunkSizeMb)
|
||||||
layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest))
|
layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest))
|
||||||
|
|
||||||
// Compress the disk file as multiple chunks and push them as disk layers
|
let (diskLayers, diskAnnotations) = try await pushDiskLayers(
|
||||||
let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64
|
registry: registry,
|
||||||
|
chunkSizeMb: chunkSizeMb,
|
||||||
defaultLogger.appendNewLine("pushing disk... this will take a while...")
|
concurrency: concurrency
|
||||||
let progress = Progress(totalUnitCount: diskSize)
|
)
|
||||||
ProgressObserver(progress).log(defaultLogger)
|
layers.append(contentsOf: diskLayers)
|
||||||
|
|
||||||
layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: registry, chunkSizeMb: chunkSizeMb, concurrency: concurrency, progress: progress))
|
|
||||||
|
|
||||||
// Read VM's NVRAM and push it as blob
|
// Read VM's NVRAM and push it as blob
|
||||||
defaultLogger.appendNewLine("pushing NVRAM...")
|
defaultLogger.appendNewLine("pushing NVRAM...")
|
||||||
|
|
@ -122,12 +182,13 @@ extension VMDirectory {
|
||||||
let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels)
|
let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels)
|
||||||
let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON()
|
let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON()
|
||||||
let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb)
|
let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb)
|
||||||
let manifest = OCIManifest(
|
var manifest = OCIManifest(
|
||||||
config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
|
config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
|
||||||
layers: layers,
|
layers: layers
|
||||||
uncompressedDiskSize: UInt64(diskSize),
|
|
||||||
uploadDate: Date()
|
|
||||||
)
|
)
|
||||||
|
var annotations = diskAnnotations
|
||||||
|
annotations[uploadTimeAnnotation] = Date().toISO()
|
||||||
|
manifest.annotations = annotations
|
||||||
|
|
||||||
// Manifest
|
// Manifest
|
||||||
for reference in references {
|
for reference in references {
|
||||||
|
|
@ -137,7 +198,160 @@ extension VMDirectory {
|
||||||
}
|
}
|
||||||
|
|
||||||
let pushedReference = Reference(digest: try manifest.digest())
|
let pushedReference = Reference(digest: try manifest.digest())
|
||||||
return RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference)
|
let name = RemoteName(host: registry.host!, namespace: registry.namespace, reference: pushedReference)
|
||||||
|
return (name, manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the disk portion of the manifest. Registry transport is shared
|
||||||
|
/// for standalone and stacked VMs; only their local disk representation
|
||||||
|
/// determines which descriptors need to be uploaded or reused.
|
||||||
|
private func pushDiskLayers(
|
||||||
|
registry: Registry,
|
||||||
|
chunkSizeMb: Int,
|
||||||
|
concurrency: UInt
|
||||||
|
) async throws -> ([OCIManifestLayer], [String: String]) {
|
||||||
|
guard isStackedVM else {
|
||||||
|
let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64
|
||||||
|
defaultLogger.appendNewLine("pushing disk... this will take a while...")
|
||||||
|
let progress = Progress(totalUnitCount: diskSize)
|
||||||
|
ProgressObserver(progress).log(defaultLogger)
|
||||||
|
|
||||||
|
let layers = try await DiskV2.push(
|
||||||
|
diskURL: diskURL,
|
||||||
|
mediaType: diskV2MediaType,
|
||||||
|
registry: registry,
|
||||||
|
chunkSizeMb: chunkSizeMb,
|
||||||
|
concurrency: concurrency,
|
||||||
|
progress: progress
|
||||||
|
)
|
||||||
|
return (layers, [uncompressedDiskSizeAnnotation: String(diskSize)])
|
||||||
|
}
|
||||||
|
|
||||||
|
let localManifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||||
|
let inheritedGroups: [TartDiskFileGroup]
|
||||||
|
switch try localManifest.tartDiskRepresentation() {
|
||||||
|
case .flat(let base) where base.contentDigest != nil:
|
||||||
|
inheritedGroups = [base]
|
||||||
|
case .stacked(let base, let overlays):
|
||||||
|
inheritedGroups = [base] + overlays
|
||||||
|
default:
|
||||||
|
throw RuntimeError.VMConfigurationError("stacked VM is missing a pinned disk stack")
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
var layers: [OCIManifestLayer] = []
|
||||||
|
for group in inheritedGroups {
|
||||||
|
layers.append(contentsOf: try await descriptorsForCachedDiskFile(
|
||||||
|
group,
|
||||||
|
contentStore: contentStore,
|
||||||
|
registry: registry,
|
||||||
|
chunkSizeMb: chunkSizeMb,
|
||||||
|
concurrency: concurrency
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the snapshot out of startup GC while this potentially long push
|
||||||
|
// hashes, uploads, and inspects it.
|
||||||
|
let frozenOverlayDirectory = try VMDirectory.temporary()
|
||||||
|
let frozenOverlayLock = try FileLock(lockURL: frozenOverlayDirectory.baseURL)
|
||||||
|
try frozenOverlayLock.lock()
|
||||||
|
defer {
|
||||||
|
try? frozenOverlayLock.unlock()
|
||||||
|
try? FileManager.default.removeItem(at: frozenOverlayDirectory.baseURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
let frozenOverlayURL = frozenOverlayDirectory.baseURL.appendingPathComponent("overlay.asif")
|
||||||
|
try FileManager.default.copyItem(at: overlayURL, to: frozenOverlayURL)
|
||||||
|
|
||||||
|
let overlaySize = try FileManager.default.attributesOfItem(atPath: frozenOverlayURL.path)[.size] as! Int64
|
||||||
|
defaultLogger.appendNewLine("pushing overlay...")
|
||||||
|
let progress = Progress(totalUnitCount: overlaySize)
|
||||||
|
ProgressObserver(progress).log(defaultLogger)
|
||||||
|
let contentDigest = try Digest.hash(frozenOverlayURL)
|
||||||
|
let chunks = try await DiskV2.push(
|
||||||
|
diskURL: frozenOverlayURL,
|
||||||
|
mediaType: asifOverlayMediaType,
|
||||||
|
registry: registry,
|
||||||
|
chunkSizeMb: chunkSizeMb,
|
||||||
|
concurrency: concurrency,
|
||||||
|
progress: progress
|
||||||
|
)
|
||||||
|
layers.append(contentsOf: annotatedChunks(chunks, kind: .asifOverlay, contentDigest: contentDigest))
|
||||||
|
|
||||||
|
let blockLayout = try DiskImageStack.diskImageBlockLayout(at: frozenOverlayURL)
|
||||||
|
let diskSize = blockLayout.blockSize.multipliedReportingOverflow(by: blockLayout.blockCount)
|
||||||
|
guard !diskSize.overflow else {
|
||||||
|
throw DiskImageStackError.invalidBlockLayout("stacked disk block layout overflows UInt64")
|
||||||
|
}
|
||||||
|
|
||||||
|
var annotations = localManifest.annotations ?? [:]
|
||||||
|
annotations[diskBlockSizeAnnotation] = String(blockLayout.blockSize)
|
||||||
|
annotations[uncompressedDiskSizeAnnotation] = String(diskSize.partialValue)
|
||||||
|
|
||||||
|
return (layers, annotations)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns transport descriptors for an immutable disk file. If the
|
||||||
|
/// target registry lacks the original blobs, recreate them from the local
|
||||||
|
/// content store.
|
||||||
|
private func descriptorsForCachedDiskFile(
|
||||||
|
_ group: TartDiskFileGroup,
|
||||||
|
contentStore: ContentStore,
|
||||||
|
registry: Registry,
|
||||||
|
chunkSizeMb: Int,
|
||||||
|
concurrency: UInt
|
||||||
|
) async throws -> [OCIManifestLayer] {
|
||||||
|
guard let contentDigest = group.contentDigest else {
|
||||||
|
throw RuntimeError.VMConfigurationError("stacked VM is missing a pinned disk file digest")
|
||||||
|
}
|
||||||
|
|
||||||
|
var allChunksExist = true
|
||||||
|
for chunk in group.chunks {
|
||||||
|
if try await !registry.blobExists(chunk.digest) {
|
||||||
|
allChunksExist = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if allChunksExist {
|
||||||
|
return group.chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let contentURL = try contentStore.existingContentURL(for: contentDigest) else {
|
||||||
|
throw RuntimeError.VMMissingFiles("stacked VM is missing cached disk content \(contentDigest)")
|
||||||
|
}
|
||||||
|
let contentSize = try FileManager.default.attributesOfItem(atPath: contentURL.path)[.size] as! Int64
|
||||||
|
let progress = Progress(totalUnitCount: contentSize)
|
||||||
|
let mediaType = group.kind == .base ? diskV2MediaType : asifOverlayMediaType
|
||||||
|
let chunks = try await DiskV2.push(
|
||||||
|
diskURL: contentURL,
|
||||||
|
mediaType: mediaType,
|
||||||
|
registry: registry,
|
||||||
|
chunkSizeMb: chunkSizeMb,
|
||||||
|
concurrency: concurrency,
|
||||||
|
progress: progress
|
||||||
|
)
|
||||||
|
|
||||||
|
return annotatedChunks(chunks, kind: group.kind, contentDigest: contentDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func annotatedChunks(
|
||||||
|
_ chunks: [OCIManifestLayer],
|
||||||
|
kind: TartDiskFileGroup.Kind,
|
||||||
|
contentDigest: String
|
||||||
|
) -> [OCIManifestLayer] {
|
||||||
|
guard !chunks.isEmpty else {
|
||||||
|
return chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
var chunks = chunks
|
||||||
|
var annotations = chunks[0].annotations ?? [:]
|
||||||
|
annotations[diskFileContentDigestAnnotation] = contentDigest
|
||||||
|
if kind == .asifOverlay {
|
||||||
|
annotations[diskFileChunkCountAnnotation] = String(chunks.count)
|
||||||
|
}
|
||||||
|
chunks[0].annotations = annotations
|
||||||
|
|
||||||
|
return chunks
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -142,6 +142,24 @@ struct VMDirectory: Prunable {
|
||||||
layout?.isRunnable == true
|
layout?.isRunnable == true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var isStandalone: Bool {
|
||||||
|
layout == .standalone
|
||||||
|
}
|
||||||
|
|
||||||
|
var isStackedVM: Bool {
|
||||||
|
layout == .stackedLocal
|
||||||
|
}
|
||||||
|
|
||||||
|
var isStackedCachedImage: Bool {
|
||||||
|
layout == .stackedOCIRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shapes that may live in the remote-image cache. A cached stacked image
|
||||||
|
/// has no writable overlay and is intentionally not runnable as a local VM.
|
||||||
|
var isCachedImage: Bool {
|
||||||
|
layout == .standalone || layout == .stackedOCIRecord
|
||||||
|
}
|
||||||
|
|
||||||
func initialize(overwrite: Bool = false) throws {
|
func initialize(overwrite: Bool = false) throws {
|
||||||
if !overwrite && initialized {
|
if !overwrite && initialized {
|
||||||
throw RuntimeError.VMDirectoryAlreadyInitialized("VM directory is already initialized, preventing overwrite")
|
throw RuntimeError.VMDirectoryAlreadyInitialized("VM directory is already initialized, preventing overwrite")
|
||||||
|
|
@ -172,6 +190,20 @@ struct VMDirectory: Prunable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateCachedImage(userFriendlyName: String) throws {
|
||||||
|
if !FileManager.default.fileExists(atPath: baseURL.path) {
|
||||||
|
throw RuntimeError.VMDoesNotExist(name: userFriendlyName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isCachedImage {
|
||||||
|
throw RuntimeError.VMMissingFiles(
|
||||||
|
"cached image is missing files for a supported layout: "
|
||||||
|
+ "standalone requires \(configURL.lastPathComponent), \(diskURL.lastPathComponent) and \(nvramURL.lastPathComponent); "
|
||||||
|
+ "stacked requires \(configURL.lastPathComponent), \(manifestURL.lastPathComponent) and \(nvramURL.lastPathComponent)"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func clone(to: VMDirectory, generateMAC: Bool) throws {
|
func clone(to: VMDirectory, generateMAC: Bool) throws {
|
||||||
try FileManager.default.copyItem(at: configURL, to: to.configURL)
|
try FileManager.default.copyItem(at: configURL, to: to.configURL)
|
||||||
try FileManager.default.copyItem(at: nvramURL, to: to.nvramURL)
|
try FileManager.default.copyItem(at: nvramURL, to: to.nvramURL)
|
||||||
|
|
@ -198,7 +230,26 @@ struct VMDirectory: Prunable {
|
||||||
try vmConfig.save(toURL: configURL)
|
try vmConfig.save(toURL: configURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
func resizeDisk(_ sizeGB: UInt16, format: DiskImageFormat = .raw) throws {
|
func resizeDisk(
|
||||||
|
_ sizeGB: UInt16,
|
||||||
|
format: DiskImageFormat = .raw,
|
||||||
|
contentStore: ContentStore? = nil
|
||||||
|
) throws {
|
||||||
|
if isStackedVM {
|
||||||
|
guard try state() == .Stopped else {
|
||||||
|
throw RuntimeError.VMConfigurationError("VM \"\(name)\" must be stopped before resizing its disk")
|
||||||
|
}
|
||||||
|
|
||||||
|
let stack = try diskImageStack(contentStore: contentStore)
|
||||||
|
let desiredSizeBytes = UInt64(sizeGB) * 1000 * 1000 * 1000
|
||||||
|
guard desiredSizeBytes.isMultiple(of: stack.blockSize) else {
|
||||||
|
throw RuntimeError.InvalidDiskSize("new disk size must align to the stacked disk block size")
|
||||||
|
}
|
||||||
|
|
||||||
|
try stack.growWritableOverlay(toBlockCount: desiredSizeBytes / stack.blockSize)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
let diskExists = FileManager.default.fileExists(atPath: diskURL.path)
|
let diskExists = FileManager.default.fileExists(atPath: diskURL.path)
|
||||||
|
|
||||||
if diskExists {
|
if diskExists {
|
||||||
|
|
@ -332,7 +383,7 @@ struct VMDirectory: Prunable {
|
||||||
}
|
}
|
||||||
|
|
||||||
func allocatedSizeBytes() throws -> Int {
|
func allocatedSizeBytes() throws -> Int {
|
||||||
try configURL.allocatedSizeBytes() + diskURL.allocatedSizeBytes() + nvramURL.allocatedSizeBytes()
|
try configURL.allocatedSizeBytes() + localDiskStorageAllocatedSizeBytes() + nvramURL.allocatedSizeBytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
func allocatedSizeGB() throws -> Int {
|
func allocatedSizeGB() throws -> Int {
|
||||||
|
|
@ -340,7 +391,7 @@ struct VMDirectory: Prunable {
|
||||||
}
|
}
|
||||||
|
|
||||||
func deduplicatedSizeBytes() throws -> Int {
|
func deduplicatedSizeBytes() throws -> Int {
|
||||||
try configURL.deduplicatedSizeBytes() + diskURL.deduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes()
|
try configURL.deduplicatedSizeBytes() + localDiskStorageDeduplicatedSizeBytes() + nvramURL.deduplicatedSizeBytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
func deduplicatedSizeGB() throws -> Int {
|
func deduplicatedSizeGB() throws -> Int {
|
||||||
|
|
@ -348,7 +399,7 @@ struct VMDirectory: Prunable {
|
||||||
}
|
}
|
||||||
|
|
||||||
func sizeBytes() throws -> Int {
|
func sizeBytes() throws -> Int {
|
||||||
try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes()
|
try configURL.sizeBytes() + localDiskStorageSizeBytes() + nvramURL.sizeBytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
func sizeGB() throws -> Int {
|
func sizeGB() throws -> Int {
|
||||||
|
|
@ -356,6 +407,30 @@ struct VMDirectory: Prunable {
|
||||||
}
|
}
|
||||||
|
|
||||||
func diskSizeBytes() throws -> Int {
|
func diskSizeBytes() throws -> Int {
|
||||||
|
if isStackedVM {
|
||||||
|
let blockLayout = try DiskImageStack.diskImageBlockLayout(at: overlayURL)
|
||||||
|
let product = blockLayout.blockSize.multipliedReportingOverflow(by: blockLayout.blockCount)
|
||||||
|
guard !product.overflow, let diskSizeBytes = Int(exactly: product.partialValue) else {
|
||||||
|
throw RuntimeError.VMConfigurationError("VM has invalid stacked disk block layout")
|
||||||
|
}
|
||||||
|
|
||||||
|
return diskSizeBytes
|
||||||
|
}
|
||||||
|
|
||||||
|
if isStackedCachedImage {
|
||||||
|
let manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||||
|
guard let blockSize = manifest.diskBlockSize(),
|
||||||
|
let blockCount = manifest.diskBlockCount() else {
|
||||||
|
throw RuntimeError.VMConfigurationError("VM has invalid stacked disk block layout")
|
||||||
|
}
|
||||||
|
let product = blockSize.multipliedReportingOverflow(by: blockCount)
|
||||||
|
guard !product.overflow, let diskSizeBytes = Int(exactly: product.partialValue) else {
|
||||||
|
throw RuntimeError.VMConfigurationError("VM has invalid stacked disk block layout")
|
||||||
|
}
|
||||||
|
|
||||||
|
return diskSizeBytes
|
||||||
|
}
|
||||||
|
|
||||||
let vmConfig = try VMConfig(fromURL: configURL)
|
let vmConfig = try VMConfig(fromURL: configURL)
|
||||||
|
|
||||||
return switch vmConfig.diskFormat {
|
return switch vmConfig.diskFormat {
|
||||||
|
|
@ -377,4 +452,23 @@ struct VMDirectory: Prunable {
|
||||||
func isExplicitlyPulled() -> Bool {
|
func isExplicitlyPulled() -> Bool {
|
||||||
FileManager.default.fileExists(atPath: explicitlyPulledMark.path)
|
FileManager.default.fileExists(atPath: explicitlyPulledMark.path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private var localDiskStorageURL: URL {
|
||||||
|
isStackedVM ? overlayURL : diskURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cached stacked images own no disk file in their VM directory. Their
|
||||||
|
// immutable disk content lives in the shared content store and must not be
|
||||||
|
// charged to every cached image that references it.
|
||||||
|
private func localDiskStorageAllocatedSizeBytes() throws -> Int {
|
||||||
|
isStackedCachedImage ? 0 : try localDiskStorageURL.allocatedSizeBytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func localDiskStorageDeduplicatedSizeBytes() throws -> Int {
|
||||||
|
isStackedCachedImage ? 0 : try localDiskStorageURL.deduplicatedSizeBytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func localDiskStorageSizeBytes() throws -> Int {
|
||||||
|
isStackedCachedImage ? 0 : try localDiskStorageURL.sizeBytes()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,109 @@ class VMStorageOCI: PrunableStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
func exists(_ name: RemoteName) -> Bool {
|
func exists(_ name: RemoteName) -> Bool {
|
||||||
VMDirectory(baseURL: vmURL(name)).initialized
|
VMDirectory(baseURL: vmURL(name)).isCachedImage
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether clone can use a cached image without pulling. Standalone images keep
|
||||||
|
/// Tart's existing structural check. Stacked cached images cheaply require every
|
||||||
|
/// immutable file with its expected length; explicit pull remains the path
|
||||||
|
/// that hashes content and repairs same-sized corruption.
|
||||||
|
func hasUsableCachedImageForClone(_ name: RemoteName, requireManifest: Bool = false) throws -> Bool {
|
||||||
|
guard exists(name) else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
let vmDir = VMDirectory(baseURL: vmURL(name))
|
||||||
|
if requireManifest && !FileManager.default.fileExists(atPath: vmDir.manifestURL.path) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
guard vmDir.isStackedCachedImage else {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
let manifest = try OCIManifest(fromJSON: Data(contentsOf: vmDir.manifestURL))
|
||||||
|
guard case .stacked(let base, let overlays) = try manifest.tartDiskRepresentation() else {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
for group in [base] + overlays {
|
||||||
|
guard let contentDigest = group.contentDigest,
|
||||||
|
let contentURL = try contentStore.contentURLIfPresent(for: contentDigest) else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var expectedSize: UInt64 = 0
|
||||||
|
for chunk in group.chunks {
|
||||||
|
guard let uncompressedSize = chunk.uncompressedSize() else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
let addition = expectedSize.addingReportingOverflow(uncompressedSize)
|
||||||
|
guard !addition.overflow else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
expectedSize = addition.partialValue
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let actualSize = UInt64(exactly: try contentURL.sizeBytes()),
|
||||||
|
actualSize == expectedSize else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a cached image is complete enough for `pull` to return without
|
||||||
|
/// repairing it. Standalone images keep Tart's existing structural cache-hit
|
||||||
|
/// behavior; stacked cached images additionally need every immutable disk file in
|
||||||
|
/// the shared content store.
|
||||||
|
func hasCompleteCachedImage(_ name: RemoteName, manifest: OCIManifest) throws -> Bool {
|
||||||
|
guard exists(name) else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
guard let missingGroups = try missingStackedDiskFileGroups(for: manifest) else {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return missingGroups.isEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The lock-free pull fast path is only useful for a tag that already
|
||||||
|
/// points at this digest. New or retargeted tags validate once after taking
|
||||||
|
/// the host lock instead of hashing a large stack twice.
|
||||||
|
func hasCompleteLinkedImage(_ name: RemoteName, digestName: RemoteName, manifest: OCIManifest) throws -> Bool {
|
||||||
|
guard exists(name), linked(from: name, to: digestName) else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return try hasCompleteCachedImage(digestName, manifest: manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bytes that this pull may need to materialize locally. For stacked images
|
||||||
|
/// this is the sum of only the missing complete disk files, not the final
|
||||||
|
/// guest-visible disk block layout.
|
||||||
|
func requiredDiskStorageBytes(for manifest: OCIManifest) throws -> UInt64? {
|
||||||
|
guard let missingGroups = try missingStackedDiskFileGroups(for: manifest) else {
|
||||||
|
return manifest.uncompressedDiskSize()
|
||||||
|
}
|
||||||
|
|
||||||
|
var total: UInt64 = 0
|
||||||
|
for group in missingGroups {
|
||||||
|
for chunk in group.chunks {
|
||||||
|
guard let uncompressedSize = chunk.uncompressedSize() else {
|
||||||
|
throw OCIManifestValidationError.invalidDiskMetadata("disk chunks need uncompressed size and content digest")
|
||||||
|
}
|
||||||
|
let addition = total.addingReportingOverflow(uncompressedSize)
|
||||||
|
guard !addition.overflow else {
|
||||||
|
throw RuntimeError.PullFailed("stacked disk storage size overflows UInt64")
|
||||||
|
}
|
||||||
|
total = addition.partialValue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return total
|
||||||
}
|
}
|
||||||
|
|
||||||
func digest(_ name: RemoteName) throws -> String {
|
func digest(_ name: RemoteName) throws -> String {
|
||||||
|
|
@ -34,7 +136,7 @@ class VMStorageOCI: PrunableStorage {
|
||||||
func open(_ name: RemoteName, _ accessDate: Date = Date()) throws -> VMDirectory {
|
func open(_ name: RemoteName, _ accessDate: Date = Date()) throws -> VMDirectory {
|
||||||
let vmDir = VMDirectory(baseURL: vmURL(name))
|
let vmDir = VMDirectory(baseURL: vmURL(name))
|
||||||
|
|
||||||
try vmDir.validate(userFriendlyName: name.description)
|
try vmDir.validateCachedImage(userFriendlyName: name.description)
|
||||||
|
|
||||||
try vmDir.baseURL.updateAccessDate(accessDate)
|
try vmDir.baseURL.updateAccessDate(accessDate)
|
||||||
|
|
||||||
|
|
@ -44,11 +146,55 @@ class VMStorageOCI: PrunableStorage {
|
||||||
func create(_ name: RemoteName, overwrite: Bool = false) throws -> VMDirectory {
|
func create(_ name: RemoteName, overwrite: Bool = false) throws -> VMDirectory {
|
||||||
let vmDir = VMDirectory(baseURL: vmURL(name))
|
let vmDir = VMDirectory(baseURL: vmURL(name))
|
||||||
|
|
||||||
|
if !overwrite && vmDir.isCachedImage {
|
||||||
|
throw RuntimeError.VMDirectoryAlreadyInitialized("VM directory is already initialized, preventing overwrite")
|
||||||
|
}
|
||||||
|
|
||||||
try vmDir.initialize(overwrite: overwrite)
|
try vmDir.initialize(overwrite: overwrite)
|
||||||
|
|
||||||
return vmDir
|
return vmDir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Materialize the digest-addressed cached image for an image Tart just
|
||||||
|
/// pushed, without routing its own local data back through the registry.
|
||||||
|
func populate(_ name: RemoteName, from source: VMDirectory, manifest: OCIManifest) throws {
|
||||||
|
if try hasCompleteCachedImage(name, manifest: manifest) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let vmDir = try create(name, overwrite: exists(name))
|
||||||
|
|
||||||
|
if source.isStackedVM {
|
||||||
|
guard case .stacked(_, let overlays) = try manifest.tartDiskRepresentation(),
|
||||||
|
let contentDigest = overlays.last?.contentDigest else {
|
||||||
|
throw RuntimeError.VMConfigurationError("pushed image is missing its writable ASIF overlay")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pushed top overlay becomes immutable in the cached image. Keep a
|
||||||
|
// semantic copy so later clones do not need to fetch it back.
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
if try contentStore.existingContentURL(for: contentDigest) == nil {
|
||||||
|
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||||
|
do {
|
||||||
|
try FileManager.default.copyItem(at: source.overlayURL, to: temporaryURL)
|
||||||
|
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||||
|
} catch {
|
||||||
|
try? FileManager.default.removeItem(at: temporaryURL)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try FileManager.default.copyItem(at: source.configURL, to: vmDir.configURL)
|
||||||
|
try FileManager.default.copyItem(at: source.nvramURL, to: vmDir.nvramURL)
|
||||||
|
} else {
|
||||||
|
try source.clone(to: vmDir, generateMAC: false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the exact manifest Tart submitted so tag links and later pushes
|
||||||
|
// refer to the same digest-addressed cached image.
|
||||||
|
try manifest.toJSON().write(to: vmDir.manifestURL)
|
||||||
|
}
|
||||||
|
|
||||||
func move(_ name: RemoteName, from: VMDirectory) throws{
|
func move(_ name: RemoteName, from: VMDirectory) throws{
|
||||||
let targetURL = vmURL(name)
|
let targetURL = vmURL(name)
|
||||||
|
|
||||||
|
|
@ -84,7 +230,7 @@ class VMStorageOCI: PrunableStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
let vmDir = VMDirectory(baseURL: foundURL.resolvingSymlinksInPath())
|
let vmDir = VMDirectory(baseURL: foundURL.resolvingSymlinksInPath())
|
||||||
if !vmDir.initialized {
|
if !vmDir.isCachedImage {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -113,7 +259,7 @@ class VMStorageOCI: PrunableStorage {
|
||||||
for case let foundURL as URL in enumerator {
|
for case let foundURL as URL in enumerator {
|
||||||
let vmDir = VMDirectory(baseURL: foundURL)
|
let vmDir = VMDirectory(baseURL: foundURL)
|
||||||
|
|
||||||
if !vmDir.initialized {
|
if !vmDir.isCachedImage {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -141,7 +287,9 @@ class VMStorageOCI: PrunableStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
func prunables() throws -> [Prunable] {
|
func prunables() throws -> [Prunable] {
|
||||||
try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
|
try list().filter { (_, vmDir, isSymlink) in
|
||||||
|
!isSymlink && vmDir.isStandalone
|
||||||
|
}.map { (_, vmDir, _) in vmDir }
|
||||||
}
|
}
|
||||||
|
|
||||||
func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
|
func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
|
||||||
|
|
@ -157,7 +305,7 @@ class VMStorageOCI: PrunableStorage {
|
||||||
let digestName = RemoteName(host: name.host, namespace: name.namespace,
|
let digestName = RemoteName(host: name.host, namespace: name.namespace,
|
||||||
reference: Reference(digest: Digest.hash(manifestData)))
|
reference: Reference(digest: Digest.hash(manifestData)))
|
||||||
|
|
||||||
if exists(name) && exists(digestName) && linked(from: name, to: digestName) {
|
if try hasCompleteLinkedImage(name, digestName: digestName, manifest: manifest) {
|
||||||
// optimistically check if we need to do anything at all before locking
|
// optimistically check if we need to do anything at all before locking
|
||||||
defaultLogger.appendNewLine("\(digestName) image is already cached and linked!")
|
defaultLogger.appendNewLine("\(digestName) image is already cached and linked!")
|
||||||
return
|
return
|
||||||
|
|
@ -181,11 +329,13 @@ class VMStorageOCI: PrunableStorage {
|
||||||
throw CancellationError()
|
throw CancellationError()
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists(digestName) {
|
if try !hasCompleteCachedImage(digestName, manifest: manifest) {
|
||||||
let span = OTel.shared.tracer.spanBuilder(spanName: "pull").setActive(true).startSpan()
|
let span = OTel.shared.tracer.spanBuilder(spanName: "pull").setActive(true).startSpan()
|
||||||
defer { span.end() }
|
defer { span.end() }
|
||||||
|
|
||||||
let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description)
|
let tmpVMDir = try VMDirectory.temporaryDeterministic(key: name.description)
|
||||||
|
let digestVMDir = VMDirectory(baseURL: vmURL(digestName))
|
||||||
|
let preserveExplicitlyPulledMark = digestVMDir.isExplicitlyPulled()
|
||||||
|
|
||||||
// Open an existing VM directory corresponding to this name, if any,
|
// Open an existing VM directory corresponding to this name, if any,
|
||||||
// marking it as outdated to speed up the garbage collection process
|
// marking it as outdated to speed up the garbage collection process
|
||||||
|
|
@ -195,22 +345,41 @@ class VMStorageOCI: PrunableStorage {
|
||||||
let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
|
let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
|
||||||
try tmpVMDirLock.lock()
|
try tmpVMDirLock.lock()
|
||||||
|
|
||||||
|
// A previously pulled standalone image already has the complete base
|
||||||
|
// disk locally as disk.img. Promote that file into the content store
|
||||||
|
// before sizing or pulling so a stacked child only fetches overlays.
|
||||||
|
try reuseStandaloneDiskForStackedBaseIfPossible(manifest)
|
||||||
|
|
||||||
// Try to reclaim some cache space if we know the VM size in advance
|
// Try to reclaim some cache space if we know the VM size in advance
|
||||||
if let uncompressedDiskSize = manifest.uncompressedDiskSize() {
|
if let requiredDiskStorageBytes = try requiredDiskStorageBytes(for: manifest) {
|
||||||
OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute(
|
if let telemetryValue = Int(exactly: requiredDiskStorageBytes) {
|
||||||
key: "oci.image-uncompressed-disk-size-bytes",
|
OpenTelemetry.instance.contextProvider.activeSpan?.setAttribute(
|
||||||
value: .int(Int(uncompressedDiskSize))
|
key: "oci.image-required-disk-storage-bytes",
|
||||||
)
|
value: .int(telemetryValue)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
let otherVMFilesSize: UInt64 = 128 * 1024 * 1024
|
let otherVMFilesSize: UInt64 = 128 * 1024 * 1024
|
||||||
|
let requiredStorage = requiredDiskStorageBytes.addingReportingOverflow(otherVMFilesSize)
|
||||||
|
guard !requiredStorage.overflow else {
|
||||||
|
throw RuntimeError.PullFailed("required pull storage size overflows UInt64")
|
||||||
|
}
|
||||||
|
|
||||||
try Prune.reclaimIfNeeded(uncompressedDiskSize + otherVMFilesSize)
|
try Prune.reclaimIfNeeded(requiredStorage.partialValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
try await withTaskCancellationHandler(operation: {
|
try await withTaskCancellationHandler(operation: {
|
||||||
try await retry(maxAttempts: 5) {
|
try await retry(maxAttempts: 5) {
|
||||||
// Choose the best base image which has the most deduplication ratio
|
// Existing standalone images can still reuse another complete local disk.
|
||||||
let localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
|
// Stacked images reconstruct their immutable files through the
|
||||||
|
// shared content store instead of materializing disk.img.
|
||||||
|
let localLayerCache: LocalLayerCache?
|
||||||
|
switch try manifest.tartDiskRepresentation() {
|
||||||
|
case .flat:
|
||||||
|
localLayerCache = try await chooseLocalLayerCache(name, manifest, registry)
|
||||||
|
case .stacked:
|
||||||
|
localLayerCache = nil
|
||||||
|
}
|
||||||
|
|
||||||
if let llc = localLayerCache {
|
if let llc = localLayerCache {
|
||||||
let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)
|
let deduplicatedHuman = ByteCountFormatter.string(fromByteCount: Int64(llc.deduplicatedBytes), countStyle: .file)
|
||||||
|
|
@ -232,6 +401,14 @@ class VMStorageOCI: PrunableStorage {
|
||||||
|
|
||||||
return .throw
|
return .throw
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Preserve the exact manifest bytes received from the registry. Its
|
||||||
|
// digest identifies this cached image and stacked VMs pin it.
|
||||||
|
try manifestData.write(to: tmpVMDir.manifestURL)
|
||||||
|
if preserveExplicitlyPulledMark {
|
||||||
|
tmpVMDir.markExplicitlyPulled()
|
||||||
|
}
|
||||||
|
|
||||||
try move(digestName, from: tmpVMDir)
|
try move(digestName, from: tmpVMDir)
|
||||||
}, onCancel: {
|
}, onCancel: {
|
||||||
try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
|
try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
|
||||||
|
|
@ -253,6 +430,82 @@ class VMStorageOCI: PrunableStorage {
|
||||||
_ = try VMStorageOCI().open(name)
|
_ = try VMStorageOCI().open(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `nil` for standalone images and the missing immutable disk-file groups
|
||||||
|
/// for stacked images. `ContentStore.existingContentURL()` intentionally
|
||||||
|
/// validates the digest so corrupt entries are repaired by a normal pull.
|
||||||
|
private func missingStackedDiskFileGroups(for manifest: OCIManifest) throws -> [TartDiskFileGroup]? {
|
||||||
|
guard case .stacked(let base, let overlays) = try manifest.tartDiskRepresentation() else {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
var missingGroups: [TartDiskFileGroup] = []
|
||||||
|
for group in [base] + overlays {
|
||||||
|
guard let contentDigest = group.contentDigest else {
|
||||||
|
throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest")
|
||||||
|
}
|
||||||
|
if try contentStore.existingContentURL(for: contentDigest) == nil {
|
||||||
|
missingGroups.append(group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return missingGroups
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed a stacked image's immutable base from an already pulled standalone
|
||||||
|
/// OCI record when both manifests describe the same transport chunks. The
|
||||||
|
/// content store still verifies the whole-file digest before publishing it.
|
||||||
|
func reuseStandaloneDiskForStackedBaseIfPossible(_ manifest: OCIManifest) throws {
|
||||||
|
guard case .stacked(let base, _) = try manifest.tartDiskRepresentation(),
|
||||||
|
let contentDigest = base.contentDigest else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
// Content-store entries are verified when installed. Avoid hashing a
|
||||||
|
// potentially large prewarmed base again on every stacked pull.
|
||||||
|
guard try contentStore.contentURLIfPresent(for: contentDigest) == nil else {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for (_, vmDir, isSymlink) in try list() where !isSymlink && vmDir.isStandalone {
|
||||||
|
guard let manifestData = try? Data(contentsOf: vmDir.manifestURL),
|
||||||
|
let candidateManifest = try? OCIManifest(fromJSON: manifestData),
|
||||||
|
case .flat(let candidateBase) = try? candidateManifest.tartDiskRepresentation(),
|
||||||
|
diskChunksMatch(candidateBase.chunks, base.chunks) else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||||
|
do {
|
||||||
|
try FileManager.default.copyItem(at: vmDir.diskURL, to: temporaryURL)
|
||||||
|
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||||
|
return
|
||||||
|
} catch ContentStoreError.contentDigestMismatch {
|
||||||
|
try? FileManager.default.removeItem(at: temporaryURL)
|
||||||
|
} catch {
|
||||||
|
try? FileManager.default.removeItem(at: temporaryURL)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compare the OCI transport identity while ignoring stacked-only
|
||||||
|
/// whole-file annotations added to the first base chunk.
|
||||||
|
private func diskChunksMatch(_ left: [OCIManifestLayer], _ right: [OCIManifestLayer]) -> Bool {
|
||||||
|
guard left.count == right.count else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return zip(left, right).allSatisfy { left, right in
|
||||||
|
left.mediaType == right.mediaType &&
|
||||||
|
left.size == right.size &&
|
||||||
|
left.digest == right.digest &&
|
||||||
|
left.uncompressedSize() == right.uncompressedSize() &&
|
||||||
|
left.uncompressedContentDigest() == right.uncompressedContentDigest()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func linked(from: RemoteName, to: RemoteName) -> Bool {
|
func linked(from: RemoteName, to: RemoteName) -> Bool {
|
||||||
do {
|
do {
|
||||||
let resolvedFrom = try FileManager.default.destinationOfSymbolicLink(atPath: vmURL(from).path)
|
let resolvedFrom = try FileManager.default.destinationOfSymbolicLink(atPath: vmURL(from).path)
|
||||||
|
|
@ -280,10 +533,16 @@ class VMStorageOCI: PrunableStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load OCI VM images and their manifests (if present)
|
// Load OCI VM images and their manifests (if present)
|
||||||
var candidates: [(name: String, vmDir: VMDirectory, manifest: OCIManifest, deduplicatedBytes: UInt64)] = []
|
var candidates: [(
|
||||||
|
name: String,
|
||||||
|
vmDir: VMDirectory,
|
||||||
|
manifest: OCIManifest,
|
||||||
|
manifestDigest: String,
|
||||||
|
deduplicatedBytes: UInt64
|
||||||
|
)] = []
|
||||||
|
|
||||||
for (name, vmDir, isSymlink) in try list() {
|
for (name, vmDir, isSymlink) in try list() {
|
||||||
if isSymlink {
|
if isSymlink || !vmDir.isStandalone {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -295,7 +554,13 @@ class VMStorageOCI: PrunableStorage {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
candidates.append((name, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
|
candidates.append((
|
||||||
|
name,
|
||||||
|
vmDir,
|
||||||
|
manifest,
|
||||||
|
Digest.hash(manifestJSON),
|
||||||
|
calculateDeduplicatedBytes(manifest)
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Previously we haven't stored the OCI VM image manifests, but still fetched the VM image manifest if
|
// Previously we haven't stored the OCI VM image manifests, but still fetched the VM image manifest if
|
||||||
|
|
@ -305,10 +570,17 @@ class VMStorageOCI: PrunableStorage {
|
||||||
// with the registry if we haven't already retrieved the manifest for that OCI VM image.
|
// with the registry if we haven't already retrieved the manifest for that OCI VM image.
|
||||||
if name.reference.type == .Tag,
|
if name.reference.type == .Tag,
|
||||||
let vmDir = try? open(name),
|
let vmDir = try? open(name),
|
||||||
|
vmDir.isStandalone,
|
||||||
let digest = try? digest(name),
|
let digest = try? digest(name),
|
||||||
try !candidates.contains(where: {try $0.manifest.digest() == digest}),
|
!candidates.contains(where: { $0.manifestDigest == digest }),
|
||||||
let (manifest, _) = try? await registry.pullManifest(reference: digest) {
|
let (manifest, manifestData) = try? await registry.pullManifest(reference: digest) {
|
||||||
candidates.append((name.description, vmDir, manifest, calculateDeduplicatedBytes(manifest)))
|
candidates.append((
|
||||||
|
name.description,
|
||||||
|
vmDir,
|
||||||
|
manifest,
|
||||||
|
Digest.hash(manifestData),
|
||||||
|
calculateDeduplicatedBytes(manifest)
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Now, find the best match based on how many bytes we'll deduplicate
|
// Now, find the best match based on how many bytes we'll deduplicate
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,135 @@
|
||||||
|
import Foundation
|
||||||
|
import ArgumentParser
|
||||||
|
import XCTest
|
||||||
|
@testable import tart
|
||||||
|
|
||||||
|
final class CommandBehaviorTests: XCTestCase {
|
||||||
|
func testSetDiskRejectsStackedVMBeforeSavingConfig() async throws {
|
||||||
|
try await withTemporaryTartHome {
|
||||||
|
let vmDir = try VMStorageLocal().create("stacked")
|
||||||
|
let originalConfig = config()
|
||||||
|
try originalConfig.save(toURL: vmDir.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data()))
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.manifestURL.path, contents: Data()))
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.overlayURL.path, contents: Data()))
|
||||||
|
|
||||||
|
let replacementURL = try temporaryDirectory().appendingPathComponent("replacement.img")
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: replacementURL.path, contents: Data("replacement".utf8)))
|
||||||
|
|
||||||
|
let command = try Set.parseAsRoot([
|
||||||
|
"stacked",
|
||||||
|
"--cpu", "4",
|
||||||
|
"--disk", replacementURL.path,
|
||||||
|
]) as! Set
|
||||||
|
|
||||||
|
do {
|
||||||
|
try await command.run()
|
||||||
|
XCTFail("expected stacked disk replacement to be rejected")
|
||||||
|
} catch let error as ValidationError {
|
||||||
|
XCTAssertEqual(error.message, "--disk is not supported for VMs with a stacked disk")
|
||||||
|
}
|
||||||
|
|
||||||
|
XCTAssertEqual(try VMConfig(fromURL: vmDir.configURL).cpuCount, originalConfig.cpuCount)
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: vmDir.diskURL.path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRemoteAdditionalDiskRetainsTemporaryBackingFileLock() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let name = try RemoteName("example.com/org/image:latest")
|
||||||
|
let cachedImage = try storage.create(name)
|
||||||
|
try config().save(toURL: cachedImage.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: cachedImage.nvramURL.path, contents: Data()))
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(
|
||||||
|
atPath: cachedImage.diskURL.path,
|
||||||
|
contents: Data(repeating: 0, count: 4096)
|
||||||
|
))
|
||||||
|
|
||||||
|
do {
|
||||||
|
let additionalDisk = try AdditionalDisk(parseFrom: name.description)
|
||||||
|
let entriesBeforeGC = try temporaryEntries()
|
||||||
|
XCTAssertEqual(entriesBeforeGC.count, 1)
|
||||||
|
|
||||||
|
try Config().gc()
|
||||||
|
XCTAssertEqual(try temporaryEntries(), entriesBeforeGC)
|
||||||
|
|
||||||
|
withExtendedLifetime(additionalDisk) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
try Config().gc()
|
||||||
|
XCTAssertTrue(try temporaryEntries().isEmpty)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testGarbageCollectionPreservesLockedTemporaryDirectory() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let temporaryVMDir = try VMDirectory.temporary()
|
||||||
|
let lock = try FileLock(lockURL: temporaryVMDir.baseURL)
|
||||||
|
try lock.lock()
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(
|
||||||
|
atPath: temporaryVMDir.overlayURL.path,
|
||||||
|
contents: Data("overlay".utf8)
|
||||||
|
))
|
||||||
|
|
||||||
|
try Config().gc()
|
||||||
|
XCTAssertTrue(FileManager.default.fileExists(atPath: temporaryVMDir.overlayURL.path))
|
||||||
|
|
||||||
|
try lock.unlock()
|
||||||
|
try Config().gc()
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: temporaryVMDir.baseURL.path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func config() -> VMConfig {
|
||||||
|
VMConfig(
|
||||||
|
platform: Linux(),
|
||||||
|
cpuCountMin: 2,
|
||||||
|
memorySizeMin: 512 * 1024 * 1024,
|
||||||
|
diskFormat: .raw
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryEntries() throws -> [URL] {
|
||||||
|
try FileManager.default.contentsOfDirectory(
|
||||||
|
at: Config().tartTmpDir,
|
||||||
|
includingPropertiesForKeys: nil
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func withTemporaryTartHome(_ body: () throws -> Void) throws {
|
||||||
|
let home = try temporaryDirectory()
|
||||||
|
let previousHome = ProcessInfo.processInfo.environment["TART_HOME"]
|
||||||
|
setenv("TART_HOME", home.path, 1)
|
||||||
|
defer { restoreEnvironment("TART_HOME", to: previousHome) }
|
||||||
|
|
||||||
|
try body()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func withTemporaryTartHome(_ body: () async throws -> Void) async throws {
|
||||||
|
let home = try temporaryDirectory()
|
||||||
|
let previousHome = ProcessInfo.processInfo.environment["TART_HOME"]
|
||||||
|
setenv("TART_HOME", home.path, 1)
|
||||||
|
defer { restoreEnvironment("TART_HOME", to: previousHome) }
|
||||||
|
|
||||||
|
try await body()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryDirectory() throws -> URL {
|
||||||
|
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||||
|
try FileManager.default.createDirectory(at: url, withIntermediateDirectories: false)
|
||||||
|
addTeardownBlock {
|
||||||
|
try? FileManager.default.removeItem(at: url)
|
||||||
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
|
||||||
|
private func restoreEnvironment(_ name: String, to value: String?) {
|
||||||
|
if let value {
|
||||||
|
setenv(name, value, 1)
|
||||||
|
} else {
|
||||||
|
unsetenv(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -33,6 +33,63 @@ final class ContentStoreTests: XCTestCase {
|
||||||
XCTAssertNil(try store.existingContentURL(for: expectedDigest))
|
XCTAssertNil(try store.existingContentURL(for: expectedDigest))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testResumableAndLockURLsAreStablePerDigest() throws {
|
||||||
|
let store = try temporaryStore()
|
||||||
|
let firstDigest = Digest.hash(Data("first".utf8))
|
||||||
|
let secondDigest = Digest.hash(Data("second".utf8))
|
||||||
|
|
||||||
|
XCTAssertEqual(
|
||||||
|
try store.resumableContentURL(for: firstDigest),
|
||||||
|
try store.resumableContentURL(for: firstDigest)
|
||||||
|
)
|
||||||
|
XCTAssertNotEqual(
|
||||||
|
try store.resumableContentURL(for: firstDigest),
|
||||||
|
try store.resumableContentURL(for: secondDigest)
|
||||||
|
)
|
||||||
|
XCTAssertEqual(
|
||||||
|
try store.lockURL(for: firstDigest),
|
||||||
|
try store.lockURL(for: firstDigest)
|
||||||
|
)
|
||||||
|
XCTAssertTrue(FileManager.default.fileExists(atPath: try store.lockURL(for: firstDigest).path))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testInstallReplacesCorruptEntry() throws {
|
||||||
|
let store = try temporaryStore()
|
||||||
|
let data = Data("expected".utf8)
|
||||||
|
let digest = Digest.hash(data)
|
||||||
|
let contentURL = try store.contentURL(for: digest)
|
||||||
|
try FileManager.default.createDirectory(at: contentURL.deletingLastPathComponent(), withIntermediateDirectories: true)
|
||||||
|
try Data("corrupt".utf8).write(to: contentURL)
|
||||||
|
let temporaryURL = try store.temporaryContentURL(for: digest)
|
||||||
|
try data.write(to: temporaryURL)
|
||||||
|
|
||||||
|
XCTAssertEqual(try store.install(temporaryURL, contentDigest: digest), contentURL)
|
||||||
|
XCTAssertEqual(try Digest.hash(contentURL), digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testInstallPreservesExistingValidEntry() throws {
|
||||||
|
let store = try temporaryStore()
|
||||||
|
let data = Data("expected".utf8)
|
||||||
|
let digest = Digest.hash(data)
|
||||||
|
let firstTemporaryURL = try store.temporaryContentURL(for: digest)
|
||||||
|
try data.write(to: firstTemporaryURL)
|
||||||
|
let installedURL = try store.install(firstTemporaryURL, contentDigest: digest)
|
||||||
|
let secondTemporaryURL = try store.temporaryContentURL(for: digest)
|
||||||
|
try data.write(to: secondTemporaryURL)
|
||||||
|
|
||||||
|
XCTAssertEqual(try store.install(secondTemporaryURL, contentDigest: digest), installedURL)
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: secondTemporaryURL.path))
|
||||||
|
XCTAssertEqual(try Digest.hash(installedURL), digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testConcurrentInstallsAcceptDigestValidWinner() throws {
|
||||||
|
try assertConcurrentInstalls(seedCorruptEntry: false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testConcurrentInstallsRepairCorruptEntry() throws {
|
||||||
|
try assertConcurrentInstalls(seedCorruptEntry: true)
|
||||||
|
}
|
||||||
|
|
||||||
func testInstallRejectsWrongContentDigest() throws {
|
func testInstallRejectsWrongContentDigest() throws {
|
||||||
let store = try temporaryStore()
|
let store = try temporaryStore()
|
||||||
let expectedDigest = Digest.hash(Data("expected".utf8))
|
let expectedDigest = Digest.hash(Data("expected".utf8))
|
||||||
|
|
@ -64,4 +121,51 @@ final class ContentStoreTests: XCTestCase {
|
||||||
|
|
||||||
return try ContentStore(baseURL: url)
|
return try ContentStore(baseURL: url)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func assertConcurrentInstalls(seedCorruptEntry: Bool) throws {
|
||||||
|
let store = try temporaryStore()
|
||||||
|
let data = Data("expected".utf8)
|
||||||
|
let digest = Digest.hash(data)
|
||||||
|
let contentURL = try store.contentURL(for: digest)
|
||||||
|
try FileManager.default.createDirectory(at: contentURL.deletingLastPathComponent(), withIntermediateDirectories: true)
|
||||||
|
if seedCorruptEntry {
|
||||||
|
try Data("corrupt".utf8).write(to: contentURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
let temporaryURLs = try (0..<16).map { _ in
|
||||||
|
let url = try store.temporaryContentURL(for: digest)
|
||||||
|
try data.write(to: url)
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
let errors = ErrorCollector()
|
||||||
|
|
||||||
|
DispatchQueue.concurrentPerform(iterations: temporaryURLs.count) { index in
|
||||||
|
do {
|
||||||
|
_ = try store.install(temporaryURLs[index], contentDigest: digest)
|
||||||
|
} catch {
|
||||||
|
errors.append(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
XCTAssertTrue(errors.values.isEmpty, "unexpected install errors: \(errors.values)")
|
||||||
|
XCTAssertEqual(try Digest.hash(contentURL), digest)
|
||||||
|
XCTAssertTrue(temporaryURLs.allSatisfy { !FileManager.default.fileExists(atPath: $0.path) })
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class ErrorCollector: @unchecked Sendable {
|
||||||
|
private let lock = NSLock()
|
||||||
|
private var errors: [Error] = []
|
||||||
|
|
||||||
|
var values: [Error] {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
return errors
|
||||||
|
}
|
||||||
|
|
||||||
|
func append(_ error: Error) {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
errors.append(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import Foundation
|
||||||
import XCTest
|
import XCTest
|
||||||
@testable import tart
|
@testable import tart
|
||||||
|
|
||||||
|
|
@ -21,4 +22,34 @@ final class DigestTests: XCTestCase {
|
||||||
|
|
||||||
XCTAssertEqual(Digest.hash(data), "sha256:d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592")
|
XCTAssertEqual(Digest.hash(data), "sha256:d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testFileAndRangeHashingMatchDataHashing() throws {
|
||||||
|
let prefix = Data(repeating: 0x61, count: 4 * 1024 * 1024 + 17)
|
||||||
|
let range = Data("range".utf8)
|
||||||
|
let suffix = Data(repeating: 0x62, count: 23)
|
||||||
|
let data = prefix + range + suffix
|
||||||
|
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||||
|
try data.write(to: url)
|
||||||
|
defer { try? FileManager.default.removeItem(at: url) }
|
||||||
|
|
||||||
|
XCTAssertEqual(try Digest.hash(url), Digest.hash(data))
|
||||||
|
XCTAssertEqual(try Digest.hash(url, offset: UInt64(prefix.count), size: UInt64(range.count)), Digest.hash(range))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRangeHashingRejectsOutOfBoundsRanges() throws {
|
||||||
|
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||||
|
try Data("range".utf8).write(to: url)
|
||||||
|
defer { try? FileManager.default.removeItem(at: url) }
|
||||||
|
|
||||||
|
XCTAssertThrowsError(try Digest.hash(url, offset: 6, size: 0)) { error in
|
||||||
|
guard case DigestError.InvalidOffset = error else {
|
||||||
|
return XCTFail("unexpected error: \(error)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
XCTAssertThrowsError(try Digest.hash(url, offset: 1, size: UInt64.max)) { error in
|
||||||
|
guard case DigestError.InvalidSize = error else {
|
||||||
|
return XCTFail("unexpected error: \(error)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,13 @@ import XCTest
|
||||||
_ = try fixture.disk.makeAttachment()
|
_ = try fixture.disk.makeAttachment()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testAttachesStackReadOnly() throws {
|
||||||
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
|
try fixture.disk.createWritableOverlay()
|
||||||
|
|
||||||
|
_ = try fixture.disk.makeAttachment(readOnly: true)
|
||||||
|
}
|
||||||
|
|
||||||
func testRejectsMissingWritableOverlayWhenAttaching() throws {
|
func testRejectsMissingWritableOverlayWhenAttaching() throws {
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
|
|
||||||
|
|
@ -74,49 +81,15 @@ import XCTest
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func testRejectsWrongContentDigest() throws {
|
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
|
||||||
fixture.disk = DiskImageStack(
|
|
||||||
base: DiskImageFile(url: fixture.disk.base.url, contentDigest: "sha256:wrong"),
|
|
||||||
baseFormat: fixture.disk.baseFormat,
|
|
||||||
overlays: fixture.disk.overlays,
|
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
|
||||||
blockSize: fixture.disk.blockSize,
|
|
||||||
blockCount: fixture.disk.blockCount
|
|
||||||
)
|
|
||||||
|
|
||||||
assertThrows(.invalidDiskImage(fixture.disk.base.url, "disk image content digest does not match")) {
|
|
||||||
try fixture.disk.createWritableOverlay()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testRejectsWrongOverlayContentDigest() throws {
|
|
||||||
let fixture = try Fixture(baseFormat: .asif, publishedOverlayCount: 1)
|
|
||||||
fixture.disk = DiskImageStack(
|
|
||||||
base: fixture.disk.base,
|
|
||||||
baseFormat: fixture.disk.baseFormat,
|
|
||||||
overlays: [
|
|
||||||
DiskImageFile(url: fixture.disk.overlays[0].url, contentDigest: "sha256:wrong"),
|
|
||||||
],
|
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
|
||||||
blockSize: fixture.disk.blockSize,
|
|
||||||
blockCount: fixture.disk.blockCount
|
|
||||||
)
|
|
||||||
|
|
||||||
assertThrows(.invalidDiskImage(fixture.disk.overlays[0].url, "disk image content digest does not match")) {
|
|
||||||
try fixture.disk.createWritableOverlay()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testRejectsNonASIFPublishedOverlay() throws {
|
func testRejectsNonASIFPublishedOverlay() throws {
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
let overlayURL = fixture.directory.appendingPathComponent("published-raw.img")
|
let overlayURL = fixture.directory.appendingPathComponent("published-raw.img")
|
||||||
_ = try DiskImage(creating: .raw(url: overlayURL, blockCount: 8))
|
_ = try DiskImage(creating: .raw(url: overlayURL, blockCount: 8))
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: fixture.disk.baseFormat,
|
baseFormat: fixture.disk.baseFormat,
|
||||||
overlays: [
|
immutableOverlayURLs: [
|
||||||
DiskImageFile(url: overlayURL, contentDigest: try Digest.hash(overlayURL)),
|
overlayURL,
|
||||||
],
|
],
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
writableOverlayURL: fixture.disk.writableOverlayURL,
|
||||||
blockSize: fixture.disk.blockSize,
|
blockSize: fixture.disk.blockSize,
|
||||||
|
|
@ -131,15 +104,15 @@ import XCTest
|
||||||
func testRejectsWrongBaseFormat() throws {
|
func testRejectsWrongBaseFormat() throws {
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: .asif,
|
baseFormat: .asif,
|
||||||
overlays: fixture.disk.overlays,
|
immutableOverlayURLs: fixture.disk.immutableOverlayURLs,
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
writableOverlayURL: fixture.disk.writableOverlayURL,
|
||||||
blockSize: fixture.disk.blockSize,
|
blockSize: fixture.disk.blockSize,
|
||||||
blockCount: fixture.disk.blockCount
|
blockCount: fixture.disk.blockCount
|
||||||
)
|
)
|
||||||
|
|
||||||
assertThrows(.invalidDiskImage(fixture.disk.base.url, "base disk format does not match")) {
|
assertThrows(.invalidDiskImage(fixture.disk.baseURL, "base disk format does not match")) {
|
||||||
try fixture.disk.createWritableOverlay()
|
try fixture.disk.createWritableOverlay()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -147,15 +120,15 @@ import XCTest
|
||||||
func testRejectsBlockSizeMismatch() throws {
|
func testRejectsBlockSizeMismatch() throws {
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: fixture.disk.baseFormat,
|
baseFormat: fixture.disk.baseFormat,
|
||||||
overlays: fixture.disk.overlays,
|
immutableOverlayURLs: fixture.disk.immutableOverlayURLs,
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
writableOverlayURL: fixture.disk.writableOverlayURL,
|
||||||
blockSize: 4096,
|
blockSize: 4096,
|
||||||
blockCount: fixture.disk.blockCount
|
blockCount: fixture.disk.blockCount
|
||||||
)
|
)
|
||||||
|
|
||||||
assertThrows(.invalidGeometry("immutable disk stack does not match manifest block size")) {
|
assertThrows(.invalidBlockLayout("immutable disk stack does not match manifest block size")) {
|
||||||
try fixture.disk.createWritableOverlay()
|
try fixture.disk.createWritableOverlay()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -163,15 +136,15 @@ import XCTest
|
||||||
func testRejectsUnsupportedBlockSize() throws {
|
func testRejectsUnsupportedBlockSize() throws {
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: fixture.disk.baseFormat,
|
baseFormat: fixture.disk.baseFormat,
|
||||||
overlays: fixture.disk.overlays,
|
immutableOverlayURLs: fixture.disk.immutableOverlayURLs,
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
writableOverlayURL: fixture.disk.writableOverlayURL,
|
||||||
blockSize: 123,
|
blockSize: 123,
|
||||||
blockCount: fixture.disk.blockCount
|
blockCount: fixture.disk.blockCount
|
||||||
)
|
)
|
||||||
|
|
||||||
assertThrows(.invalidGeometry("unsupported stacked disk block size 123")) {
|
assertThrows(.invalidBlockLayout("unsupported stacked disk block size 123")) {
|
||||||
try fixture.disk.createWritableOverlay()
|
try fixture.disk.createWritableOverlay()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -179,15 +152,15 @@ import XCTest
|
||||||
func testRejectsManifestBlockCountMismatch() throws {
|
func testRejectsManifestBlockCountMismatch() throws {
|
||||||
let fixture = try Fixture(baseFormat: .raw)
|
let fixture = try Fixture(baseFormat: .raw)
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: fixture.disk.baseFormat,
|
baseFormat: fixture.disk.baseFormat,
|
||||||
overlays: fixture.disk.overlays,
|
immutableOverlayURLs: fixture.disk.immutableOverlayURLs,
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
writableOverlayURL: fixture.disk.writableOverlayURL,
|
||||||
blockSize: fixture.disk.blockSize,
|
blockSize: fixture.disk.blockSize,
|
||||||
blockCount: fixture.disk.blockCount + 1
|
blockCount: fixture.disk.blockCount + 1
|
||||||
)
|
)
|
||||||
|
|
||||||
assertThrows(.invalidGeometry("immutable disk stack does not match manifest block count")) {
|
assertThrows(.invalidBlockLayout("immutable disk stack does not match manifest block count")) {
|
||||||
try fixture.disk.createWritableOverlay()
|
try fixture.disk.createWritableOverlay()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -199,9 +172,9 @@ import XCTest
|
||||||
let copiedURL = fixture.directory.appendingPathComponent("copied-overlay.asif")
|
let copiedURL = fixture.directory.appendingPathComponent("copied-overlay.asif")
|
||||||
try fixture.disk.copyWritableOverlay(to: copiedURL)
|
try fixture.disk.copyWritableOverlay(to: copiedURL)
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: fixture.disk.baseFormat,
|
baseFormat: fixture.disk.baseFormat,
|
||||||
overlays: fixture.disk.overlays,
|
immutableOverlayURLs: fixture.disk.immutableOverlayURLs,
|
||||||
writableOverlayURL: copiedURL,
|
writableOverlayURL: copiedURL,
|
||||||
blockSize: fixture.disk.blockSize,
|
blockSize: fixture.disk.blockSize,
|
||||||
blockCount: fixture.disk.blockCount
|
blockCount: fixture.disk.blockCount
|
||||||
|
|
@ -216,15 +189,15 @@ import XCTest
|
||||||
let fixture = try Fixture(baseFormat: .asif)
|
let fixture = try Fixture(baseFormat: .asif)
|
||||||
let other = try Fixture(baseFormat: .asif, publishedOverlayCount: 1)
|
let other = try Fixture(baseFormat: .asif, publishedOverlayCount: 1)
|
||||||
fixture.disk = DiskImageStack(
|
fixture.disk = DiskImageStack(
|
||||||
base: fixture.disk.base,
|
baseURL: fixture.disk.baseURL,
|
||||||
baseFormat: fixture.disk.baseFormat,
|
baseFormat: fixture.disk.baseFormat,
|
||||||
overlays: other.disk.overlays,
|
immutableOverlayURLs: other.disk.immutableOverlayURLs,
|
||||||
writableOverlayURL: fixture.disk.writableOverlayURL,
|
writableOverlayURL: fixture.disk.writableOverlayURL,
|
||||||
blockSize: fixture.disk.blockSize,
|
blockSize: fixture.disk.blockSize,
|
||||||
blockCount: fixture.disk.blockCount
|
blockCount: fixture.disk.blockCount
|
||||||
)
|
)
|
||||||
|
|
||||||
assertThrows(.invalidDiskImage(other.disk.overlays[0].url, "ASIF overlay is incompatible with its parent")) {
|
assertThrows(.invalidDiskImage(other.disk.immutableOverlayURLs[0], "ASIF overlay is incompatible with its parent")) {
|
||||||
try fixture.disk.createWritableOverlay()
|
try fixture.disk.createWritableOverlay()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -263,19 +236,19 @@ import XCTest
|
||||||
_ = try DiskImage(creating: .asif(url: baseURL, blockCount: 8, blockSize: .bytes512))
|
_ = try DiskImage(creating: .asif(url: baseURL, blockCount: 8, blockSize: .bytes512))
|
||||||
}
|
}
|
||||||
|
|
||||||
var overlays: [DiskImageFile] = []
|
var immutableOverlayURLs: [URL] = []
|
||||||
var image = try DiskImage(opening: .open(url: baseURL, mode: .readOnly))
|
var image = try DiskImage(opening: .open(url: baseURL, mode: .readOnly))
|
||||||
for index in 0..<publishedOverlayCount {
|
for index in 0..<publishedOverlayCount {
|
||||||
let overlayURL = directory.appendingPathComponent("published-\(index).asif")
|
let overlayURL = directory.appendingPathComponent("published-\(index).asif")
|
||||||
let stack = try image.appending(.asifLayer(url: overlayURL, type: .overlay))
|
let stack = try image.appending(.asifLayer(url: overlayURL, type: .overlay))
|
||||||
overlays.append(DiskImageFile(url: overlayURL, contentDigest: try Digest.hash(overlayURL)))
|
immutableOverlayURLs.append(overlayURL)
|
||||||
image = stack
|
image = stack
|
||||||
}
|
}
|
||||||
|
|
||||||
disk = DiskImageStack(
|
disk = DiskImageStack(
|
||||||
base: DiskImageFile(url: baseURL, contentDigest: try Digest.hash(baseURL)),
|
baseURL: baseURL,
|
||||||
baseFormat: baseFormat,
|
baseFormat: baseFormat,
|
||||||
overlays: overlays,
|
immutableOverlayURLs: immutableOverlayURLs,
|
||||||
writableOverlayURL: directory.appendingPathComponent("overlay.asif"),
|
writableOverlayURL: directory.appendingPathComponent("overlay.asif"),
|
||||||
blockSize: 512,
|
blockSize: 512,
|
||||||
blockCount: 8
|
blockCount: 8
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,14 @@ final class LayerizerTests: XCTestCase {
|
||||||
let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
let pulledDiskFileURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||||
|
|
||||||
print("pushing disk...")
|
print("pushing disk...")
|
||||||
let diskLayers = try await DiskV2.push(diskURL: originalDiskFileURL, registry: registry, chunkSizeMb: 0, concurrency: 4, progress: Progress())
|
let diskLayers = try await DiskV2.push(
|
||||||
|
diskURL: originalDiskFileURL,
|
||||||
|
mediaType: diskV2MediaType,
|
||||||
|
registry: registry,
|
||||||
|
chunkSizeMb: 0,
|
||||||
|
concurrency: 4,
|
||||||
|
progress: Progress()
|
||||||
|
)
|
||||||
|
|
||||||
print("pulling disk...")
|
print("pulling disk...")
|
||||||
try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
|
try await DiskV2.pull(registry: registry, diskLayers: diskLayers, diskURL: pulledDiskFileURL, concurrency: 16, progress: Progress())
|
||||||
|
|
|
||||||
|
|
@ -106,7 +106,7 @@ final class OCIManifestTests: XCTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func testManifestBlockGeometry() throws {
|
func testManifestBlockLayout() throws {
|
||||||
var manifest = manifest(diskDescriptors: [chunk(mediaType: diskV2MediaType, suffix: "base-0")])
|
var manifest = manifest(diskDescriptors: [chunk(mediaType: diskV2MediaType, suffix: "base-0")])
|
||||||
manifest.annotations = [
|
manifest.annotations = [
|
||||||
uncompressedDiskSizeAnnotation: "100000000000",
|
uncompressedDiskSizeAnnotation: "100000000000",
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,185 @@
|
||||||
|
import Foundation
|
||||||
|
import XCTest
|
||||||
|
@testable import tart
|
||||||
|
|
||||||
|
#if canImport(DiskImageKit)
|
||||||
|
import DiskImageKit
|
||||||
|
|
||||||
|
@available(macOS 27.0, *)
|
||||||
|
final class VMDirectoryDiskImageStackTests: XCTestCase {
|
||||||
|
override func setUpWithError() throws {
|
||||||
|
try super.setUpWithError()
|
||||||
|
|
||||||
|
if #unavailable(macOS 27.0) {
|
||||||
|
throw XCTSkip("DiskImageKit tests require macOS 27 or newer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testBaseBlockLayoutReadsRawAndASIFImages() throws {
|
||||||
|
let directory = try temporaryDirectory()
|
||||||
|
let rawURL = directory.appendingPathComponent("base.raw")
|
||||||
|
let asifURL = directory.appendingPathComponent("base.asif")
|
||||||
|
_ = try DiskImage(creating: .raw(url: rawURL, blockCount: 8))
|
||||||
|
_ = try DiskImage(creating: .asif(url: asifURL, blockCount: 16, blockSize: .bytes512))
|
||||||
|
|
||||||
|
XCTAssertEqual(try DiskImageStack.baseBlockLayout(at: rawURL, expectedFormat: .raw).blockCount, 8)
|
||||||
|
XCTAssertEqual(try DiskImageStack.baseBlockLayout(at: asifURL, expectedFormat: .asif).blockCount, 16)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testCloneAsStackedBasePinsFlatManifestAndCreatesOverlay() throws {
|
||||||
|
let contentStore = try temporaryContentStore()
|
||||||
|
let source = try flatSource()
|
||||||
|
let destination = try temporaryVMDirectory()
|
||||||
|
|
||||||
|
try source.cloneAsStackedBase(to: destination, generateMAC: false, contentStore: contentStore)
|
||||||
|
|
||||||
|
XCTAssertTrue(destination.isStackedVM)
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: destination.diskURL.path))
|
||||||
|
|
||||||
|
let contentDigest = try Digest.hash(source.diskURL)
|
||||||
|
let manifest = try OCIManifest(fromJSON: Data(contentsOf: destination.manifestURL))
|
||||||
|
guard case .flat(let base) = try manifest.tartDiskRepresentation() else {
|
||||||
|
return XCTFail("expected a pinned base-only manifest")
|
||||||
|
}
|
||||||
|
XCTAssertEqual(base.contentDigest, contentDigest)
|
||||||
|
XCTAssertEqual(manifest.diskBlockSize(), 512)
|
||||||
|
XCTAssertEqual(manifest.diskBlockCount(), 8)
|
||||||
|
|
||||||
|
let stack = try destination.diskImageStack(contentStore: contentStore)
|
||||||
|
XCTAssertEqual(stack.baseURL, try contentStore.contentURL(for: contentDigest))
|
||||||
|
XCTAssertTrue(FileManager.default.fileExists(atPath: destination.overlayURL.path))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testCloneAsStackedBaseSupportsASIFDisk() throws {
|
||||||
|
let contentStore = try temporaryContentStore()
|
||||||
|
let source = try flatSource(diskFormat: .asif)
|
||||||
|
let destination = try temporaryVMDirectory()
|
||||||
|
|
||||||
|
try source.cloneAsStackedBase(to: destination, generateMAC: false, contentStore: contentStore)
|
||||||
|
|
||||||
|
let stack = try destination.diskImageStack(contentStore: contentStore)
|
||||||
|
XCTAssertEqual(stack.baseFormat, .asif)
|
||||||
|
XCTAssertTrue(destination.isStackedVM)
|
||||||
|
_ = try stack.makeAttachment()
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedCloneCanCopyOrCreateWritableOverlay() throws {
|
||||||
|
let contentStore = try temporaryContentStore()
|
||||||
|
let source = try flatSource()
|
||||||
|
let stacked = try temporaryVMDirectory()
|
||||||
|
try source.cloneAsStackedBase(to: stacked, generateMAC: false, contentStore: contentStore)
|
||||||
|
|
||||||
|
let copied = try temporaryVMDirectory()
|
||||||
|
try stacked.cloneStacked(to: copied, copyWritableOverlay: true, generateMAC: false, contentStore: contentStore)
|
||||||
|
XCTAssertEqual(try Digest.hash(copied.overlayURL), try Digest.hash(stacked.overlayURL))
|
||||||
|
|
||||||
|
let fresh = try temporaryVMDirectory()
|
||||||
|
try stacked.cloneStacked(to: fresh, copyWritableOverlay: false, generateMAC: false, contentStore: contentStore)
|
||||||
|
XCTAssertTrue(fresh.isStackedVM)
|
||||||
|
XCTAssertTrue(FileManager.default.fileExists(atPath: fresh.overlayURL.path))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testResizeDiskGrowsWritableOverlay() throws {
|
||||||
|
let contentStore = try temporaryContentStore()
|
||||||
|
let source = try flatSource()
|
||||||
|
let stacked = try temporaryVMDirectory()
|
||||||
|
try source.cloneAsStackedBase(to: stacked, generateMAC: false, contentStore: contentStore)
|
||||||
|
|
||||||
|
try stacked.resizeDisk(1, contentStore: contentStore)
|
||||||
|
|
||||||
|
let image = try DiskImage(opening: .open(url: stacked.overlayURL, mode: .readOnly))
|
||||||
|
XCTAssertEqual(image.blockCount, 1_000_000_000 / 512)
|
||||||
|
XCTAssertEqual(try stacked.diskSizeBytes(), 1_000_000_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testResolvesPublishedOverlayFromManifestAndCache() throws {
|
||||||
|
let contentStore = try temporaryContentStore()
|
||||||
|
let source = try flatSource()
|
||||||
|
let baseOnly = try temporaryVMDirectory()
|
||||||
|
try source.cloneAsStackedBase(to: baseOnly, generateMAC: false, contentStore: contentStore)
|
||||||
|
|
||||||
|
let contentDigest = try Digest.hash(baseOnly.overlayURL)
|
||||||
|
let temporaryContentURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||||
|
try FileManager.default.copyItem(at: baseOnly.overlayURL, to: temporaryContentURL)
|
||||||
|
_ = try contentStore.install(temporaryContentURL, contentDigest: contentDigest)
|
||||||
|
|
||||||
|
var manifest = try OCIManifest(fromJSON: Data(contentsOf: baseOnly.manifestURL))
|
||||||
|
var overlay = OCIManifestLayer(
|
||||||
|
mediaType: asifOverlayMediaType,
|
||||||
|
size: 1,
|
||||||
|
digest: "sha256:overlay-transport",
|
||||||
|
uncompressedSize: 1,
|
||||||
|
uncompressedContentDigest: "sha256:overlay-chunk"
|
||||||
|
)
|
||||||
|
overlay.annotations?[diskFileContentDigestAnnotation] = contentDigest
|
||||||
|
overlay.annotations?[diskFileChunkCountAnnotation] = "1"
|
||||||
|
manifest.layers.insert(overlay, at: manifest.layers.count - 1)
|
||||||
|
|
||||||
|
let destination = try temporaryVMDirectory()
|
||||||
|
try FileManager.default.copyItem(at: baseOnly.configURL, to: destination.configURL)
|
||||||
|
try FileManager.default.copyItem(at: baseOnly.nvramURL, to: destination.nvramURL)
|
||||||
|
try manifest.toJSON().write(to: destination.manifestURL)
|
||||||
|
|
||||||
|
let stack = try destination.diskImageStack(contentStore: contentStore)
|
||||||
|
XCTAssertEqual(stack.immutableOverlayURLs, [try contentStore.contentURL(for: contentDigest)])
|
||||||
|
try stack.createWritableOverlay()
|
||||||
|
_ = try stack.makeAttachment()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func flatSource(diskFormat: DiskImageFormat = .raw) throws -> VMDirectory {
|
||||||
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
let config = VMConfig(
|
||||||
|
platform: Linux(),
|
||||||
|
cpuCountMin: 2,
|
||||||
|
memorySizeMin: 512 * 1024 * 1024,
|
||||||
|
diskFormat: diskFormat
|
||||||
|
)
|
||||||
|
try config.save(toURL: vmDir.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data()))
|
||||||
|
switch diskFormat {
|
||||||
|
case .raw:
|
||||||
|
_ = try DiskImage(creating: .raw(url: vmDir.diskURL, blockCount: 8))
|
||||||
|
case .asif:
|
||||||
|
_ = try DiskImage(creating: .asif(url: vmDir.diskURL, blockCount: 8, blockSize: .bytes512))
|
||||||
|
}
|
||||||
|
|
||||||
|
let diskChunk = OCIManifestLayer(
|
||||||
|
mediaType: diskV2MediaType,
|
||||||
|
size: 1,
|
||||||
|
digest: "sha256:transport",
|
||||||
|
uncompressedSize: 4096,
|
||||||
|
uncompressedContentDigest: "sha256:chunk"
|
||||||
|
)
|
||||||
|
let manifest = OCIManifest(
|
||||||
|
config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"),
|
||||||
|
layers: [
|
||||||
|
OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"),
|
||||||
|
diskChunk,
|
||||||
|
OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
try manifest.toJSON().write(to: vmDir.manifestURL)
|
||||||
|
|
||||||
|
return vmDir
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryContentStore() throws -> ContentStore {
|
||||||
|
let url = try temporaryDirectory()
|
||||||
|
return try ContentStore(baseURL: url)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryVMDirectory() throws -> VMDirectory {
|
||||||
|
VMDirectory(baseURL: try temporaryDirectory())
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryDirectory() throws -> URL {
|
||||||
|
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||||
|
try FileManager.default.createDirectory(at: url, withIntermediateDirectories: false)
|
||||||
|
addTeardownBlock {
|
||||||
|
try? FileManager.default.removeItem(at: url)
|
||||||
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
@ -13,9 +13,11 @@ final class VMDirectoryLayoutTests: XCTestCase {
|
||||||
|
|
||||||
XCTAssertEqual(vmDir.layout, .standalone)
|
XCTAssertEqual(vmDir.layout, .standalone)
|
||||||
XCTAssertTrue(vmDir.initialized)
|
XCTAssertTrue(vmDir.initialized)
|
||||||
|
XCTAssertTrue(vmDir.isCachedImage)
|
||||||
|
XCTAssertNoThrow(try vmDir.validateCachedImage(userFriendlyName: "standalone"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func testStackedLocalLayout() throws {
|
func testStackedVMLayout() throws {
|
||||||
let vmDir = try temporaryVMDirectory()
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
|
||||||
try touch(vmDir.configURL)
|
try touch(vmDir.configURL)
|
||||||
|
|
@ -25,9 +27,10 @@ final class VMDirectoryLayoutTests: XCTestCase {
|
||||||
|
|
||||||
XCTAssertEqual(vmDir.layout, .stackedLocal)
|
XCTAssertEqual(vmDir.layout, .stackedLocal)
|
||||||
XCTAssertTrue(vmDir.initialized)
|
XCTAssertTrue(vmDir.initialized)
|
||||||
|
XCTAssertFalse(vmDir.isCachedImage)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testStackedOCIRecordLayout() throws {
|
func testStackedCachedImageLayout() throws {
|
||||||
let vmDir = try temporaryVMDirectory()
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
|
||||||
try touch(vmDir.configURL)
|
try touch(vmDir.configURL)
|
||||||
|
|
@ -36,6 +39,8 @@ final class VMDirectoryLayoutTests: XCTestCase {
|
||||||
|
|
||||||
XCTAssertEqual(vmDir.layout, .stackedOCIRecord)
|
XCTAssertEqual(vmDir.layout, .stackedOCIRecord)
|
||||||
XCTAssertFalse(vmDir.initialized)
|
XCTAssertFalse(vmDir.initialized)
|
||||||
|
XCTAssertTrue(vmDir.isCachedImage)
|
||||||
|
XCTAssertNoThrow(try vmDir.validateCachedImage(userFriendlyName: "stacked"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func testAmbiguousDiskAndOverlayIsNotInitialized() throws {
|
func testAmbiguousDiskAndOverlayIsNotInitialized() throws {
|
||||||
|
|
@ -49,6 +54,52 @@ final class VMDirectoryLayoutTests: XCTestCase {
|
||||||
|
|
||||||
XCTAssertNil(vmDir.layout)
|
XCTAssertNil(vmDir.layout)
|
||||||
XCTAssertFalse(vmDir.initialized)
|
XCTAssertFalse(vmDir.initialized)
|
||||||
|
XCTAssertFalse(vmDir.isCachedImage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedVMAccountingUsesOverlay() throws {
|
||||||
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
|
||||||
|
try Data("config".utf8).write(to: vmDir.configURL)
|
||||||
|
try Data("nvram".utf8).write(to: vmDir.nvramURL)
|
||||||
|
try Data("overlay".utf8).write(to: vmDir.overlayURL)
|
||||||
|
try stackedManifest(blockSize: 512, blockCount: 8).toJSON().write(to: vmDir.manifestURL)
|
||||||
|
|
||||||
|
XCTAssertEqual(
|
||||||
|
try vmDir.sizeBytes(),
|
||||||
|
try vmDir.configURL.sizeBytes() + vmDir.overlayURL.sizeBytes() + vmDir.nvramURL.sizeBytes()
|
||||||
|
)
|
||||||
|
XCTAssertEqual(
|
||||||
|
try vmDir.allocatedSizeBytes(),
|
||||||
|
try vmDir.configURL.allocatedSizeBytes() + vmDir.overlayURL.allocatedSizeBytes() + vmDir.nvramURL.allocatedSizeBytes()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedExportIsRejected() throws {
|
||||||
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
try touch(vmDir.configURL)
|
||||||
|
try touch(vmDir.nvramURL)
|
||||||
|
try touch(vmDir.manifestURL)
|
||||||
|
try touch(vmDir.overlayURL)
|
||||||
|
let archiveURL = vmDir.baseURL.appendingPathComponent("export.tvm")
|
||||||
|
|
||||||
|
XCTAssertThrowsError(try vmDir.exportToArchive(path: archiveURL.path)) { error in
|
||||||
|
guard case RuntimeError.ExportFailed(let message) = error else {
|
||||||
|
return XCTFail("unexpected error: \(error)")
|
||||||
|
}
|
||||||
|
XCTAssertEqual(message, "exporting stacked VMs is not supported yet")
|
||||||
|
}
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: archiveURL.path))
|
||||||
|
|
||||||
|
try FileManager.default.removeItem(at: vmDir.overlayURL)
|
||||||
|
XCTAssertTrue(vmDir.isStackedCachedImage)
|
||||||
|
XCTAssertThrowsError(try vmDir.exportToArchive(path: archiveURL.path)) { error in
|
||||||
|
guard case RuntimeError.ExportFailed(let message) = error else {
|
||||||
|
return XCTFail("unexpected error: \(error)")
|
||||||
|
}
|
||||||
|
XCTAssertEqual(message, "exporting stacked VMs is not supported yet")
|
||||||
|
}
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: archiveURL.path))
|
||||||
}
|
}
|
||||||
|
|
||||||
private func temporaryVMDirectory() throws -> VMDirectory {
|
private func temporaryVMDirectory() throws -> VMDirectory {
|
||||||
|
|
@ -64,4 +115,30 @@ final class VMDirectoryLayoutTests: XCTestCase {
|
||||||
private func touch(_ url: URL) throws {
|
private func touch(_ url: URL) throws {
|
||||||
XCTAssertTrue(FileManager.default.createFile(atPath: url.path, contents: Data()))
|
XCTAssertTrue(FileManager.default.createFile(atPath: url.path, contents: Data()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private func stackedManifest(blockSize: UInt64, blockCount: UInt64) -> OCIManifest {
|
||||||
|
var disk = OCIManifestLayer(
|
||||||
|
mediaType: diskV2MediaType,
|
||||||
|
size: 1,
|
||||||
|
digest: "sha256:transport",
|
||||||
|
uncompressedSize: blockSize * blockCount,
|
||||||
|
uncompressedContentDigest: "sha256:chunk"
|
||||||
|
)
|
||||||
|
disk.annotations?[diskFileContentDigestAnnotation] = "sha256:base"
|
||||||
|
|
||||||
|
var manifest = OCIManifest(
|
||||||
|
config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"),
|
||||||
|
layers: [
|
||||||
|
OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"),
|
||||||
|
disk,
|
||||||
|
OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
manifest.annotations = [
|
||||||
|
uncompressedDiskSizeAnnotation: String(blockSize * blockCount),
|
||||||
|
diskBlockSizeAnnotation: String(blockSize),
|
||||||
|
]
|
||||||
|
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,391 @@
|
||||||
|
import Foundation
|
||||||
|
import XCTest
|
||||||
|
@testable import tart
|
||||||
|
|
||||||
|
#if canImport(DiskImageKit)
|
||||||
|
import DiskImageKit
|
||||||
|
#endif
|
||||||
|
|
||||||
|
final class VMStorageOCITests: XCTestCase {
|
||||||
|
func testPopulateStandalonePushedImageCachesDiskAndManifest() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let source = try standaloneSource(diskData: Data("disk".utf8))
|
||||||
|
let manifest = try flatManifest()
|
||||||
|
let name = try digestName(for: manifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
|
||||||
|
try storage.populate(name, from: source, manifest: manifest)
|
||||||
|
|
||||||
|
let cached = try storage.open(name)
|
||||||
|
XCTAssertTrue(cached.isStandalone)
|
||||||
|
XCTAssertEqual(try Data(contentsOf: cached.diskURL), Data("disk".utf8))
|
||||||
|
XCTAssertEqual(try OCIManifest(fromJSON: Data(contentsOf: cached.manifestURL)), manifest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedCloneRequiresManifestForLegacyStandaloneCachedImage() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let manifest = try flatManifest()
|
||||||
|
let name = try digestName(for: manifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let record = try storage.create(name)
|
||||||
|
try config().save(toURL: record.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: record.diskURL.path, contents: Data()))
|
||||||
|
|
||||||
|
XCTAssertTrue(try storage.hasUsableCachedImageForClone(name))
|
||||||
|
XCTAssertFalse(try storage.hasUsableCachedImageForClone(name, requireManifest: true))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testCloneCacheCheckRejectsMissingOrWrongSizedStackedContent() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let baseData = Data("base".utf8)
|
||||||
|
let overlayData = Data("overlay".utf8)
|
||||||
|
let baseDigest = Digest.hash(baseData)
|
||||||
|
let overlayDigest = Digest.hash(overlayData)
|
||||||
|
let manifest = try stackedManifest(
|
||||||
|
baseContentDigest: baseDigest,
|
||||||
|
overlayContentDigest: overlayDigest,
|
||||||
|
baseUncompressedSize: UInt64(baseData.count),
|
||||||
|
overlayUncompressedSize: UInt64(overlayData.count)
|
||||||
|
)
|
||||||
|
let name = try digestName(for: manifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let record = try storage.create(name)
|
||||||
|
try config().save(toURL: record.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||||
|
try manifest.toJSON().write(to: record.manifestURL)
|
||||||
|
|
||||||
|
XCTAssertFalse(try storage.hasUsableCachedImageForClone(name))
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
try installContent(baseData, contentDigest: baseDigest, into: contentStore)
|
||||||
|
try installContent(overlayData, contentDigest: overlayDigest, into: contentStore)
|
||||||
|
XCTAssertTrue(try storage.hasUsableCachedImageForClone(name))
|
||||||
|
|
||||||
|
try Data("bad".utf8).write(to: try contentStore.contentURL(for: overlayDigest))
|
||||||
|
XCTAssertFalse(try storage.hasUsableCachedImageForClone(name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testListIncludesStackedCachedImage() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let manifest = try stackedManifest()
|
||||||
|
let name = try digestName(for: manifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let record = try storage.create(name)
|
||||||
|
try config().save(toURL: record.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||||
|
try manifest.toJSON().write(to: record.manifestURL)
|
||||||
|
|
||||||
|
XCTAssertTrue(try storage.list().contains { $0.0 == name.description })
|
||||||
|
XCTAssertEqual(try record.diskSizeBytes(), 4096)
|
||||||
|
XCTAssertNoThrow(try record.allocatedSizeBytes())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedCacheHitRequiresVerifiedContentAndSizesMissingFiles() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let baseData = Data("base".utf8)
|
||||||
|
let overlayData = Data("overlay".utf8)
|
||||||
|
let baseDigest = Digest.hash(baseData)
|
||||||
|
let overlayDigest = Digest.hash(overlayData)
|
||||||
|
let manifest = try stackedManifest(
|
||||||
|
baseContentDigest: baseDigest,
|
||||||
|
overlayContentDigest: overlayDigest,
|
||||||
|
baseUncompressedSize: 10,
|
||||||
|
overlayUncompressedSize: 20
|
||||||
|
)
|
||||||
|
let name = try digestName(for: manifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let record = try storage.create(name)
|
||||||
|
try config().save(toURL: record.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||||
|
try manifest.toJSON().write(to: record.manifestURL)
|
||||||
|
|
||||||
|
XCTAssertFalse(try storage.hasCompleteCachedImage(name, manifest: manifest))
|
||||||
|
XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 30)
|
||||||
|
|
||||||
|
let contentStore = try ContentStore()
|
||||||
|
try installContent(baseData, contentDigest: baseDigest, into: contentStore)
|
||||||
|
XCTAssertFalse(try storage.hasCompleteCachedImage(name, manifest: manifest))
|
||||||
|
XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 20)
|
||||||
|
|
||||||
|
try installContent(overlayData, contentDigest: overlayDigest, into: contentStore)
|
||||||
|
XCTAssertTrue(try storage.hasCompleteCachedImage(name, manifest: manifest))
|
||||||
|
XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 0)
|
||||||
|
|
||||||
|
let overlayURL = try contentStore.contentURL(for: overlayDigest)
|
||||||
|
try Data("corrupt".utf8).write(to: overlayURL)
|
||||||
|
XCTAssertFalse(try storage.hasCompleteCachedImage(name, manifest: manifest))
|
||||||
|
XCTAssertEqual(try storage.requiredDiskStorageBytes(for: manifest), 20)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedPullReusesPreviouslyPulledStandaloneDisk() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let diskData = Data([0])
|
||||||
|
let contentDigest = Digest.hash(diskData)
|
||||||
|
let flatManifest = try flatManifest()
|
||||||
|
let flatName = try digestName(for: flatManifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let flatRecord = try storage.create(flatName)
|
||||||
|
try config().save(toURL: flatRecord.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: flatRecord.nvramURL.path, contents: Data()))
|
||||||
|
try diskData.write(to: flatRecord.diskURL)
|
||||||
|
try flatManifest.toJSON().write(to: flatRecord.manifestURL)
|
||||||
|
|
||||||
|
let stackedManifest = try stackedManifest(baseContentDigest: contentDigest)
|
||||||
|
XCTAssertNil(try ContentStore().existingContentURL(for: contentDigest))
|
||||||
|
|
||||||
|
try storage.reuseStandaloneDiskForStackedBaseIfPossible(stackedManifest)
|
||||||
|
|
||||||
|
let reusedURL = try XCTUnwrap(try ContentStore().existingContentURL(for: contentDigest))
|
||||||
|
XCTAssertEqual(try Data(contentsOf: reusedURL), diskData)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStackedPullDoesNotRehashInstalledBaseBeforeReuse() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let contentDigest = Digest.hash(Data("base".utf8))
|
||||||
|
let manifest = try stackedManifest(baseContentDigest: contentDigest)
|
||||||
|
let contentURL = try ContentStore().contentURL(for: contentDigest)
|
||||||
|
|
||||||
|
// Hashing this path would throw. Once an entry is published, this
|
||||||
|
// fast path must trust its presence and let normal pull validation
|
||||||
|
// repair unusable content later.
|
||||||
|
try FileManager.default.createDirectory(at: contentURL, withIntermediateDirectories: false)
|
||||||
|
|
||||||
|
XCTAssertNoThrow(try VMStorageOCI().reuseStandaloneDiskForStackedBaseIfPossible(manifest))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testNewTagDoesNotValidateCachedStackBeforeLock() throws {
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let baseDigest = Digest.hash(Data("base".utf8))
|
||||||
|
let overlayDigest = Digest.hash(Data("overlay".utf8))
|
||||||
|
let manifest = try stackedManifest(
|
||||||
|
baseContentDigest: baseDigest,
|
||||||
|
overlayContentDigest: overlayDigest
|
||||||
|
)
|
||||||
|
let digestName = try digestName(for: manifest)
|
||||||
|
let tagName = RemoteName(
|
||||||
|
host: digestName.host,
|
||||||
|
namespace: digestName.namespace,
|
||||||
|
reference: Reference(tag: "latest")
|
||||||
|
)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let record = try storage.create(digestName)
|
||||||
|
try config().save(toURL: record.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||||
|
try manifest.toJSON().write(to: record.manifestURL)
|
||||||
|
|
||||||
|
// Hashing this directory as a disk file throws. A new tag must skip
|
||||||
|
// validation until after it has taken the host lock.
|
||||||
|
let contentURL = try ContentStore().contentURL(for: baseDigest)
|
||||||
|
try FileManager.default.createDirectory(at: contentURL, withIntermediateDirectories: false)
|
||||||
|
XCTAssertFalse(try storage.hasCompleteLinkedImage(tagName, digestName: digestName, manifest: manifest))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testStandaloneLayerCacheIgnoresStackedCachedImages() async throws {
|
||||||
|
try await withTemporaryTartHome {
|
||||||
|
var targetManifest = try flatManifest()
|
||||||
|
var stackedCandidateManifest = try stackedManifest()
|
||||||
|
let sharedDiskSize = 2 * 1024 * 1024 * 1024
|
||||||
|
targetManifest.layers[1].size = sharedDiskSize
|
||||||
|
stackedCandidateManifest.layers[1] = targetManifest.layers[1]
|
||||||
|
|
||||||
|
let candidateName = try digestName(for: stackedCandidateManifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
let candidate = try storage.create(candidateName)
|
||||||
|
try config().save(toURL: candidate.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: candidate.nvramURL.path, contents: Data()))
|
||||||
|
try stackedCandidateManifest.toJSON().write(to: candidate.manifestURL)
|
||||||
|
|
||||||
|
let targetName = RemoteName(
|
||||||
|
host: "example.com",
|
||||||
|
namespace: "org/target",
|
||||||
|
reference: Reference(digest: try targetManifest.digest())
|
||||||
|
)
|
||||||
|
let registry = try Registry(host: targetName.host, namespace: targetName.namespace)
|
||||||
|
|
||||||
|
let layerCache = try await storage.chooseLocalLayerCache(targetName, targetManifest, registry)
|
||||||
|
XCTAssertNil(layerCache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#if canImport(DiskImageKit)
|
||||||
|
@available(macOS 27.0, *)
|
||||||
|
func testPopulateStackedPushedImageCachesImmutableTopOverlay() throws {
|
||||||
|
if #unavailable(macOS 27.0) {
|
||||||
|
throw XCTSkip("DiskImageKit tests require macOS 27 or newer")
|
||||||
|
}
|
||||||
|
|
||||||
|
try withTemporaryTartHome {
|
||||||
|
let source = try diskImageSource()
|
||||||
|
let stacked = try temporaryVMDirectory()
|
||||||
|
try source.cloneAsStackedBase(to: stacked, generateMAC: false)
|
||||||
|
|
||||||
|
var manifest = try OCIManifest(fromJSON: Data(contentsOf: stacked.manifestURL))
|
||||||
|
let contentDigest = try Digest.hash(stacked.overlayURL)
|
||||||
|
var overlay = OCIManifestLayer(
|
||||||
|
mediaType: asifOverlayMediaType,
|
||||||
|
size: 1,
|
||||||
|
digest: "sha256:overlay-transport",
|
||||||
|
uncompressedSize: 1,
|
||||||
|
uncompressedContentDigest: "sha256:overlay-chunk"
|
||||||
|
)
|
||||||
|
overlay.annotations?[diskFileContentDigestAnnotation] = contentDigest
|
||||||
|
overlay.annotations?[diskFileChunkCountAnnotation] = "1"
|
||||||
|
manifest.layers.insert(overlay, at: manifest.layers.count - 1)
|
||||||
|
|
||||||
|
let name = try digestName(for: manifest)
|
||||||
|
let storage = try VMStorageOCI()
|
||||||
|
try storage.populate(name, from: stacked, manifest: manifest)
|
||||||
|
|
||||||
|
let cached = try storage.open(name)
|
||||||
|
XCTAssertTrue(cached.isStackedCachedImage)
|
||||||
|
XCTAssertFalse(FileManager.default.fileExists(atPath: cached.overlayURL.path))
|
||||||
|
XCTAssertEqual(try OCIManifest(fromJSON: Data(contentsOf: cached.manifestURL)), manifest)
|
||||||
|
|
||||||
|
let cachedContent = try XCTUnwrap(try ContentStore().existingContentURL(for: contentDigest))
|
||||||
|
XCTAssertEqual(try Digest.hash(cachedContent), contentDigest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
private func standaloneSource(diskData: Data) throws -> VMDirectory {
|
||||||
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
try config().save(toURL: vmDir.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data()))
|
||||||
|
try diskData.write(to: vmDir.diskURL)
|
||||||
|
|
||||||
|
return vmDir
|
||||||
|
}
|
||||||
|
|
||||||
|
#if canImport(DiskImageKit)
|
||||||
|
@available(macOS 27.0, *)
|
||||||
|
private func diskImageSource() throws -> VMDirectory {
|
||||||
|
let vmDir = try temporaryVMDirectory()
|
||||||
|
try config().save(toURL: vmDir.configURL)
|
||||||
|
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data()))
|
||||||
|
_ = try DiskImage(creating: .raw(url: vmDir.diskURL, blockCount: 8))
|
||||||
|
try flatManifest().toJSON().write(to: vmDir.manifestURL)
|
||||||
|
|
||||||
|
return vmDir
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
private func config() -> VMConfig {
|
||||||
|
VMConfig(
|
||||||
|
platform: Linux(),
|
||||||
|
cpuCountMin: 2,
|
||||||
|
memorySizeMin: 512 * 1024 * 1024,
|
||||||
|
diskFormat: .raw
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func flatManifest() throws -> OCIManifest {
|
||||||
|
let disk = OCIManifestLayer(
|
||||||
|
mediaType: diskV2MediaType,
|
||||||
|
size: 1,
|
||||||
|
digest: "sha256:disk-transport",
|
||||||
|
uncompressedSize: 1,
|
||||||
|
uncompressedContentDigest: "sha256:disk-chunk"
|
||||||
|
)
|
||||||
|
|
||||||
|
return OCIManifest(
|
||||||
|
config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"),
|
||||||
|
layers: [
|
||||||
|
OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"),
|
||||||
|
disk,
|
||||||
|
OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func stackedManifest(
|
||||||
|
baseContentDigest: String = "sha256:base",
|
||||||
|
overlayContentDigest: String = "sha256:overlay",
|
||||||
|
baseUncompressedSize: UInt64 = 1,
|
||||||
|
overlayUncompressedSize: UInt64 = 1
|
||||||
|
) throws -> OCIManifest {
|
||||||
|
var manifest = try flatManifest()
|
||||||
|
manifest.annotations?[diskBlockSizeAnnotation] = "512"
|
||||||
|
manifest.annotations?[uncompressedDiskSizeAnnotation] = "4096"
|
||||||
|
manifest.layers[1].annotations?[diskFileContentDigestAnnotation] = baseContentDigest
|
||||||
|
manifest.layers[1].annotations?[uncompressedSizeAnnotation] = String(baseUncompressedSize)
|
||||||
|
var overlay = OCIManifestLayer(
|
||||||
|
mediaType: asifOverlayMediaType,
|
||||||
|
size: 1,
|
||||||
|
digest: "sha256:overlay-transport",
|
||||||
|
uncompressedSize: overlayUncompressedSize,
|
||||||
|
uncompressedContentDigest: "sha256:overlay-chunk"
|
||||||
|
)
|
||||||
|
overlay.annotations?[diskFileContentDigestAnnotation] = overlayContentDigest
|
||||||
|
overlay.annotations?[diskFileChunkCountAnnotation] = "1"
|
||||||
|
manifest.layers.insert(overlay, at: manifest.layers.count - 1)
|
||||||
|
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
private func installContent(_ data: Data, contentDigest: String, into contentStore: ContentStore) throws {
|
||||||
|
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||||
|
try data.write(to: temporaryURL)
|
||||||
|
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func digestName(for manifest: OCIManifest) throws -> RemoteName {
|
||||||
|
RemoteName(
|
||||||
|
host: "example.com",
|
||||||
|
namespace: "org/image",
|
||||||
|
reference: Reference(digest: try manifest.digest())
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func withTemporaryTartHome(_ body: () throws -> Void) throws {
|
||||||
|
let home = try temporaryDirectory()
|
||||||
|
let previousHome = ProcessInfo.processInfo.environment["TART_HOME"]
|
||||||
|
setenv("TART_HOME", home.path, 1)
|
||||||
|
defer {
|
||||||
|
if let previousHome {
|
||||||
|
setenv("TART_HOME", previousHome, 1)
|
||||||
|
} else {
|
||||||
|
unsetenv("TART_HOME")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try body()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func withTemporaryTartHome(_ body: () async throws -> Void) async throws {
|
||||||
|
let home = try temporaryDirectory()
|
||||||
|
let previousHome = ProcessInfo.processInfo.environment["TART_HOME"]
|
||||||
|
setenv("TART_HOME", home.path, 1)
|
||||||
|
defer {
|
||||||
|
if let previousHome {
|
||||||
|
setenv("TART_HOME", previousHome, 1)
|
||||||
|
} else {
|
||||||
|
unsetenv("TART_HOME")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try await body()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryVMDirectory() throws -> VMDirectory {
|
||||||
|
VMDirectory(baseURL: try temporaryDirectory())
|
||||||
|
}
|
||||||
|
|
||||||
|
private func temporaryDirectory() throws -> URL {
|
||||||
|
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||||
|
try FileManager.default.createDirectory(at: url, withIntermediateDirectories: false)
|
||||||
|
addTeardownBlock {
|
||||||
|
try? FileManager.default.removeItem(at: url)
|
||||||
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue