mirror of https://github.com/cirruslabs/tart.git
Make stacked content pruning reference-aware (#1313)
This commit is contained in:
Yibo Zhuang
GitHub
parent
4162ca1831
commit
32a627c8c5
|
|
@ -81,27 +81,34 @@ struct Prune: AsyncParsableCommand {
|
|||
}
|
||||
|
||||
static func pruneSpaceBudget(prunableStorages: [PrunableStorage], spaceBudgetBytes: UInt64) throws {
|
||||
let prunables: [Prunable] = try prunableStorages
|
||||
.flatMap { try $0.prunables() }
|
||||
.sorted { try $0.accessDate() > $1.accessDate() }
|
||||
while true {
|
||||
let prunables: [Prunable] = try prunableStorages
|
||||
.flatMap { try $0.prunables() }
|
||||
.sorted { try $0.accessDate() > $1.accessDate() }
|
||||
|
||||
var spaceBudgetBytes = spaceBudgetBytes
|
||||
var prunablesToDelete: [Prunable] = []
|
||||
var remainingBudgetBytes = spaceBudgetBytes
|
||||
var prunableToDelete: Prunable?
|
||||
|
||||
for prunable in prunables {
|
||||
let prunableSizeBytes = UInt64(try prunable.allocatedSizeBytes())
|
||||
for prunable in prunables {
|
||||
let prunableSizeBytes = UInt64(try prunable.allocatedSizeBytes())
|
||||
|
||||
if prunableSizeBytes <= spaceBudgetBytes {
|
||||
// Don't mark for deletion as
|
||||
// there's a budget available
|
||||
spaceBudgetBytes -= prunableSizeBytes
|
||||
} else {
|
||||
// Mark for deletion
|
||||
prunablesToDelete.append(prunable)
|
||||
if prunableSizeBytes <= remainingBudgetBytes {
|
||||
// Don't mark for deletion as there is budget available
|
||||
remainingBudgetBytes -= prunableSizeBytes
|
||||
} else {
|
||||
prunableToDelete = prunable
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try prunablesToDelete.forEach { try $0.delete() }
|
||||
guard let prunableToDelete else {
|
||||
return
|
||||
}
|
||||
|
||||
// Deleting one cached stacked image can change which remaining image
|
||||
// owns shared immutable content. Rebuild before choosing another.
|
||||
try prunableToDelete.delete()
|
||||
}
|
||||
}
|
||||
|
||||
static func reclaimIfNeeded(_ requiredBytes: UInt64, _ initiator: Prunable? = nil) throws {
|
||||
|
|
@ -145,46 +152,51 @@ struct Prune: AsyncParsableCommand {
|
|||
try Prune.reclaimIfPossible(requiredBytes - volumeAvailableCapacityCalculated, initiator)
|
||||
}
|
||||
|
||||
private static func reclaimIfPossible(_ reclaimBytes: UInt64, _ initiator: Prunable? = nil) throws {
|
||||
static func reclaimIfPossible(_ reclaimBytes: UInt64, _ initiator: Prunable? = nil) throws {
|
||||
let span = OTel.shared.tracer.spanBuilder(spanName: "prune").startSpan()
|
||||
defer { span.end() }
|
||||
|
||||
let prunableStorages: [PrunableStorage] = [try VMStorageOCI(), try IPSWCache()]
|
||||
let prunables: [Prunable] = try prunableStorages
|
||||
.flatMap { try $0.prunables() }
|
||||
.sorted { try $0.accessDate() < $1.accessDate() }
|
||||
let prunables = {
|
||||
try prunableStorages
|
||||
.flatMap { try $0.prunables() }
|
||||
.sorted { try $0.accessDate() < $1.accessDate() }
|
||||
}
|
||||
|
||||
// Does it even make sense to start?
|
||||
let cacheUsedBytes = try prunables.map { try $0.allocatedSizeBytes() }.reduce(0, +)
|
||||
if cacheUsedBytes < reclaimBytes {
|
||||
let initialPrunables = try prunables()
|
||||
let initialCacheUsedBytes = try initialPrunables.map { try $0.allocatedSizeBytes() }.reduce(0, +)
|
||||
guard let reclaimBytes = Int(exactly: reclaimBytes), initialCacheUsedBytes >= reclaimBytes else {
|
||||
return
|
||||
}
|
||||
|
||||
var cacheReclaimedBytes: Int = 0
|
||||
let targetCacheUsedBytes = initialCacheUsedBytes - reclaimBytes
|
||||
var currentCacheUsedBytes = initialCacheUsedBytes
|
||||
let initiatorPath = initiator.map {
|
||||
$0.url.resolvingSymlinksInPath().standardizedFileURL.path
|
||||
}
|
||||
|
||||
var it = prunables.makeIterator()
|
||||
|
||||
while cacheReclaimedBytes <= reclaimBytes {
|
||||
guard let prunable = it.next() else {
|
||||
while currentCacheUsedBytes > targetCacheUsedBytes {
|
||||
// Deleting one cached stacked image can transfer ownership of shared
|
||||
// immutable content to another record without reclaiming those bytes.
|
||||
// Rebuild the candidates after every deletion so automatic pruning
|
||||
// measures the cache that remains rather than a stale ownership snapshot.
|
||||
guard let prunable = try prunables().first(where: {
|
||||
$0.url.resolvingSymlinksInPath().standardizedFileURL.path != initiatorPath
|
||||
}) else {
|
||||
break
|
||||
}
|
||||
|
||||
if prunable.url == initiator?.url.resolvingSymlinksInPath() {
|
||||
// do not prune the initiator
|
||||
continue
|
||||
}
|
||||
|
||||
let allocatedSizeBytes = try prunable.allocatedSizeBytes()
|
||||
|
||||
OpenTelemetry.instance.contextProvider.activeSpan?
|
||||
.addEvent(name: "Pruned \(allocatedSizeBytes) bytes for \(prunable.url.path)")
|
||||
|
||||
cacheReclaimedBytes += allocatedSizeBytes
|
||||
|
||||
try prunable.delete()
|
||||
currentCacheUsedBytes = try prunables().map { try $0.allocatedSizeBytes() }.reduce(0, +)
|
||||
}
|
||||
|
||||
OpenTelemetry.instance.contextProvider.activeSpan?
|
||||
.addEvent(name: "Reclaimed \(cacheReclaimedBytes) bytes")
|
||||
.addEvent(name: "Reclaimed \(initialCacheUsedBytes - currentCacheUsedBytes) bytes")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1041,20 +1041,24 @@ struct AdditionalDisk {
|
|||
// A cached stacked image has no writable top overlay. Create one in a
|
||||
// disposable directory for this additional-disk attachment.
|
||||
let temporaryVMDir = try VMDirectory.temporary()
|
||||
try FileManager.default.copyItem(at: vmDir.configURL, to: temporaryVMDir.configURL)
|
||||
try FileManager.default.copyItem(at: vmDir.nvramURL, to: temporaryVMDir.nvramURL)
|
||||
try FileManager.default.copyItem(at: vmDir.manifestURL, to: temporaryVMDir.manifestURL)
|
||||
let lock = try FileLock(lockURL: temporaryVMDir.baseURL)
|
||||
try lock.lock()
|
||||
let temporaryVMDirLock = try FileLock(lockURL: temporaryVMDir.baseURL)
|
||||
try temporaryVMDirLock.lock()
|
||||
try vmDir.cloneStacked(
|
||||
to: temporaryVMDir,
|
||||
copyWritableOverlay: false,
|
||||
generateMAC: false
|
||||
)
|
||||
let stack = try temporaryVMDir.diskImageStack()
|
||||
try stack.createWritableOverlay()
|
||||
let attachment = try stack.makeAttachment(
|
||||
readOnly: diskReadOnly,
|
||||
cachingMode: try VZDiskImageCachingMode(cachingModeRaw) ?? .automatic,
|
||||
synchronizationMode: try VZDiskImageSynchronizationMode(syncModeRaw)
|
||||
)
|
||||
|
||||
return AdditionalDisk(configuration: VZVirtioBlockDeviceConfiguration(attachment: attachment), temporaryDiskLock: lock)
|
||||
return AdditionalDisk(
|
||||
configuration: VZVirtioBlockDeviceConfiguration(attachment: attachment),
|
||||
temporaryDiskLock: temporaryVMDirLock
|
||||
)
|
||||
}
|
||||
|
||||
// Unfortunately, VZDiskImageStorageDeviceAttachment does not support
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ struct Config {
|
|||
continue
|
||||
}
|
||||
|
||||
try FileManager.default.removeItem(at: entry)
|
||||
try VMDirectory(baseURL: entry).removeFromDisk()
|
||||
|
||||
try lock.unlock()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ struct ContentStore {
|
|||
|
||||
let baseURL: URL
|
||||
private let digestDirectoryURL: URL
|
||||
private let pruneLockURL: URL
|
||||
|
||||
init() throws {
|
||||
try self.init(baseURL: Config().tartCacheDir.appendingPathComponent("content", isDirectory: true))
|
||||
|
|
@ -24,13 +25,41 @@ struct ContentStore {
|
|||
init(baseURL: URL) throws {
|
||||
self.baseURL = baseURL
|
||||
self.digestDirectoryURL = baseURL.appendingPathComponent(Self.digestAlgorithm, isDirectory: true)
|
||||
self.pruneLockURL = baseURL.appendingPathComponent(".gc.lock")
|
||||
try FileManager.default.createDirectory(at: digestDirectoryURL, withIntermediateDirectories: true)
|
||||
if !FileManager.default.fileExists(atPath: pruneLockURL.path) {
|
||||
_ = FileManager.default.createFile(atPath: pruneLockURL.path, contents: Data())
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes reference publication with the final reference check and
|
||||
/// deletion of immutable cache entries across Tart processes.
|
||||
func withPruneLock<T>(_ body: () throws -> T) throws -> T {
|
||||
let lock = try FileLock(lockURL: pruneLockURL)
|
||||
try lock.lock()
|
||||
defer { try? lock.unlock() }
|
||||
|
||||
return try body()
|
||||
}
|
||||
|
||||
/// Waits for any prune already scanning references to finish. After this
|
||||
/// returns, later prune runs can see a reference the caller already wrote.
|
||||
func synchronizePublishedReferences() throws {
|
||||
try withPruneLock {}
|
||||
}
|
||||
|
||||
func contentURL(for contentDigest: String) throws -> URL {
|
||||
try contentURL(for: contentDigest, under: baseURL)
|
||||
}
|
||||
|
||||
/// Returns the canonical path for a digest under an arbitrary content-store
|
||||
/// root without creating directories or lock files.
|
||||
func contentURL(for contentDigest: String, under baseURL: URL) throws -> URL {
|
||||
let digestHex = try validatedDigestHex(contentDigest)
|
||||
|
||||
return digestDirectoryURL.appendingPathComponent(digestHex)
|
||||
return baseURL
|
||||
.appendingPathComponent(Self.digestAlgorithm, isDirectory: true)
|
||||
.appendingPathComponent(digestHex)
|
||||
}
|
||||
|
||||
func temporaryContentURL(for contentDigest: String) throws -> URL {
|
||||
|
|
@ -61,9 +90,9 @@ struct ContentStore {
|
|||
return lockURL
|
||||
}
|
||||
|
||||
/// Returns a digest-addressed entry without rereading it. Pull verifies
|
||||
/// content hashes before accepting a cache hit; clone only needs a cheap
|
||||
/// structural check, like Tart's existing disk.img path.
|
||||
/// Returns an immutable digest-addressed entry without rereading it. Files
|
||||
/// are verified when installed and when deciding whether a pull is a cache
|
||||
/// hit; normal clone/run/push paths trust the store like Tart's disk.img.
|
||||
func contentURLIfPresent(for contentDigest: String) throws -> URL? {
|
||||
let url = try contentURL(for: contentDigest)
|
||||
|
||||
|
|
@ -71,6 +100,8 @@ struct ContentStore {
|
|||
return nil
|
||||
}
|
||||
|
||||
try url.updateAccessDate()
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
|
|
@ -88,6 +119,33 @@ struct ContentStore {
|
|||
return url
|
||||
}
|
||||
|
||||
/// Returns immutable content files that no retained cached image or local VM
|
||||
/// references. Callers may prune these like other cache entries.
|
||||
func prunables(excluding referencedContentDigests: Swift.Set<String>) throws -> [URL] {
|
||||
guard let enumerator = FileManager.default.enumerator(
|
||||
at: digestDirectoryURL,
|
||||
includingPropertiesForKeys: [.isRegularFileKey],
|
||||
options: [.skipsSubdirectoryDescendants]
|
||||
) else {
|
||||
return []
|
||||
}
|
||||
|
||||
return try enumerator.compactMap { element in
|
||||
guard let url = element as? URL,
|
||||
try url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile == true else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let contentDigest = "\(Self.digestPrefix)\(url.lastPathComponent)"
|
||||
guard (try? validatedDigestHex(contentDigest)) != nil,
|
||||
!referencedContentDigests.contains(contentDigest) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
}
|
||||
|
||||
/// Move a fully reconstructed temporary file into the cache after verifying
|
||||
/// its semantic identity. The caller should create the temporary file with
|
||||
/// temporaryContentURL(for:) or resumableContentURL(for:) so rename stays on
|
||||
|
|
|
|||
|
|
@ -37,6 +37,24 @@ struct TartDiskFileGroup: Equatable {
|
|||
/// Whole reconstructed-file digest. Existing flat manifests do not have
|
||||
/// this until a macOS 27 clone normalizes its local manifest copy.
|
||||
var contentDigest: String?
|
||||
|
||||
/// Expected size of the complete disk file reconstructed from these chunks.
|
||||
func uncompressedSize() -> UInt64? {
|
||||
var result: UInt64 = 0
|
||||
for chunk in chunks {
|
||||
guard let size = chunk.uncompressedSize() else {
|
||||
return nil
|
||||
}
|
||||
|
||||
let addition = result.addingReportingOverflow(size)
|
||||
guard !addition.overflow else {
|
||||
return nil
|
||||
}
|
||||
result = addition.partialValue
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
enum TartDiskRepresentation: Equatable {
|
||||
|
|
@ -172,6 +190,16 @@ struct OCIManifest: Codable, Equatable {
|
|||
return .stacked(base: base, overlays: overlays)
|
||||
}
|
||||
|
||||
/// Returns content-store digests needed to reconstruct this disk stack.
|
||||
func diskContentDigests() throws -> [String] {
|
||||
switch try tartDiskRepresentation() {
|
||||
case .flat(let base):
|
||||
return base.contentDigest.map { [$0] } ?? []
|
||||
case .stacked(let base, let overlays):
|
||||
return ([base] + overlays).compactMap(\.contentDigest)
|
||||
}
|
||||
}
|
||||
|
||||
private func validateChunkMetadata(_ chunks: [OCIManifestLayer]) throws {
|
||||
guard chunks.allSatisfy({ $0.uncompressedSize() != nil && $0.uncompressedContentDigest() != nil }) else {
|
||||
throw OCIManifestValidationError.invalidDiskMetadata("disk chunks need uncompressed size and content digest")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import Foundation
|
||||
|
||||
extension VMDirectory {
|
||||
/// Returns content-store digests needed to reconstruct this VM's disk stack.
|
||||
func diskContentDigests() throws -> [String] {
|
||||
let manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||
return try manifest.diskContentDigests()
|
||||
}
|
||||
|
||||
func diskImageStack(contentStore providedStore: ContentStore? = nil) throws -> DiskImageStack {
|
||||
let manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||
let base: TartDiskFileGroup
|
||||
|
|
@ -42,13 +48,18 @@ extension VMDirectory {
|
|||
generateMAC: Bool,
|
||||
contentStore: ContentStore? = nil
|
||||
) throws {
|
||||
try FileManager.default.copyItem(at: configURL, to: destination.configURL)
|
||||
try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL)
|
||||
try FileManager.default.copyItem(at: manifestURL, to: destination.manifestURL)
|
||||
let contentStore = try contentStore ?? ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
try FileManager.default.copyItem(at: configURL, to: destination.configURL)
|
||||
try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL)
|
||||
try FileManager.default.copyItem(at: manifestURL, to: destination.manifestURL)
|
||||
|
||||
if copyWritableOverlay {
|
||||
try FileManager.default.copyItem(at: overlayURL, to: destination.overlayURL)
|
||||
} else {
|
||||
if copyWritableOverlay {
|
||||
try FileManager.default.copyItem(at: overlayURL, to: destination.overlayURL)
|
||||
}
|
||||
}
|
||||
|
||||
if !copyWritableOverlay {
|
||||
try destination.diskImageStack(contentStore: contentStore).createWritableOverlay()
|
||||
}
|
||||
|
||||
|
|
@ -67,17 +78,6 @@ extension VMDirectory {
|
|||
let contentDigest = try Digest.hash(diskURL)
|
||||
let contentStore = try providedStore ?? ContentStore()
|
||||
|
||||
if try contentStore.existingContentURL(for: contentDigest) == nil {
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||
do {
|
||||
try FileManager.default.copyItem(at: diskURL, to: temporaryURL)
|
||||
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: temporaryURL)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
var manifest = try OCIManifest(fromJSON: Data(contentsOf: manifestURL))
|
||||
guard case .flat = try manifest.tartDiskRepresentation() else {
|
||||
throw RuntimeError.VMConfigurationError("--stacked cannot use an image that already has a stacked disk")
|
||||
|
|
@ -101,7 +101,24 @@ extension VMDirectory {
|
|||
|
||||
try FileManager.default.copyItem(at: configURL, to: destination.configURL)
|
||||
try FileManager.default.copyItem(at: nvramURL, to: destination.nvramURL)
|
||||
try manifest.toJSON().write(to: destination.manifestURL)
|
||||
try contentStore.withPruneLock {
|
||||
try manifest.toJSON().write(to: destination.manifestURL)
|
||||
}
|
||||
|
||||
// Publish the temporary VM's manifest before installing the shared base.
|
||||
// Reference-aware pruning includes in-progress manifests, so the content
|
||||
// cannot be collected in the window before this VM is moved into place.
|
||||
if try contentStore.contentURLIfPresent(for: contentDigest) == nil {
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||
do {
|
||||
try FileManager.default.copyItem(at: diskURL, to: temporaryURL)
|
||||
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: temporaryURL)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
try destination.diskImageStack(contentStore: contentStore).createWritableOverlay()
|
||||
|
||||
if generateMAC {
|
||||
|
|
|
|||
|
|
@ -373,11 +373,27 @@ struct VMDirectory: Prunable {
|
|||
throw RuntimeError.VMIsRunning(name)
|
||||
}
|
||||
|
||||
try FileManager.default.removeItem(at: baseURL)
|
||||
// Standalone local VMs do not reference the shared content store. Delete
|
||||
// them directly so a full disk can still be recovered before the content
|
||||
// store has ever been initialized.
|
||||
if isStandalone {
|
||||
try FileManager.default.removeItem(at: baseURL)
|
||||
} else {
|
||||
try removeFromDisk()
|
||||
}
|
||||
|
||||
try lock.unlock()
|
||||
}
|
||||
|
||||
/// Removes a VM directory while preserving the content-store reference
|
||||
/// protocol for any complete or partially published manifest it contains.
|
||||
func removeFromDisk() throws {
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
try FileManager.default.removeItem(at: baseURL)
|
||||
}
|
||||
}
|
||||
|
||||
func accessDate() throws -> Date {
|
||||
try baseURL.accessDate()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,11 +35,27 @@ class VMStorageLocal: PrunableStorage {
|
|||
|
||||
func move(_ name: String, from: VMDirectory) throws {
|
||||
_ = try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
|
||||
_ = try FileManager.default.replaceItemAt(vmURL(name), withItemAt: from.baseURL)
|
||||
try replace(VMDirectory(baseURL: vmURL(name)), with: from)
|
||||
}
|
||||
|
||||
func rename(_ name: String, _ newName: String) throws {
|
||||
_ = try FileManager.default.replaceItemAt(vmURL(newName), withItemAt: vmURL(name))
|
||||
let source = VMDirectory(baseURL: vmURL(name))
|
||||
let destination = VMDirectory(baseURL: vmURL(newName))
|
||||
try replace(destination, with: source)
|
||||
}
|
||||
|
||||
/// References in a manifest must not disappear while content GC is deciding
|
||||
/// whether their immutable disk files are still in use.
|
||||
private func replace(_ destination: VMDirectory, with source: VMDirectory) throws {
|
||||
if FileManager.default.fileExists(atPath: source.manifestURL.path) ||
|
||||
FileManager.default.fileExists(atPath: destination.manifestURL.path) {
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
_ = try FileManager.default.replaceItemAt(destination.baseURL, withItemAt: source.baseURL)
|
||||
}
|
||||
} else {
|
||||
_ = try FileManager.default.replaceItemAt(destination.baseURL, withItemAt: source.baseURL)
|
||||
}
|
||||
}
|
||||
|
||||
func delete(_ name: String) throws {
|
||||
|
|
|
|||
|
|
@ -22,9 +22,8 @@ class VMStorageOCI: PrunableStorage {
|
|||
}
|
||||
|
||||
/// Whether clone can use a cached image without pulling. Standalone images keep
|
||||
/// Tart's existing structural check. Stacked cached images cheaply require every
|
||||
/// immutable file with its expected length; explicit pull remains the path
|
||||
/// that hashes content and repairs same-sized corruption.
|
||||
/// Tart's existing structural check. Stacked cached images require every
|
||||
/// immutable file with its expected length.
|
||||
func hasUsableCachedImageForClone(_ name: RemoteName, requireManifest: Bool = false) throws -> Bool {
|
||||
guard exists(name) else {
|
||||
return false
|
||||
|
|
@ -45,25 +44,7 @@ class VMStorageOCI: PrunableStorage {
|
|||
|
||||
let contentStore = try ContentStore()
|
||||
for group in [base] + overlays {
|
||||
guard let contentDigest = group.contentDigest,
|
||||
let contentURL = try contentStore.contentURLIfPresent(for: contentDigest) else {
|
||||
return false
|
||||
}
|
||||
|
||||
var expectedSize: UInt64 = 0
|
||||
for chunk in group.chunks {
|
||||
guard let uncompressedSize = chunk.uncompressedSize() else {
|
||||
return false
|
||||
}
|
||||
let addition = expectedSize.addingReportingOverflow(uncompressedSize)
|
||||
guard !addition.overflow else {
|
||||
return false
|
||||
}
|
||||
expectedSize = addition.partialValue
|
||||
}
|
||||
|
||||
guard let actualSize = UInt64(exactly: try contentURL.sizeBytes()),
|
||||
actualSize == expectedSize else {
|
||||
guard try hasUsableCachedDiskFile(group, contentStore: contentStore) else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -164,35 +145,44 @@ class VMStorageOCI: PrunableStorage {
|
|||
|
||||
let vmDir = try create(name, overwrite: exists(name))
|
||||
|
||||
if source.isStackedVM {
|
||||
guard case .stacked(_, let overlays) = try manifest.tartDiskRepresentation(),
|
||||
let contentDigest = overlays.last?.contentDigest else {
|
||||
throw RuntimeError.VMConfigurationError("pushed image is missing its writable ASIF overlay")
|
||||
}
|
||||
|
||||
// The pushed top overlay becomes immutable in the cached image. Keep a
|
||||
// semantic copy so later clones do not need to fetch it back.
|
||||
let contentStore = try ContentStore()
|
||||
if try contentStore.existingContentURL(for: contentDigest) == nil {
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||
do {
|
||||
try FileManager.default.copyItem(at: source.overlayURL, to: temporaryURL)
|
||||
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: temporaryURL)
|
||||
throw error
|
||||
do {
|
||||
if source.isStackedVM {
|
||||
guard case .stacked(_, let overlays) = try manifest.tartDiskRepresentation(),
|
||||
let contentDigest = overlays.last?.contentDigest else {
|
||||
throw RuntimeError.VMConfigurationError("pushed image is missing its writable ASIF overlay")
|
||||
}
|
||||
|
||||
// The pushed top overlay becomes immutable in the cached image. Keep a
|
||||
// semantic copy so later clones do not need to fetch it back.
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
try FileManager.default.copyItem(at: source.configURL, to: vmDir.configURL)
|
||||
try FileManager.default.copyItem(at: source.nvramURL, to: vmDir.nvramURL)
|
||||
// Publish the reference before installing the immutable top overlay,
|
||||
// so reference-aware pruning cannot collect it in between.
|
||||
try manifest.toJSON().write(to: vmDir.manifestURL)
|
||||
}
|
||||
|
||||
if try contentStore.contentURLIfPresent(for: contentDigest) == nil {
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||
do {
|
||||
try FileManager.default.copyItem(at: source.overlayURL, to: temporaryURL)
|
||||
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: temporaryURL)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try source.clone(to: vmDir, generateMAC: false)
|
||||
// Keep the exact manifest Tart submitted so tag links and later pushes
|
||||
// refer to the same digest-addressed cached image.
|
||||
try manifest.toJSON().write(to: vmDir.manifestURL)
|
||||
}
|
||||
|
||||
try FileManager.default.copyItem(at: source.configURL, to: vmDir.configURL)
|
||||
try FileManager.default.copyItem(at: source.nvramURL, to: vmDir.nvramURL)
|
||||
} else {
|
||||
try source.clone(to: vmDir, generateMAC: false)
|
||||
} catch {
|
||||
try? vmDir.removeFromDisk()
|
||||
throw error
|
||||
}
|
||||
|
||||
// Keep the exact manifest Tart submitted so tag links and later pushes
|
||||
// refer to the same digest-addressed cached image.
|
||||
try manifest.toJSON().write(to: vmDir.manifestURL)
|
||||
}
|
||||
|
||||
func move(_ name: RemoteName, from: VMDirectory) throws{
|
||||
|
|
@ -203,11 +193,20 @@ class VMStorageOCI: PrunableStorage {
|
|||
try FileManager.default.createDirectory(at: targetURL.deletingLastPathComponent(),
|
||||
withIntermediateDirectories: true)
|
||||
|
||||
_ = try FileManager.default.replaceItemAt(targetURL, withItemAt: from.baseURL)
|
||||
let target = VMDirectory(baseURL: targetURL)
|
||||
if FileManager.default.fileExists(atPath: from.manifestURL.path) ||
|
||||
FileManager.default.fileExists(atPath: target.manifestURL.path) {
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
_ = try FileManager.default.replaceItemAt(targetURL, withItemAt: from.baseURL)
|
||||
}
|
||||
} else {
|
||||
_ = try FileManager.default.replaceItemAt(targetURL, withItemAt: from.baseURL)
|
||||
}
|
||||
}
|
||||
|
||||
func delete(_ name: RemoteName) throws {
|
||||
try FileManager.default.removeItem(at: vmURL(name))
|
||||
try removeRecord(at: vmURL(name))
|
||||
try gc()
|
||||
}
|
||||
|
||||
|
|
@ -216,6 +215,7 @@ class VMStorageOCI: PrunableStorage {
|
|||
|
||||
guard let enumerator = FileManager.default.enumerator(at: baseURL,
|
||||
includingPropertiesForKeys: [.isSymbolicLinkKey]) else {
|
||||
try gcContent()
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -243,7 +243,28 @@ class VMStorageOCI: PrunableStorage {
|
|||
let vmDir = VMDirectory(baseURL: baseURL)
|
||||
|
||||
if !vmDir.isExplicitlyPulled() && incRefCount == 0 {
|
||||
try FileManager.default.removeItem(at: baseURL)
|
||||
try removeRecord(at: baseURL)
|
||||
}
|
||||
}
|
||||
|
||||
try gcContent()
|
||||
}
|
||||
|
||||
/// Cached images with a manifest publish references into the shared content
|
||||
/// store. Remove them through VMDirectory so reference removal is serialized
|
||||
/// with clone, export, pull, and content GC, even if a record is incomplete.
|
||||
private func removeRecord(at url: URL) throws {
|
||||
try VMDirectory(baseURL: url).removeFromDisk()
|
||||
}
|
||||
|
||||
/// Remove immutable files whose final published or in-progress reference
|
||||
/// has disappeared, without collecting unrelated cached images.
|
||||
fileprivate func gcContent() throws {
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
let referencedContentDigests = try referencedContentDigests(includeCachedImages: true)
|
||||
for contentURL in try contentStore.prunables(excluding: referencedContentDigests) {
|
||||
try FileManager.default.removeItem(at: contentURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -287,9 +308,57 @@ class VMStorageOCI: PrunableStorage {
|
|||
}
|
||||
|
||||
func prunables() throws -> [Prunable] {
|
||||
try list().filter { (_, vmDir, isSymlink) in
|
||||
!isSymlink && vmDir.isStandalone
|
||||
let records = try list().filter { (_, _, isSymlink) in
|
||||
!isSymlink
|
||||
}.map { (_, vmDir, _) in vmDir }
|
||||
|
||||
// Attribute shared content to the newest cached image that references it.
|
||||
// This counts each file once while charging it to the last record that
|
||||
// normally needs to be removed before the file becomes reclaimable.
|
||||
let nonCacheContentDigests = try referencedContentDigests(includeCachedImages: false)
|
||||
var contentOwners = [String: VMDirectory]()
|
||||
for record in records where record.isStackedCachedImage {
|
||||
// Interrupted cache population can leave a truncated manifest in an
|
||||
// otherwise recognizable cached record. It has no reliable content
|
||||
// references, but it must not prevent pruning other cache entries.
|
||||
for contentDigest in (try? record.diskContentDigests()) ?? []
|
||||
where !nonCacheContentDigests.contains(contentDigest) {
|
||||
guard let currentOwner = contentOwners[contentDigest] else {
|
||||
contentOwners[contentDigest] = record
|
||||
continue
|
||||
}
|
||||
|
||||
let recordAccessDate = try record.accessDate()
|
||||
let currentAccessDate = try currentOwner.accessDate()
|
||||
if recordAccessDate > currentAccessDate ||
|
||||
(recordAccessDate == currentAccessDate && record.url.path > currentOwner.url.path) {
|
||||
contentOwners[contentDigest] = record
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let contentStore = try ContentStore()
|
||||
var ownedContentURLs = [URL: [URL]]()
|
||||
for (contentDigest, owner) in contentOwners {
|
||||
let contentURL = try contentStore.contentURL(for: contentDigest)
|
||||
guard FileManager.default.fileExists(atPath: contentURL.path) else {
|
||||
continue
|
||||
}
|
||||
|
||||
ownedContentURLs[owner.url, default: []].append(contentURL)
|
||||
}
|
||||
|
||||
var result: [Prunable] = records.map { record in
|
||||
CachedImagePrunable(
|
||||
vmDir: record,
|
||||
ownedContentURLs: ownedContentURLs[record.url] ?? []
|
||||
)
|
||||
}
|
||||
|
||||
result += try contentStore.prunables(excluding: referencedContentDigests(includeCachedImages: true))
|
||||
.map(ContentPrunable.init)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func pull(_ name: RemoteName, registry: Registry, concurrency: UInt, deduplicate: Bool) async throws {
|
||||
|
|
@ -345,6 +414,12 @@ class VMStorageOCI: PrunableStorage {
|
|||
let tmpVMDirLock = try FileLock(lockURL: tmpVMDir.baseURL)
|
||||
try tmpVMDirLock.lock()
|
||||
|
||||
// Make in-progress stacked content references visible before reclaiming
|
||||
// space or reconstructing immutable files.
|
||||
try ContentStore().withPruneLock {
|
||||
try manifestData.write(to: tmpVMDir.manifestURL)
|
||||
}
|
||||
|
||||
// A previously pulled standalone image already has the complete base
|
||||
// disk locally as disk.img. Promote that file into the content store
|
||||
// before sizing or pulling so a stacked child only fetches overlays.
|
||||
|
|
@ -402,16 +477,13 @@ class VMStorageOCI: PrunableStorage {
|
|||
return .throw
|
||||
}
|
||||
|
||||
// Preserve the exact manifest bytes received from the registry. Its
|
||||
// digest identifies this cached image and stacked VMs pin it.
|
||||
try manifestData.write(to: tmpVMDir.manifestURL)
|
||||
if preserveExplicitlyPulledMark {
|
||||
tmpVMDir.markExplicitlyPulled()
|
||||
}
|
||||
|
||||
try move(digestName, from: tmpVMDir)
|
||||
}, onCancel: {
|
||||
try? FileManager.default.removeItem(at: tmpVMDir.baseURL)
|
||||
try? tmpVMDir.removeFromDisk()
|
||||
})
|
||||
} else {
|
||||
defaultLogger.appendNewLine("\(digestName) image is already cached! creating a symlink...")
|
||||
|
|
@ -430,9 +502,10 @@ class VMStorageOCI: PrunableStorage {
|
|||
_ = try VMStorageOCI().open(name)
|
||||
}
|
||||
|
||||
/// Returns `nil` for standalone images and the missing immutable disk-file groups
|
||||
/// for stacked images. `ContentStore.existingContentURL()` intentionally
|
||||
/// validates the digest so corrupt entries are repaired by a normal pull.
|
||||
/// Returns nil for standalone images and the missing immutable disk-file
|
||||
/// groups for stacked images. Like existing standalone cached images, cache hits trust
|
||||
/// already-installed files; checking size still repairs truncated entries
|
||||
/// without hashing a large prewarmed base on every pull.
|
||||
private func missingStackedDiskFileGroups(for manifest: OCIManifest) throws -> [TartDiskFileGroup]? {
|
||||
guard case .stacked(let base, let overlays) = try manifest.tartDiskRepresentation() else {
|
||||
return nil
|
||||
|
|
@ -441,10 +514,7 @@ class VMStorageOCI: PrunableStorage {
|
|||
let contentStore = try ContentStore()
|
||||
var missingGroups: [TartDiskFileGroup] = []
|
||||
for group in [base] + overlays {
|
||||
guard let contentDigest = group.contentDigest else {
|
||||
throw OCIManifestValidationError.invalidDiskMetadata("stacked disk files need a whole-file content digest")
|
||||
}
|
||||
if try contentStore.existingContentURL(for: contentDigest) == nil {
|
||||
if try !hasUsableCachedDiskFile(group, contentStore: contentStore) {
|
||||
missingGroups.append(group)
|
||||
}
|
||||
}
|
||||
|
|
@ -462,23 +532,47 @@ class VMStorageOCI: PrunableStorage {
|
|||
}
|
||||
|
||||
let contentStore = try ContentStore()
|
||||
// Content-store entries are verified when installed. Avoid hashing a
|
||||
// potentially large prewarmed base again on every stacked pull.
|
||||
guard try contentStore.contentURLIfPresent(for: contentDigest) == nil else {
|
||||
return
|
||||
}
|
||||
var attemptedCandidates = Swift.Set<String>()
|
||||
while true {
|
||||
// Keep the source record alive only while cloning its disk. The pull's
|
||||
// in-progress manifest already protects the destination content digest,
|
||||
// so hashing and installing the staged clone need not hold the global
|
||||
// prune lock.
|
||||
let temporaryURL = try contentStore.withPruneLock { () -> URL? in
|
||||
// Content-store entries are verified when installed. Avoid hashing a
|
||||
// potentially large prewarmed base again on every stacked pull.
|
||||
guard try contentStore.contentURLIfPresent(for: contentDigest) == nil else {
|
||||
return nil
|
||||
}
|
||||
|
||||
for (_, vmDir, isSymlink) in try list() where !isSymlink && vmDir.isStandalone {
|
||||
guard let manifestData = try? Data(contentsOf: vmDir.manifestURL),
|
||||
let candidateManifest = try? OCIManifest(fromJSON: manifestData),
|
||||
case .flat(let candidateBase) = try? candidateManifest.tartDiskRepresentation(),
|
||||
diskChunksMatch(candidateBase.chunks, base.chunks) else {
|
||||
continue
|
||||
for (_, vmDir, isSymlink) in try list() where !isSymlink && vmDir.isStandalone {
|
||||
guard !attemptedCandidates.contains(vmDir.baseURL.path),
|
||||
let manifestData = try? Data(contentsOf: vmDir.manifestURL),
|
||||
let candidateManifest = try? OCIManifest(fromJSON: manifestData),
|
||||
case .flat(let candidateBase) = try? candidateManifest.tartDiskRepresentation(),
|
||||
diskChunksMatch(candidateBase.chunks, base.chunks) else {
|
||||
continue
|
||||
}
|
||||
|
||||
attemptedCandidates.insert(vmDir.baseURL.path)
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||
do {
|
||||
try FileManager.default.copyItem(at: vmDir.diskURL, to: temporaryURL)
|
||||
return temporaryURL
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: temporaryURL)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
guard let temporaryURL else {
|
||||
return
|
||||
}
|
||||
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: contentDigest)
|
||||
do {
|
||||
try FileManager.default.copyItem(at: vmDir.diskURL, to: temporaryURL)
|
||||
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||
return
|
||||
} catch ContentStoreError.contentDigestMismatch {
|
||||
|
|
@ -506,6 +600,17 @@ class VMStorageOCI: PrunableStorage {
|
|||
}
|
||||
}
|
||||
|
||||
private func hasUsableCachedDiskFile(_ group: TartDiskFileGroup, contentStore: ContentStore) throws -> Bool {
|
||||
guard let contentDigest = group.contentDigest,
|
||||
let contentURL = try contentStore.contentURLIfPresent(for: contentDigest),
|
||||
let actualSize = UInt64(exactly: try contentURL.sizeBytes()),
|
||||
let expectedSize = group.uncompressedSize() else {
|
||||
return false
|
||||
}
|
||||
|
||||
return actualSize == expectedSize
|
||||
}
|
||||
|
||||
func linked(from: RemoteName, to: RemoteName) -> Bool {
|
||||
do {
|
||||
let resolvedFrom = try FileManager.default.destinationOfSymbolicLink(atPath: vmURL(from).path)
|
||||
|
|
@ -516,9 +621,13 @@ class VMStorageOCI: PrunableStorage {
|
|||
}
|
||||
|
||||
func link(from: RemoteName, to: RemoteName) throws {
|
||||
try? FileManager.default.removeItem(at: vmURL(from))
|
||||
|
||||
try FileManager.default.createSymbolicLink(at: vmURL(from), withDestinationURL: vmURL(to))
|
||||
// Export resolves mutable tags while holding this same lock, so replace
|
||||
// the symlink atomically with respect to stacked archive staging.
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
try? FileManager.default.removeItem(at: vmURL(from))
|
||||
try FileManager.default.createSymbolicLink(at: vmURL(from), withDestinationURL: vmURL(to))
|
||||
}
|
||||
|
||||
try gc()
|
||||
}
|
||||
|
|
@ -594,6 +703,108 @@ class VMStorageOCI: PrunableStorage {
|
|||
try LocalLayerCache(choosen.name, choosen.deduplicatedBytes, choosen.vmDir.diskURL, choosen.manifest)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns content referenced outside the OCI cache, optionally including
|
||||
/// references published by retained cached images.
|
||||
private func referencedContentDigests(includeCachedImages: Bool) throws -> Swift.Set<String> {
|
||||
var result = Swift.Set<String>()
|
||||
|
||||
for (_, vmDir) in try VMStorageLocal().list() where vmDir.isStackedVM {
|
||||
result.formUnion(try vmDir.diskContentDigests())
|
||||
}
|
||||
|
||||
// Clone, pull, and import publish their manifest before installing
|
||||
// immutable content. Include partially populated temporary directories so
|
||||
// pruning cannot race those operations.
|
||||
for url in try FileManager.default.contentsOfDirectory(
|
||||
at: Config().tartTmpDir,
|
||||
includingPropertiesForKeys: [],
|
||||
options: .skipsHiddenFiles
|
||||
) {
|
||||
let vmDir = VMDirectory(baseURL: url)
|
||||
guard FileManager.default.fileExists(atPath: vmDir.manifestURL.path),
|
||||
let contentDigests = try? vmDir.diskContentDigests() else {
|
||||
continue
|
||||
}
|
||||
|
||||
result.formUnion(contentDigests)
|
||||
}
|
||||
|
||||
if includeCachedImages {
|
||||
for (_, vmDir, isSymlink) in try list() where !isSymlink && vmDir.isStackedCachedImage {
|
||||
// Malformed cached records are invalid references. Keep scanning so
|
||||
// one interrupted population does not disable content GC globally.
|
||||
if let contentDigests = try? vmDir.diskContentDigests() {
|
||||
result.formUnion(contentDigests)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
fileprivate func deleteContentIfUnused(_ url: URL) throws {
|
||||
let contentStore = try ContentStore()
|
||||
try contentStore.withPruneLock {
|
||||
let referencedContentDigests = try referencedContentDigests(includeCachedImages: true)
|
||||
let stillPrunable = try contentStore.prunables(excluding: referencedContentDigests).contains {
|
||||
$0.resolvingSymlinksInPath() == url.resolvingSymlinksInPath()
|
||||
}
|
||||
if stillPrunable {
|
||||
try FileManager.default.removeItem(at: url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private struct ContentPrunable: Prunable {
|
||||
let url: URL
|
||||
|
||||
func delete() throws {
|
||||
try VMStorageOCI().deleteContentIfUnused(url)
|
||||
}
|
||||
|
||||
func accessDate() throws -> Date {
|
||||
try url.accessDate()
|
||||
}
|
||||
|
||||
func sizeBytes() throws -> Int {
|
||||
try url.sizeBytes()
|
||||
}
|
||||
|
||||
func allocatedSizeBytes() throws -> Int {
|
||||
try url.allocatedSizeBytes()
|
||||
}
|
||||
}
|
||||
|
||||
/// A digest-addressed cached image plus immutable content attributed to the
|
||||
/// final remote reference that can release it.
|
||||
private struct CachedImagePrunable: Prunable {
|
||||
let vmDir: VMDirectory
|
||||
let ownedContentURLs: [URL]
|
||||
|
||||
var url: URL {
|
||||
vmDir.url
|
||||
}
|
||||
|
||||
func delete() throws {
|
||||
try vmDir.delete()
|
||||
// Deleting a record can make attributed content unreferenced. Run GC now
|
||||
// so one prune invocation reclaims those bytes.
|
||||
try VMStorageOCI().gcContent()
|
||||
}
|
||||
|
||||
func accessDate() throws -> Date {
|
||||
try vmDir.accessDate()
|
||||
}
|
||||
|
||||
func sizeBytes() throws -> Int {
|
||||
try vmDir.sizeBytes() + ownedContentURLs.map { try $0.sizeBytes() }.reduce(0, +)
|
||||
}
|
||||
|
||||
func allocatedSizeBytes() throws -> Int {
|
||||
try vmDir.allocatedSizeBytes() + ownedContentURLs.map { try $0.allocatedSizeBytes() }.reduce(0, +)
|
||||
}
|
||||
}
|
||||
|
||||
extension URL {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,23 @@ import XCTest
|
|||
@testable import tart
|
||||
|
||||
final class CommandBehaviorTests: XCTestCase {
|
||||
func testStandaloneDeleteDoesNotInitializeContentStore() throws {
|
||||
try withTemporaryTartHome {
|
||||
let vmDir = try VMStorageLocal().create("standalone")
|
||||
try config().save(toURL: vmDir.configURL)
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.nvramURL.path, contents: Data()))
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: vmDir.diskURL.path, contents: Data()))
|
||||
|
||||
let contentStoreURL = try Config().tartCacheDir.appendingPathComponent("content", isDirectory: true)
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: contentStoreURL.path))
|
||||
|
||||
try vmDir.delete()
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: vmDir.baseURL.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: contentStoreURL.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testSetDiskRejectsStackedVMBeforeSavingConfig() async throws {
|
||||
try await withTemporaryTartHome {
|
||||
let vmDir = try VMStorageLocal().create("stacked")
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ final class ContentStoreTests: XCTestCase {
|
|||
try Data("corrupt".utf8).write(to: contentURL)
|
||||
|
||||
XCTAssertNil(try store.existingContentURL(for: expectedDigest))
|
||||
XCTAssertEqual(try store.contentURLIfPresent(for: expectedDigest), contentURL)
|
||||
}
|
||||
|
||||
func testResumableAndLockURLsAreStablePerDigest() throws {
|
||||
|
|
@ -113,6 +114,24 @@ final class ContentStoreTests: XCTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
func testContentURLUnderArbitraryRootHasNoSideEffects() throws {
|
||||
let rootURL = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||
addTeardownBlock {
|
||||
try? FileManager.default.removeItem(at: rootURL)
|
||||
}
|
||||
let digest = Digest.hash(Data("content".utf8))
|
||||
|
||||
let contentStore = try temporaryStore()
|
||||
let contentURL = try contentStore.contentURL(for: digest, under: rootURL)
|
||||
|
||||
XCTAssertEqual(
|
||||
contentURL,
|
||||
rootURL.appendingPathComponent("sha256", isDirectory: true)
|
||||
.appendingPathComponent(String(digest.dropFirst("sha256:".count)))
|
||||
)
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: rootURL.path))
|
||||
}
|
||||
|
||||
private func temporaryStore() throws -> ContentStore {
|
||||
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
|
||||
addTeardownBlock {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,33 @@ final class OCIManifestTests: XCTestCase {
|
|||
))
|
||||
}
|
||||
|
||||
func testDiskFileGroupUncompressedSize() {
|
||||
let first = chunk(mediaType: diskV2MediaType, suffix: "base-0")
|
||||
let second = chunk(mediaType: diskV2MediaType, suffix: "base-1")
|
||||
XCTAssertEqual(TartDiskFileGroup(kind: .base, chunks: [first, second], contentDigest: nil).uncompressedSize(), 2)
|
||||
|
||||
var overflowing = first
|
||||
overflowing.annotations?[uncompressedSizeAnnotation] = String(UInt64.max)
|
||||
XCTAssertNil(TartDiskFileGroup(kind: .base, chunks: [overflowing, second], contentDigest: nil).uncompressedSize())
|
||||
}
|
||||
|
||||
func testDiskContentDigests() throws {
|
||||
let flatBase = chunk(mediaType: diskV2MediaType, suffix: "flat", diskFileDigest: "sha256:flat")
|
||||
XCTAssertEqual(try manifest(diskDescriptors: [flatBase]).diskContentDigests(), ["sha256:flat"])
|
||||
|
||||
let stackedBase = chunk(mediaType: diskV2MediaType, suffix: "base", diskFileDigest: "sha256:base")
|
||||
let overlay = chunk(
|
||||
mediaType: asifOverlayMediaType,
|
||||
suffix: "overlay",
|
||||
diskFileDigest: "sha256:overlay",
|
||||
chunkCount: 1
|
||||
)
|
||||
XCTAssertEqual(
|
||||
try manifest(diskDescriptors: [stackedBase, overlay]).diskContentDigests(),
|
||||
["sha256:base", "sha256:overlay"]
|
||||
)
|
||||
}
|
||||
|
||||
func testStackedRepresentationRequiresBaseDigest() throws {
|
||||
let base = chunk(mediaType: diskV2MediaType, suffix: "base-0")
|
||||
let overlay = chunk(mediaType: asifOverlayMediaType, suffix: "overlay-0", diskFileDigest: "sha256:overlay", chunkCount: 1)
|
||||
|
|
|
|||
|
|
@ -85,10 +85,10 @@ final class VMStorageOCITests: XCTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
func testStackedCacheHitRequiresVerifiedContentAndSizesMissingFiles() throws {
|
||||
func testStackedCacheHitRequiresExpectedContentSizes() throws {
|
||||
try withTemporaryTartHome {
|
||||
let baseData = Data("base".utf8)
|
||||
let overlayData = Data("overlay".utf8)
|
||||
let baseData = Data(repeating: 0x41, count: 10)
|
||||
let overlayData = Data(repeating: 0x42, count: 20)
|
||||
let baseDigest = Digest.hash(baseData)
|
||||
let overlayDigest = Digest.hash(overlayData)
|
||||
let manifest = try stackedManifest(
|
||||
|
|
@ -216,6 +216,518 @@ final class VMStorageOCITests: XCTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
func testGCPrunesOnlyUnreferencedContent() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let referenced = try installContent(Data("referenced".utf8), into: contentStore)
|
||||
let unreferenced = try installContent(Data("unreferenced".utf8), into: contentStore)
|
||||
|
||||
let stacked = try VMStorageLocal().create("stacked")
|
||||
try config().save(toURL: stacked.configURL)
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: stacked.nvramURL.path, contents: Data()))
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: stacked.overlayURL.path, contents: Data()))
|
||||
try pinnedBaseManifest(contentDigest: referenced.digest).toJSON().write(to: stacked.manifestURL)
|
||||
|
||||
try VMStorageOCI().gc()
|
||||
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: referenced.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: unreferenced.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testGCPrunesContentWithoutOCIStorageDirectory() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let unreferenced = try installContent(Data("unreferenced".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: storage.baseURL.path))
|
||||
|
||||
try storage.gc()
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: unreferenced.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testGCDoesNotPruneContentReferencedByInProgressManifest() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let referenced = try installContent(Data("in-progress".utf8), into: contentStore)
|
||||
let temporaryVMDir = try VMDirectory.temporary()
|
||||
|
||||
// Pull and clone publish the manifest before config, NVRAM, or a
|
||||
// writable overlay necessarily exist.
|
||||
try pinnedBaseManifest(contentDigest: referenced.digest).toJSON().write(to: temporaryVMDir.manifestURL)
|
||||
|
||||
try VMStorageOCI().gc()
|
||||
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: referenced.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testStalePrunableDoesNotDeleteNewlyReferencedContent() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let content = try installContent(Data("new-reference".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
let candidate = try XCTUnwrap(storage.prunables().first {
|
||||
$0.url.resolvingSymlinksInPath() == content.url.resolvingSymlinksInPath()
|
||||
})
|
||||
|
||||
// Simulate a clone or pull publishing its manifest after prune built
|
||||
// the candidate list but before deletion starts.
|
||||
let temporaryVMDir = try VMDirectory.temporary()
|
||||
try contentStore.withPruneLock {
|
||||
try pinnedBaseManifest(contentDigest: content.digest).toJSON().write(to: temporaryVMDir.manifestURL)
|
||||
}
|
||||
|
||||
try candidate.delete()
|
||||
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: content.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testVMDirectoryDeletionOfStackedOCIRecordWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let record = try createRecord(for: stackedManifest(), in: storage)
|
||||
|
||||
try assertDeletionWaitsForPruneLock(record: record) {
|
||||
try record.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testStorageDeletionOfStackedOCIRecordWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let manifest = try stackedManifest()
|
||||
let name = try digestName(for: manifest)
|
||||
let record = try createRecord(for: manifest, in: storage)
|
||||
|
||||
try assertDeletionWaitsForPruneLock(record: record) {
|
||||
try storage.delete(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testStorageDeletionOfIncompleteManifestRecordWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let name = RemoteName(
|
||||
host: "example.com",
|
||||
namespace: "org/image",
|
||||
reference: Reference(digest: "sha256:incomplete")
|
||||
)
|
||||
let record = try storage.create(name)
|
||||
try Data("{}".utf8).write(to: record.manifestURL)
|
||||
|
||||
try assertDeletionWaitsForPruneLock(record: record) {
|
||||
try storage.delete(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testVMDirectoryDeletionOfUnpublishedRecordWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let record = try storage.create(RemoteName(
|
||||
host: "example.com",
|
||||
namespace: "org/image",
|
||||
reference: Reference(digest: "sha256:unpublished")
|
||||
))
|
||||
|
||||
try assertDeletionWaitsForPruneLock(record: record) {
|
||||
try record.removeFromDisk()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testTagReplacementWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let firstManifest = try stackedManifest(baseContentDigest: "sha256:first")
|
||||
let secondManifest = try stackedManifest(baseContentDigest: "sha256:second")
|
||||
let firstName = try digestName(for: firstManifest)
|
||||
let secondName = try digestName(for: secondManifest)
|
||||
_ = try createRecord(for: firstManifest, in: storage)
|
||||
_ = try createRecord(for: secondManifest, in: storage)
|
||||
let tagName = RemoteName(
|
||||
host: secondName.host,
|
||||
namespace: secondName.namespace,
|
||||
reference: Reference(tag: "latest")
|
||||
)
|
||||
|
||||
let contentStore = try ContentStore()
|
||||
let lockHeld = DispatchSemaphore(value: 0)
|
||||
let releaseLock = DispatchSemaphore(value: 0)
|
||||
let replacementStarted = DispatchSemaphore(value: 0)
|
||||
let replacementFinished = DispatchSemaphore(value: 0)
|
||||
|
||||
DispatchQueue.global().async {
|
||||
try? contentStore.withPruneLock {
|
||||
lockHeld.signal()
|
||||
releaseLock.wait()
|
||||
}
|
||||
}
|
||||
XCTAssertEqual(lockHeld.wait(timeout: .now() + 1), .success)
|
||||
|
||||
DispatchQueue.global().async {
|
||||
replacementStarted.signal()
|
||||
try? storage.link(from: tagName, to: secondName)
|
||||
replacementFinished.signal()
|
||||
}
|
||||
XCTAssertEqual(replacementStarted.wait(timeout: .now() + 1), .success)
|
||||
XCTAssertEqual(replacementFinished.wait(timeout: .now() + 0.1), .timedOut)
|
||||
|
||||
releaseLock.signal()
|
||||
XCTAssertEqual(replacementFinished.wait(timeout: .now() + 1), .success)
|
||||
XCTAssertTrue(storage.linked(from: tagName, to: secondName))
|
||||
XCTAssertFalse(storage.linked(from: tagName, to: firstName))
|
||||
}
|
||||
}
|
||||
|
||||
func testGCDeletionOfStackedOCIRecordWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let record = try createRecord(for: stackedManifest(), in: storage)
|
||||
|
||||
try assertDeletionWaitsForPruneLock(record: record) {
|
||||
try storage.gc()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testTemporaryManifestGCWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let temporaryVMDir = try VMDirectory.temporary()
|
||||
try stackedManifest().toJSON().write(to: temporaryVMDir.manifestURL)
|
||||
|
||||
try assertDeletionWaitsForPruneLock(record: temporaryVMDir) {
|
||||
try Config().gc()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testLockedTemporaryDirectorySurvivesGarbageCollection() throws {
|
||||
try withTemporaryTartHome {
|
||||
let temporaryVMDir = try VMDirectory.temporary()
|
||||
let lock = try FileLock(lockURL: temporaryVMDir.baseURL)
|
||||
try lock.lock()
|
||||
|
||||
try Config().gc()
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: temporaryVMDir.baseURL.path))
|
||||
|
||||
try lock.unlock()
|
||||
try Config().gc()
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: temporaryVMDir.baseURL.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testMovingStackedOCIRecordWaitsForPruneLock() throws {
|
||||
try withTemporaryTartHome {
|
||||
let storage = try VMStorageOCI()
|
||||
let source = try temporaryVMDirectory()
|
||||
let manifest = try stackedManifest()
|
||||
let name = try digestName(for: manifest)
|
||||
try config().save(toURL: source.configURL)
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: source.nvramURL.path, contents: Data()))
|
||||
try manifest.toJSON().write(to: source.manifestURL)
|
||||
|
||||
let contentStore = try ContentStore()
|
||||
let lockHeld = DispatchSemaphore(value: 0)
|
||||
let releaseLock = DispatchSemaphore(value: 0)
|
||||
let moveStarted = DispatchSemaphore(value: 0)
|
||||
let moveFinished = DispatchSemaphore(value: 0)
|
||||
|
||||
DispatchQueue.global().async {
|
||||
try? contentStore.withPruneLock {
|
||||
lockHeld.signal()
|
||||
releaseLock.wait()
|
||||
}
|
||||
}
|
||||
XCTAssertEqual(lockHeld.wait(timeout: .now() + 1), .success)
|
||||
|
||||
DispatchQueue.global().async {
|
||||
moveStarted.signal()
|
||||
try? storage.move(name, from: source)
|
||||
moveFinished.signal()
|
||||
}
|
||||
XCTAssertEqual(moveStarted.wait(timeout: .now() + 1), .success)
|
||||
XCTAssertEqual(moveFinished.wait(timeout: .now() + 0.1), .timedOut)
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: source.baseURL.path))
|
||||
|
||||
releaseLock.signal()
|
||||
XCTAssertEqual(moveFinished.wait(timeout: .now() + 1), .success)
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: source.baseURL.path))
|
||||
XCTAssertTrue(try storage.open(name).isStackedCachedImage)
|
||||
}
|
||||
}
|
||||
|
||||
func testPruningLastStackedOCIRecordReclaimsItsContent() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let baseContent = try installContent(Data("record-only-base".utf8), into: contentStore)
|
||||
let overlayContent = try installContent(Data("record-only-overlay".utf8), into: contentStore)
|
||||
let manifest = try stackedManifest(
|
||||
baseContentDigest: baseContent.digest,
|
||||
overlayContentDigest: overlayContent.digest,
|
||||
baseUncompressedSize: UInt64(try baseContent.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try overlayContent.url.sizeBytes())
|
||||
)
|
||||
let name = try digestName(for: manifest)
|
||||
let storage = try VMStorageOCI()
|
||||
let record = try storage.create(name)
|
||||
try config().save(toURL: record.configURL)
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||
try manifest.toJSON().write(to: record.manifestURL)
|
||||
|
||||
let candidate = try XCTUnwrap(storage.prunables().first {
|
||||
$0.url.lastPathComponent == record.url.lastPathComponent
|
||||
})
|
||||
XCTAssertGreaterThanOrEqual(
|
||||
try candidate.allocatedSizeBytes(),
|
||||
try baseContent.url.allocatedSizeBytes() + overlayContent.url.allocatedSizeBytes()
|
||||
)
|
||||
|
||||
// The record itself fits in this budget, so pruning only succeeds if it
|
||||
// accounts for the immutable content released with the final reference.
|
||||
try Prune.pruneSpaceBudget(
|
||||
prunableStorages: [storage],
|
||||
spaceBudgetBytes: UInt64(try record.allocatedSizeBytes())
|
||||
)
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: record.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: baseContent.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: overlayContent.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testMalformedStackedOCIRecordDoesNotBlockPruning() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let baseContent = try installContent(Data("valid-base".utf8), into: contentStore)
|
||||
let overlayContent = try installContent(Data("valid-overlay".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
let validRecord = try createRecord(for: stackedManifest(
|
||||
baseContentDigest: baseContent.digest,
|
||||
overlayContentDigest: overlayContent.digest,
|
||||
baseUncompressedSize: UInt64(try baseContent.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try overlayContent.url.sizeBytes())
|
||||
), in: storage)
|
||||
|
||||
let malformedRecord = try storage.create(RemoteName(
|
||||
host: "example.com",
|
||||
namespace: "org/image",
|
||||
reference: Reference(digest: "sha256:malformed")
|
||||
))
|
||||
try config().save(toURL: malformedRecord.configURL)
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: malformedRecord.nvramURL.path, contents: Data()))
|
||||
try Data("{".utf8).write(to: malformedRecord.manifestURL)
|
||||
|
||||
XCTAssertNoThrow(try storage.prunables())
|
||||
try Prune.pruneSpaceBudget(prunableStorages: [storage], spaceBudgetBytes: 0)
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: validRecord.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: malformedRecord.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: baseContent.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: overlayContent.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testPruningOneStackedOCIRecordPreservesSharedContent() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let baseContent = try installContent(Data("shared-base".utf8), into: contentStore)
|
||||
let firstOverlay = try installContent(Data("first-overlay".utf8), into: contentStore)
|
||||
let secondOverlay = try installContent(Data("second-overlay".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
|
||||
let firstManifest = try stackedManifest(
|
||||
baseContentDigest: baseContent.digest,
|
||||
overlayContentDigest: firstOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try baseContent.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try firstOverlay.url.sizeBytes())
|
||||
)
|
||||
let secondManifest = try stackedManifest(
|
||||
baseContentDigest: baseContent.digest,
|
||||
overlayContentDigest: secondOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try baseContent.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try secondOverlay.url.sizeBytes())
|
||||
)
|
||||
let firstRecord = try createRecord(for: firstManifest, in: storage)
|
||||
let secondRecord = try createRecord(for: secondManifest, in: storage)
|
||||
|
||||
let firstCandidate = try XCTUnwrap(storage.prunables().first {
|
||||
$0.url.lastPathComponent == firstRecord.url.lastPathComponent
|
||||
})
|
||||
try firstCandidate.delete()
|
||||
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: baseContent.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: firstOverlay.url.path))
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: secondOverlay.url.path))
|
||||
|
||||
let secondCandidate = try XCTUnwrap(storage.prunables().first {
|
||||
$0.url.lastPathComponent == secondRecord.url.lastPathComponent
|
||||
})
|
||||
try secondCandidate.delete()
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: baseContent.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: secondOverlay.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testSpaceBudgetRecomputesSharedContentAfterOwnerDeletion() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let baseContent = try installContent(Data("shared-base".utf8), into: contentStore)
|
||||
let firstOverlay = try installContent(Data("first-overlay".utf8), into: contentStore)
|
||||
let secondOverlay = try installContent(Data("second-overlay".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
|
||||
let firstManifest = try stackedManifest(
|
||||
baseContentDigest: baseContent.digest,
|
||||
overlayContentDigest: firstOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try baseContent.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try firstOverlay.url.sizeBytes())
|
||||
)
|
||||
let secondManifest = try stackedManifest(
|
||||
baseContentDigest: baseContent.digest,
|
||||
overlayContentDigest: secondOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try baseContent.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try secondOverlay.url.sizeBytes())
|
||||
)
|
||||
let olderRecord = try createRecord(for: firstManifest, in: storage)
|
||||
let newerRecord = try createRecord(for: secondManifest, in: storage)
|
||||
try olderRecord.url.updateAccessDate(Date(timeIntervalSince1970: 1))
|
||||
try newerRecord.url.updateAccessDate(Date(timeIntervalSince1970: 2))
|
||||
|
||||
let olderCandidate = try XCTUnwrap(storage.prunables().first {
|
||||
$0.url.lastPathComponent == olderRecord.url.lastPathComponent
|
||||
})
|
||||
let budget = UInt64(try olderCandidate.allocatedSizeBytes())
|
||||
|
||||
// On the first pass the newer record owns the shared base and is
|
||||
// selected for deletion, while the older record fits this budget.
|
||||
// Recomputing must then charge the surviving record for the base and
|
||||
// prune it too.
|
||||
try Prune.pruneSpaceBudget(
|
||||
prunableStorages: [storage],
|
||||
spaceBudgetBytes: budget
|
||||
)
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: olderRecord.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: newerRecord.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: baseContent.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: firstOverlay.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: secondOverlay.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testSpaceBudgetRecomputesBeforeDeletingAnotherCandidate() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let sharedBase = try installContent(Data(repeating: 0x41, count: 128 * 1024), into: contentStore)
|
||||
let newestOverlay = try installContent(Data("newest-overlay".utf8), into: contentStore)
|
||||
let middleOverlay = try installContent(Data("middle-overlay".utf8), into: contentStore)
|
||||
let retainedBase = try installContent(Data(repeating: 0x42, count: 32 * 1024), into: contentStore)
|
||||
let retainedOverlay = try installContent(Data("retained-overlay".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
|
||||
let newest = try createRecord(for: stackedManifest(
|
||||
baseContentDigest: sharedBase.digest,
|
||||
overlayContentDigest: newestOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try sharedBase.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try newestOverlay.url.sizeBytes())
|
||||
), in: storage)
|
||||
let middle = try createRecord(for: stackedManifest(
|
||||
baseContentDigest: sharedBase.digest,
|
||||
overlayContentDigest: middleOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try sharedBase.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try middleOverlay.url.sizeBytes())
|
||||
), in: storage)
|
||||
let retained = try createRecord(for: stackedManifest(
|
||||
baseContentDigest: retainedBase.digest,
|
||||
overlayContentDigest: retainedOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try retainedBase.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try retainedOverlay.url.sizeBytes())
|
||||
), in: storage)
|
||||
try newest.url.updateAccessDate(Date(timeIntervalSince1970: 3))
|
||||
try middle.url.updateAccessDate(Date(timeIntervalSince1970: 2))
|
||||
try retained.url.updateAccessDate(Date(timeIntervalSince1970: 1))
|
||||
|
||||
let retainedCandidate = try XCTUnwrap(storage.prunables().first {
|
||||
$0.url.lastPathComponent == retained.url.lastPathComponent
|
||||
})
|
||||
|
||||
// Initially the newest record owns the shared base and is too large.
|
||||
// The middle record appears small enough to retain, making the oldest
|
||||
// unrelated record look like a second deletion candidate. After the
|
||||
// first deletion, ownership moves to the middle record; recomputing
|
||||
// before selecting again must delete it and preserve the unrelated one.
|
||||
try Prune.pruneSpaceBudget(
|
||||
prunableStorages: [storage],
|
||||
spaceBudgetBytes: UInt64(try retainedCandidate.allocatedSizeBytes())
|
||||
)
|
||||
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: newest.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: middle.url.path))
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: retained.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: sharedBase.url.path))
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: retainedBase.url.path))
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: retainedOverlay.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
func testAutomaticReclaimRecomputesSharedContentAfterOwnerDeletion() throws {
|
||||
try withTemporaryTartHome {
|
||||
let contentStore = try ContentStore()
|
||||
let sharedBase = try installContent(Data(repeating: 0x41, count: 128 * 1024), into: contentStore)
|
||||
let initiatorOverlay = try installContent(Data("initiator-overlay".utf8), into: contentStore)
|
||||
let ownerOverlay = try installContent(Data("owner-overlay".utf8), into: contentStore)
|
||||
let unrelatedBase = try installContent(Data("unrelated-base".utf8), into: contentStore)
|
||||
let unrelatedOverlay = try installContent(Data("unrelated-overlay".utf8), into: contentStore)
|
||||
let storage = try VMStorageOCI()
|
||||
|
||||
let initiatorManifest = try stackedManifest(
|
||||
baseContentDigest: sharedBase.digest,
|
||||
overlayContentDigest: initiatorOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try sharedBase.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try initiatorOverlay.url.sizeBytes())
|
||||
)
|
||||
let ownerManifest = try stackedManifest(
|
||||
baseContentDigest: sharedBase.digest,
|
||||
overlayContentDigest: ownerOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try sharedBase.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try ownerOverlay.url.sizeBytes())
|
||||
)
|
||||
let unrelatedManifest = try stackedManifest(
|
||||
baseContentDigest: unrelatedBase.digest,
|
||||
overlayContentDigest: unrelatedOverlay.digest,
|
||||
baseUncompressedSize: UInt64(try unrelatedBase.url.sizeBytes()),
|
||||
overlayUncompressedSize: UInt64(try unrelatedOverlay.url.sizeBytes())
|
||||
)
|
||||
let initiator = try createRecord(for: initiatorManifest, in: storage)
|
||||
let owner = try createRecord(for: ownerManifest, in: storage)
|
||||
let unrelated = try createRecord(for: unrelatedManifest, in: storage)
|
||||
try initiator.url.updateAccessDate(Date(timeIntervalSince1970: 1))
|
||||
try owner.url.updateAccessDate(Date(timeIntervalSince1970: 2))
|
||||
try unrelated.url.updateAccessDate(Date(timeIntervalSince1970: 3))
|
||||
|
||||
let sharedBaseSize = UInt64(try sharedBase.url.allocatedSizeBytes())
|
||||
|
||||
// The owner is the first deletable record and is initially charged for
|
||||
// the shared base. Deleting it cannot reclaim that base because the
|
||||
// protected initiator still references it, so reclaim must continue.
|
||||
try Prune.reclaimIfPossible(sharedBaseSize, initiator)
|
||||
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: initiator.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: owner.url.path))
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: unrelated.url.path))
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: sharedBase.url.path))
|
||||
}
|
||||
}
|
||||
|
||||
#if canImport(DiskImageKit)
|
||||
@available(macOS 27.0, *)
|
||||
func testPopulateStackedPushedImageCachesImmutableTopOverlay() throws {
|
||||
|
|
@ -265,6 +777,47 @@ final class VMStorageOCITests: XCTestCase {
|
|||
return vmDir
|
||||
}
|
||||
|
||||
private func createRecord(for manifest: OCIManifest, in storage: VMStorageOCI) throws -> VMDirectory {
|
||||
let record = try storage.create(try digestName(for: manifest))
|
||||
try config().save(toURL: record.configURL)
|
||||
XCTAssertTrue(FileManager.default.createFile(atPath: record.nvramURL.path, contents: Data()))
|
||||
try manifest.toJSON().write(to: record.manifestURL)
|
||||
|
||||
return record
|
||||
}
|
||||
|
||||
private func assertDeletionWaitsForPruneLock(
|
||||
record: VMDirectory,
|
||||
deletion: @escaping () throws -> Void
|
||||
) throws {
|
||||
let contentStore = try ContentStore()
|
||||
let lockHeld = DispatchSemaphore(value: 0)
|
||||
let releaseLock = DispatchSemaphore(value: 0)
|
||||
let deletionStarted = DispatchSemaphore(value: 0)
|
||||
let deletionFinished = DispatchSemaphore(value: 0)
|
||||
|
||||
DispatchQueue.global().async {
|
||||
try? contentStore.withPruneLock {
|
||||
lockHeld.signal()
|
||||
releaseLock.wait()
|
||||
}
|
||||
}
|
||||
XCTAssertEqual(lockHeld.wait(timeout: .now() + 1), .success)
|
||||
|
||||
DispatchQueue.global().async {
|
||||
deletionStarted.signal()
|
||||
try? deletion()
|
||||
deletionFinished.signal()
|
||||
}
|
||||
XCTAssertEqual(deletionStarted.wait(timeout: .now() + 1), .success)
|
||||
XCTAssertEqual(deletionFinished.wait(timeout: .now() + 0.1), .timedOut)
|
||||
XCTAssertTrue(FileManager.default.fileExists(atPath: record.baseURL.path))
|
||||
|
||||
releaseLock.signal()
|
||||
XCTAssertEqual(deletionFinished.wait(timeout: .now() + 1), .success)
|
||||
XCTAssertFalse(FileManager.default.fileExists(atPath: record.baseURL.path))
|
||||
}
|
||||
|
||||
#if canImport(DiskImageKit)
|
||||
@available(macOS 27.0, *)
|
||||
private func diskImageSource() throws -> VMDirectory {
|
||||
|
|
@ -337,6 +890,34 @@ final class VMStorageOCITests: XCTestCase {
|
|||
_ = try contentStore.install(temporaryURL, contentDigest: contentDigest)
|
||||
}
|
||||
|
||||
private func pinnedBaseManifest(contentDigest: String) -> OCIManifest {
|
||||
var disk = OCIManifestLayer(
|
||||
mediaType: diskV2MediaType,
|
||||
size: 1,
|
||||
digest: "sha256:disk-transport",
|
||||
uncompressedSize: 1,
|
||||
uncompressedContentDigest: "sha256:disk-chunk"
|
||||
)
|
||||
disk.annotations?[diskFileContentDigestAnnotation] = contentDigest
|
||||
|
||||
return OCIManifest(
|
||||
config: OCIManifestConfig(size: 1, digest: "sha256:oci-config"),
|
||||
layers: [
|
||||
OCIManifestLayer(mediaType: configMediaType, size: 1, digest: "sha256:config"),
|
||||
disk,
|
||||
OCIManifestLayer(mediaType: nvramMediaType, size: 1, digest: "sha256:nvram"),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
private func installContent(_ data: Data, into contentStore: ContentStore) throws -> (digest: String, url: URL) {
|
||||
let digest = Digest.hash(data)
|
||||
let temporaryURL = try contentStore.temporaryContentURL(for: digest)
|
||||
try data.write(to: temporaryURL)
|
||||
|
||||
return (digest, try contentStore.install(temporaryURL, contentDigest: digest))
|
||||
}
|
||||
|
||||
private func digestName(for manifest: OCIManifest) throws -> RemoteName {
|
||||
RemoteName(
|
||||
host: "example.com",
|
||||
|
|
|
|||
Loading…
Reference in New Issue