diff --git a/.cirrus.yml b/.cirrus.yml index 4d8c126..a41c8d8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,11 +8,11 @@ task: name: dev-mini resources: tart-vms: 1 + build_script: + - swift build test_script: - swift test integration_test_script: - # Build Tart - - swift build - codesign --sign - --entitlements Resources/tart-dev.entitlements --force .build/debug/tart - export PATH=$(pwd)/.build/arm64-apple-macosx/debug:$PATH # Run integration tests diff --git a/Sources/tart/LocalLayerCache.swift b/Sources/tart/LocalLayerCache.swift index 58c21ba..3118379 100644 --- a/Sources/tart/LocalLayerCache.swift +++ b/Sources/tart/LocalLayerCache.swift @@ -1,12 +1,19 @@ import Foundation struct LocalLayerCache { + struct DigestInfo { + let range: Range + let compressedDigest: String + let uncompressedContentDigest: String? + } + let name: String let deduplicatedBytes: UInt64 let diskURL: URL private let mappedDisk: Data - private var digestToRange: [String : Range] = [:] + private var digestToRange: [String: DigestInfo] = [:] + private var offsetToRange: [UInt64: DigestInfo] = [:] init?(_ name: String, _ deduplicatedBytes: UInt64, _ diskURL: URL, _ manifest: OCIManifest) throws { self.name = name @@ -24,17 +31,27 @@ struct LocalLayerCache { return nil } - self.digestToRange[layer.digest] = Int(offset).. Data? { - guard let foundRange = self.digestToRange[digest] else { - return nil + func findInfo(digest: String, offsetHint: UInt64) -> DigestInfo? { + // Layers can have the same digests, for example, empty ones. Let's use the offset hint to make a better guess. + if let info = self.offsetToRange[offsetHint], info.compressedDigest == digest { + return info } + return self.digestToRange[digest] + } - return self.mappedDisk.subdata(in: foundRange) + func subdata(_ range: Range) -> Data { + return self.mappedDisk.subdata(in: range) } } diff --git a/Sources/tart/OCI/Layerizer/DiskV2.swift b/Sources/tart/OCI/Layerizer/DiskV2.swift index f315d2f..c91972a 100644 --- a/Sources/tart/OCI/Layerizer/DiskV2.swift +++ b/Sources/tart/OCI/Layerizer/DiskV2.swift @@ -121,11 +121,14 @@ class DiskV2: Disk { // Launch a fetching and decompression task group.addTask { // No need to fetch and decompress anything if we've already done so - if try pullResumed && Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest { - // Update the progress - progress.completedUnitCount += Int64(diskLayer.size) + if pullResumed { + // do not check hash in the condition above to make it lazy e.g. only do expensive calculations if needed + if try Digest.hash(diskURL, offset: diskWritingOffset, size: uncompressedLayerSize) == uncompressedLayerContentDigest { + // Update the progress + progress.completedUnitCount += Int64(diskLayer.size) - return + return + } } // Open the disk file for writing @@ -140,9 +143,17 @@ class DiskV2: Disk { } // Check if we already have this layer contents in the local layer cache - if let localLayerCache = localLayerCache, let data = localLayerCache.find(diskLayer.digest), Digest.hash(data) == uncompressedLayerContentDigest { - // Fulfil the layer contents from the local blob cache - _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data) + if let localLayerCache = localLayerCache, let localLayerInfo = localLayerCache.findInfo(digest: diskLayer.digest, offsetHint: diskWritingOffset) { + // indicates that the locally cloned disk image has the same content at the given offset + let localHit = localLayerInfo.uncompressedContentDigest == uncompressedLayerContentDigest + && localLayerInfo.range.lowerBound == diskWritingOffset + // doesn't seem that localHit can ever be false if the localLayerCache is not nil + // but let's just add extra safety here and check it + if !localHit { + // Fulfil the layer contents from the local blob cache + let data = localLayerCache.subdata(localLayerInfo.range) + _ = try zeroSkippingWrite(disk, rdisk, fsBlockSize, diskWritingOffset, data) + } try disk.close() // Update the progress diff --git a/Sources/tart/URL+Prunable.swift b/Sources/tart/URL+Prunable.swift index 12e3dd1..b16f44e 100644 --- a/Sources/tart/URL+Prunable.swift +++ b/Sources/tart/URL+Prunable.swift @@ -17,7 +17,6 @@ extension URL: Prunable { func deduplicatedSizeBytes() throws -> Int { let values = try resourceValues(forKeys: [.totalFileAllocatedSizeKey, .mayShareFileContentKey]) // make sure the file's origin file is there and duplication works - var dedublicatedSize = 0 if values.mayShareFileContent == true { return Int(deduplicatedBytes()) } diff --git a/Sources/tart/VMStorageOCI.swift b/Sources/tart/VMStorageOCI.swift index 100189e..6be1508 100644 --- a/Sources/tart/VMStorageOCI.swift +++ b/Sources/tart/VMStorageOCI.swift @@ -297,7 +297,7 @@ class VMStorageOCI: PrunableStorage { // Now, find the best match based on how many bytes we'll deduplicate let choosen = candidates.filter { - $0.deduplicatedBytes > 0 + $0.deduplicatedBytes > 1024 * 1024 * 1024 // save at least 1GB }.max { left, right in return left.deduplicatedBytes < right.deduplicatedBytes }