Compare commits

...
13 Commits
Author SHA1 Message Date
Nikolay Edigaryev 59393df2e1 Linux support (#190)
* Linux support

* Support macOS 12

* Improve readability a bit

* Remove useless variable
2022-08-23 22:38:50 +04:00
Nikolay Edigaryev 732c8244a4 Run garbage collection after each symlink and expose --gc in tart prune (#191) 2022-08-23 17:31:02 +04:00
Nikolay Edigaryev 9e69c8c161 testGetAndSetAccessTime: fix flakiness (#187) 2022-08-18 14:42:54 -04:00
Nikolay Edigaryev e4ac2275b9 tart ip: show a warning when DHCP lease and ARP cache entries mismatch (#186) 2022-08-18 22:25:02 +04:00
Nikolay Edigaryev 1a1f19e169 Implement garbage collection when deleting cached OCI VMs (#185)
* Properly calculate remotely-retrieved manifest digests

* Implement garbage collection when deleting cached OCI VMs
2022-08-18 17:13:06 +04:00
Nikolay Edigaryev fea916dacc OCI blob compression: fix Data memory leak when using InputFilter (#183)
* OCI blob compression: fix Data memory leak when using InputFilter

* Rename mappedDiskOffset to mappedDiskReadOffset

* Update progress.completedUnitCount differently
2022-08-17 12:57:39 -04:00
Fedor Korotkov b1be9730c7 Build and release from Ventrura (#182)
Might be related to #180
2022-08-17 17:42:24 +04:00
Nikolay EdigaryevandNikolay Edigaryev 14059a1d6f tart {pull,clone}: add missing --insecure support (#181)
Co-authored-by: Nikolay Edigaryev <edi@microsun.local>
2022-08-17 01:31:19 +04:00
Nikolay Edigaryev 440681320a Introduce "tart prune" command (#164) 2022-08-12 16:43:04 +03:00
Fedor KorotkovandNikolay Edigaryev a80954c888 Do not include Content-Range for monolithic uploads (#179)
* Do not include Content-Range for monolithic uploads

Some registries still assumes it's a chunked upload and verifies the "chunk" size which is too big.

* Update Sources/tart/OCI/Registry.swift

Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com>

Co-authored-by: Nikolay Edigaryev <edigaryev@gmail.com>
2022-08-12 16:33:15 +03:00
Nikolay Edigaryev cc8201dee6 OCI: support insecure registries and custom ports (#174) 2022-08-12 16:24:14 +03:00
Fedor Korotkov 2cab49b3f1 Use Ventura APIs for recovery mode (#175) 2022-08-12 08:02:44 +03:00
Fedor Korotkov 131827802a Friendly error message on Intel (#177)
In case someone tries to install Tart. Fixes #176
2022-08-10 11:13:21 -04:00
34 changed files with 754 additions and 126 deletions
+2 -2
View File
@@ -10,7 +10,7 @@ task:
name: Build name: Build
only_if: $CIRRUS_TAG == '' only_if: $CIRRUS_TAG == ''
macos_instance: macos_instance:
image: ghcr.io/cirruslabs/macos-monterey-xcode:14 image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
build_script: swift build --product tart build_script: swift build --product tart
sign_script: codesign --sign - --entitlements Resources/tart.entitlements --force .build/debug/tart sign_script: codesign --sign - --entitlements Resources/tart.entitlements --force .build/debug/tart
binary_artifacts: binary_artifacts:
@@ -20,7 +20,7 @@ task:
name: Release name: Release
only_if: $CIRRUS_TAG != '' only_if: $CIRRUS_TAG != ''
macos_instance: macos_instance:
image: ghcr.io/cirruslabs/macos-monterey-xcode:14 image: ghcr.io/cirruslabs/macos-ventura-xcode:latest
env: env:
GITHUB_TOKEN: ENCRYPTED[!98ace8259c6024da912c14d5a3c5c6aac186890a8d4819fad78f3e0c41a4e0cd3a2537dd6e91493952fb056fa434be7c!] GITHUB_TOKEN: ENCRYPTED[!98ace8259c6024da912c14d5a3c5c6aac186890a8d4819fad78f3e0c41a4e0cd3a2537dd6e91493952fb056fa434be7c!]
GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!] GORELEASER_KEY: ENCRYPTED[!9b80b6ef684ceaf40edd4c7af93014ee156c8aba7e6e5795f41c482729887b5c31f36b651491d790f1f668670888d9fd!]
+7 -1
View File
@@ -32,7 +32,7 @@ brews:
tap: tap:
owner: cirruslabs owner: cirruslabs
name: homebrew-cli name: homebrew-cli
caveats: See the Github repository for more information caveats: See the GitHub repository for more information
homepage: https://github.com/cirruslabs/tart homepage: https://github.com/cirruslabs/tart
description: Run macOS VMs on Apple Silicon description: Run macOS VMs on Apple Silicon
skip_upload: auto skip_upload: auto
@@ -40,3 +40,9 @@ brews:
- "cirruslabs/cli/softnet" - "cirruslabs/cli/softnet"
custom_block: | custom_block: |
depends_on :macos => :monterey depends_on :macos => :monterey
on_macos do
unless Hardware::CPU.arm?
odie "Tart only works on Apple Silicon!"
end
end
+9
View File
@@ -126,6 +126,15 @@
"version" : "0.9.2" "version" : "0.9.2"
} }
}, },
{
"identity" : "swiftdate",
"kind" : "remoteSourceControl",
"location" : "https://github.com/malcommac/SwiftDate",
"state" : {
"revision" : "6190d0cefff3013e77ed567e6b074f324e5c5bf5",
"version" : "6.3.1"
}
},
{ {
"identity" : "xctest-dynamic-overlay", "identity" : "xctest-dynamic-overlay",
"kind" : "remoteSourceControl", "kind" : "remoteSourceControl",
+3 -1
View File
@@ -1,4 +1,4 @@
// swift-tools-version:5.6 // swift-tools-version:5.7
import PackageDescription import PackageDescription
let package = Package( let package = Package(
@@ -15,6 +15,7 @@ let package = Package(
.package(url: "https://github.com/pointfreeco/swift-parsing", from: "0.9.2"), .package(url: "https://github.com/pointfreeco/swift-parsing", from: "0.9.2"),
.package(url: "https://github.com/swift-server/async-http-client", from: "1.11.4"), .package(url: "https://github.com/swift-server/async-http-client", from: "1.11.4"),
.package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"), .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"),
.package(url: "https://github.com/malcommac/SwiftDate", from: "6.3.1")
], ],
targets: [ targets: [
.executableTarget(name: "tart", dependencies: [ .executableTarget(name: "tart", dependencies: [
@@ -23,6 +24,7 @@ let package = Package(
.product(name: "AsyncHTTPClient", package: "async-http-client"), .product(name: "AsyncHTTPClient", package: "async-http-client"),
.product(name: "Dynamic", package: "Dynamic"), .product(name: "Dynamic", package: "Dynamic"),
.product(name: "Parsing", package: "swift-parsing"), .product(name: "Parsing", package: "swift-parsing"),
.product(name: "SwiftDate", package: "SwiftDate"),
]), ]),
.testTarget(name: "TartTests", dependencies: ["tart"]) .testTarget(name: "TartTests", dependencies: ["tart"])
] ]
+4 -1
View File
@@ -11,6 +11,9 @@ struct Clone: AsyncParsableCommand {
@Argument(help: "new VM name") @Argument(help: "new VM name")
var newName: String var newName: String
@Flag(help: "connect to the OCI registry via insecure HTTP protocol")
var insecure: Bool = false
func validate() throws { func validate() throws {
if newName.contains("/") { if newName.contains("/") {
throw ValidationError("<new-name> should be a local name") throw ValidationError("<new-name> should be a local name")
@@ -24,7 +27,7 @@ struct Clone: AsyncParsableCommand {
if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) { if let remoteName = try? RemoteName(sourceName), !ociStorage.exists(remoteName) {
// Pull the VM in case it's OCI-based and doesn't exist locally yet // Pull the VM in case it's OCI-based and doesn't exist locally yet
let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace) let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
try await ociStorage.pull(remoteName, registry: registry) try await ociStorage.pull(remoteName, registry: registry)
} }
+20 -7
View File
@@ -9,15 +9,18 @@ struct Create: AsyncParsableCommand {
@Argument(help: "VM name") @Argument(help: "VM name")
var name: String var name: String
@Option(help: ArgumentHelp("Path to the IPSW file (or \"latest\") to fetch the latest appropriate IPSW", valueName: "path")) @Option(help: ArgumentHelp("create a macOS VM using path to the IPSW file (or \"latest\") to fetch the latest appropriate IPSW", valueName: "path"))
var fromIPSW: String? var fromIPSW: String?
@Flag(help: "create a Linux VM")
var linux: Bool = false
@Option(help: ArgumentHelp("Disk size in Gb")) @Option(help: ArgumentHelp("Disk size in Gb"))
var diskSize: UInt16 = 50 var diskSize: UInt16 = 50
func validate() throws { func validate() throws {
if fromIPSW == nil { if fromIPSW == nil && !linux {
throw ValidationError("Please specify a --from-ipsw option!") throw ValidationError("Please specify either a --from-ipsw or --linux option!")
} }
} }
@@ -25,10 +28,20 @@ struct Create: AsyncParsableCommand {
do { do {
let tmpVMDir = try VMDirectory.temporary() let tmpVMDir = try VMDirectory.temporary()
try await withTaskCancellationHandler(operation: { try await withTaskCancellationHandler(operation: {
if fromIPSW! == "latest" { if let fromIPSW = fromIPSW {
_ = try await VM(vmDir: tmpVMDir, ipswURL: nil, diskSizeGB: diskSize) if fromIPSW == "latest" {
} else { _ = try await VM(vmDir: tmpVMDir, ipswURL: nil, diskSizeGB: diskSize)
_ = try await VM(vmDir: tmpVMDir, ipswURL: URL(fileURLWithPath: fromIPSW!), diskSizeGB: diskSize) } else {
_ = try await VM(vmDir: tmpVMDir, ipswURL: URL(fileURLWithPath: fromIPSW), diskSizeGB: diskSize)
}
}
if linux {
if #available(macOS 13, *) {
_ = try await VM.linux(vmDir: tmpVMDir, diskSizeGB: diskSize)
} else {
throw UnsupportedOSError()
}
} }
try VMStorageLocal().move(name, from: tmpVMDir) try VMStorageLocal().move(name, from: tmpVMDir)
+11 -5
View File
@@ -16,14 +16,21 @@ struct IP: AsyncParsableCommand {
do { do {
let vmDir = try VMStorageLocal().open(name) let vmDir = try VMStorageLocal().open(name)
let vmConfig = try VMConfig.init(fromURL: vmDir.configURL) let vmConfig = try VMConfig.init(fromURL: vmDir.configURL)
let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!
guard let ip = try await IP.resolveIP(vmConfig, secondsToWait: wait) else { guard let ipViaDHCP = try await IP.resolveIP(vmMACAddress, secondsToWait: wait) else {
print("no IP address found, is your VM running?") print("no IP address found, is your VM running?")
Foundation.exit(1) Foundation.exit(1)
} }
print(ip) if let ipViaARP = try ARPCache.ResolveMACAddress(macAddress: vmMACAddress), ipViaARP != ipViaDHCP {
fputs("WARNING: DHCP lease and ARP cache entries for MAC address \(vmMACAddress) differ: "
+ "got \(ipViaDHCP) and \(ipViaARP) respectively, consider reporting this case to"
+ " https://github.com/cirruslabs/tart/issues/172\n", stderr)
}
print(ipViaDHCP)
Foundation.exit(0) Foundation.exit(0)
} catch { } catch {
@@ -33,12 +40,11 @@ struct IP: AsyncParsableCommand {
} }
} }
static public func resolveIP(_ config: VMConfig, secondsToWait: UInt16) async throws -> IPv4Address? { static public func resolveIP(_ vmMACAddress: MACAddress, secondsToWait: UInt16) async throws -> IPv4Address? {
let waitUntil = Calendar.current.date(byAdding: .second, value: Int(secondsToWait), to: Date.now)! let waitUntil = Calendar.current.date(byAdding: .second, value: Int(secondsToWait), to: Date.now)!
let vmMacAddress = MACAddress(fromString: config.macAddress.string)!
repeat { repeat {
if let ip = try Leases().resolveMACAddress(macAddress: vmMacAddress) { if let ip = try Leases().resolveMACAddress(macAddress: vmMACAddress) {
return ip return ip
} }
+1 -1
View File
@@ -10,7 +10,7 @@ struct List: AsyncParsableCommand {
print("Source\tName") print("Source\tName")
displayTable("local", try VMStorageLocal().list()) displayTable("local", try VMStorageLocal().list())
displayTable("oci", try VMStorageOCI().list()) displayTable("oci", try VMStorageOCI().list().map { (name, vmDir, _) in (name, vmDir) })
Foundation.exit(0) Foundation.exit(0)
} catch { } catch {
+5 -1
View File
@@ -14,6 +14,9 @@ struct Login: AsyncParsableCommand {
@Flag(help: "password-stdin") @Flag(help: "password-stdin")
var passwordStdin: Bool = false var passwordStdin: Bool = false
@Flag(help: "connect to the OCI registry via insecure HTTP protocol")
var insecure: Bool = false
func validate() throws { func validate() throws {
let usernameProvided = username != nil let usernameProvided = username != nil
let passwordProvided = passwordStdin let passwordProvided = passwordStdin
@@ -41,7 +44,8 @@ struct Login: AsyncParsableCommand {
]) ])
do { do {
let registry = try Registry(host: host, namespace: "", credentialsProvider: credentialsProvider) let registry = try Registry(host: host, namespace: "", insecure: insecure,
credentialsProvider: credentialsProvider)
try await registry.ping() try await registry.ping()
} catch { } catch {
print("invalid credentials: \(error)") print("invalid credentials: \(error)")
+108
View File
@@ -0,0 +1,108 @@
import ArgumentParser
import Dispatch
import SwiftUI
import SwiftDate
struct Prune: AsyncParsableCommand {
static var configuration = CommandConfiguration(abstract: "Prune OCI and IPSW caches")
@Option(help: ArgumentHelp("Remove cache entries last accessed more than n days ago",
discussion: "For example, --older-than=7 will remove entries that weren't accessed by Tart in the last 7 days.",
valueName: "n"))
var olderThan: UInt?
@Option(help: ArgumentHelp("Remove least recently used cache entries that do not fit the specified cache size budget n, expressed in gigabytes",
discussion: "For example, --cache-budget=50 will effectively shrink all caches to a total size of 50 gigabytes.",
valueName: "n"))
var cacheBudget: UInt?
@Flag(help: .hidden)
var gc: Bool = false
func validate() throws {
if olderThan == nil && cacheBudget == nil {
throw ValidationError("at least one criteria must be specified")
}
}
func run() async throws {
do {
if gc {
try VMStorageOCI().gc()
}
// Clean up cache entries based on last accessed date
if let olderThan = olderThan {
let olderThanInterval = Int(exactly: olderThan)!.days.timeInterval
let olderThanDate = Date().addingTimeInterval(olderThanInterval)
try Prune.pruneOlderThan(olderThanDate: olderThanDate)
}
// Clean up cache entries based on imposed cache size limit and entry's last accessed date
if let cacheBudget = cacheBudget {
try Prune.pruneCacheBudget(cacheBudgetBytes: UInt64(cacheBudget) * 1024 * 1024 * 1024)
}
Foundation.exit(0)
} catch {
print(error)
Foundation.exit(1)
}
}
static func pruneOlderThan(olderThanDate: Date) throws {
let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
let prunables: [Prunable] = try prunableStorages.flatMap { try $0.prunables() }
try prunables.filter { try $0.accessDate() <= olderThanDate }.forEach { try $0.delete() }
}
static func pruneCacheBudget(cacheBudgetBytes: UInt64) throws {
let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
let prunables: [Prunable] = try prunableStorages
.flatMap { try $0.prunables() }
.sorted { try $0.accessDate() < $1.accessDate() }
let cacheUsedBytes = try prunables.map { try $0.sizeBytes() }.reduce(0, +)
var cacheReclaimedBytes: Int = 0
var it = prunables.makeIterator()
while (cacheUsedBytes - cacheReclaimedBytes) > cacheBudgetBytes {
guard let prunable = it.next() else {
break
}
cacheReclaimedBytes -= try prunable.sizeBytes()
try prunable.delete()
}
}
static func pruneReclaim(reclaimBytes: UInt64) throws {
let prunableStorages: [PrunableStorage] = [VMStorageOCI(), try IPSWCache()]
let prunables: [Prunable] = try prunableStorages
.flatMap { try $0.prunables() }
.sorted { try $0.accessDate() < $1.accessDate() }
// Does it even make sense to start?
let cacheUsedBytes = try prunables.map { try $0.sizeBytes() }.reduce(0, +)
if cacheUsedBytes < reclaimBytes {
return
}
var cacheReclaimedBytes: Int = 0
var it = prunables.makeIterator()
while cacheReclaimedBytes <= reclaimBytes {
guard let prunable = it.next() else {
break
}
cacheReclaimedBytes -= try prunable.sizeBytes()
try prunable.delete()
}
}
}
+4 -1
View File
@@ -8,6 +8,9 @@ struct Pull: AsyncParsableCommand {
@Argument(help: "remote VM name") @Argument(help: "remote VM name")
var remoteName: String var remoteName: String
@Flag(help: "connect to the OCI registry via insecure HTTP protocol")
var insecure: Bool = false
func run() async throws { func run() async throws {
do { do {
// Be more liberal when accepting local image as argument, // Be more liberal when accepting local image as argument,
@@ -19,7 +22,7 @@ struct Pull: AsyncParsableCommand {
} }
let remoteName = try RemoteName(remoteName) let remoteName = try RemoteName(remoteName)
let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace) let registry = try Registry(host: remoteName.host, namespace: remoteName.namespace, insecure: insecure)
defaultLogger.appendNewLine("pulling \(remoteName)...") defaultLogger.appendNewLine("pulling \(remoteName)...")
+5 -1
View File
@@ -12,6 +12,9 @@ struct Push: AsyncParsableCommand {
@Argument(help: "remote VM name(s)") @Argument(help: "remote VM name(s)")
var remoteNames: [String] var remoteNames: [String]
@Flag(help: "connect to the OCI registry via insecure HTTP protocol")
var insecure: Bool = false
@Option(help: ArgumentHelp("chunk size in MB if registry supports chunked uploads", @Option(help: ArgumentHelp("chunk size in MB if registry supports chunked uploads",
discussion: """ discussion: """
By default monolithic method is used for uploading blobs to the registry but some registries support a more efficient chunked method. By default monolithic method is used for uploading blobs to the registry but some registries support a more efficient chunked method.
@@ -45,7 +48,8 @@ struct Push: AsyncParsableCommand {
// Push VM // Push VM
for (registryIdentifier, remoteNamesForRegistry) in registryGroups { for (registryIdentifier, remoteNamesForRegistry) in registryGroups {
let registry = try Registry(host: registryIdentifier.host, namespace: registryIdentifier.namespace) let registry = try Registry(host: registryIdentifier.host, namespace: registryIdentifier.namespace,
insecure: insecure)
defaultLogger.appendNewLine("pushing \(localName) to " defaultLogger.appendNewLine("pushing \(localName) to "
+ "\(registryIdentifier.host)/\(registryIdentifier.namespace)\(remoteNamesForRegistry.referenceNames())...") + "\(registryIdentifier.host)/\(registryIdentifier.namespace)\(remoteNamesForRegistry.referenceNames())...")
+20
View File
@@ -0,0 +1,20 @@
import Foundation
import Virtualization
class IPSWCache: PrunableStorage {
let baseURL: URL
init() throws {
baseURL = Config().tartCacheDir.appendingPathComponent("IPSWs", isDirectory: true)
try FileManager.default.createDirectory(at: baseURL, withIntermediateDirectories: true)
}
func locationFor(image: VZMacOSRestoreImage) -> URL {
baseURL.appendingPathComponent("\(image.buildVersion).ipsw", isDirectory: false)
}
func prunables() throws -> [Prunable] {
try FileManager.default.contentsOfDirectory(at: baseURL, includingPropertiesForKeys: nil)
.filter { $0.lastPathComponent.hasSuffix(".ipsw")}
}
}
@@ -0,0 +1,119 @@
import Foundation
import Network
import Virtualization
struct ARPCommandFailedError: Error, CustomStringConvertible {
var terminationReason: Process.TerminationReason
var terminationStatus: Int32
var description: String {
var reason: String
switch terminationReason {
case .exit:
reason = "exit code \(terminationStatus)"
case .uncaughtSignal:
reason = "uncaught signal"
default:
reason = "unknown reason"
}
return "arp command failed: \(reason)"
}
}
struct ARPCommandYieldedInvalidOutputError: Error, CustomStringConvertible {
var explanation: String
var description: String {
"arp command yielded invalid output: \(explanation)"
}
}
struct ARPCacheInternalError: Error, CustomStringConvertible {
var explanation: String
var description: String {
"ARPCache internal error: \(explanation)"
}
}
struct ARPCache {
static func ResolveMACAddress(macAddress: MACAddress, bridgeOnly: Bool = true) throws -> IPv4Address? {
let process = Process.init()
process.executableURL = URL.init(fileURLWithPath: "/usr/sbin/arp")
process.arguments = ["-an"]
let pipe = Pipe()
process.standardOutput = pipe
process.standardError = pipe
process.standardInput = FileHandle.nullDevice
try process.run()
process.waitUntilExit()
if !(process.terminationReason == .exit && process.terminationStatus == 0) {
throw ARPCommandFailedError(
terminationReason: process.terminationReason,
terminationStatus: process.terminationStatus)
}
guard let rawLines = try pipe.fileHandleForReading.readToEnd() else {
throw ARPCommandYieldedInvalidOutputError(explanation: "empty output")
}
let lines = String(decoding: rawLines, as: UTF8.self)
.trimmingCharacters(in: .whitespacesAndNewlines)
.components(separatedBy: "\n")
// Based on https://opensource.apple.com/source/network_cmds/network_cmds-606.40.2/arp.tproj/arp.c.auto.html
let regex = try NSRegularExpression(pattern: #"^.* \((?<ip>.*)\) at (?<mac>.*) on (?<interface>.*) .*$"#)
for line in lines {
let nsLineRange = NSRange(line.startIndex..<line.endIndex, in: line)
guard let match = regex.firstMatch(in: line, range: nsLineRange) else {
throw ARPCommandYieldedInvalidOutputError(explanation: "unparseable entry \"\(line)\"")
}
let rawIP = try match.getCaptureGroup(name: "ip", for: line)
guard let ip = IPv4Address(rawIP) else {
throw ARPCommandYieldedInvalidOutputError(explanation: "failed to parse IPv4 address \(rawIP)")
}
let rawMAC = try match.getCaptureGroup(name: "mac", for: line)
if rawMAC == "(incomplete)" {
continue
}
guard let mac = MACAddress(fromString: rawMAC) else {
throw ARPCommandYieldedInvalidOutputError(explanation: "failed to parse MAC address \(rawMAC)")
}
let interface = try match.getCaptureGroup(name: "interface", for: line)
if bridgeOnly && !interface.starts(with: "bridge") {
continue
}
if macAddress == mac {
return ip
}
}
return nil
}
}
extension NSTextCheckingResult {
func getCaptureGroup(name: String, for string: String) throws -> String {
let nsRange = self.range(withName: name)
if nsRange.location == NSNotFound {
throw ARPCacheInternalError(explanation: "attempted to retrieve non-existent named capture group \(name)")
}
guard let range = Range.init(nsRange, in: string) else {
throw ARPCacheInternalError(explanation: "failed to convert NSRange to Range")
}
return String(string[range])
}
}
@@ -16,6 +16,6 @@ struct MACAddress: Equatable, Hashable, CustomStringConvertible {
} }
var description: String { var description: String {
return String(format: "%02x:%02x:%02x:%02x:%02x:%02x", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]) String(format: "%02x:%02x:%02x:%02x:%02x:%02x", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5])
} }
} }
+2 -2
View File
@@ -46,8 +46,8 @@ struct OCIManifest: Codable, Equatable {
} }
struct OCIConfig: Codable { struct OCIConfig: Codable {
var architecture: String = "arm64" var architecture: Architecture = .arm64
var os: String = "darwin" var os: OS = .darwin
func toJSON() throws -> Data { func toJSON() throws -> Data {
try Config.jsonEncoder().encode(self) try Config.jsonEncoder().encode(self)
+23 -5
View File
@@ -104,13 +104,11 @@ class Registry {
convenience init( convenience init(
host: String, host: String,
namespace: String, namespace: String,
insecure: Bool = false,
credentialsProvider: CredentialsProvider = KeychainCredentialsProvider() credentialsProvider: CredentialsProvider = KeychainCredentialsProvider()
) throws { ) throws {
var baseURLComponents = URLComponents() let proto = insecure ? "http" : "https"
let baseURLComponents = URLComponents(string: proto + "://" + host + "/v2/")!
baseURLComponents.scheme = "https"
baseURLComponents.host = host
baseURLComponents.path = "/v2/"
try self.init(urlComponents: baseURLComponents, namespace: namespace, credentialsProvider: credentialsProvider) try self.init(urlComponents: baseURLComponents, namespace: namespace, credentialsProvider: credentialsProvider)
} }
@@ -177,7 +175,27 @@ class Registry {
var uploadLocation = try uploadLocationFromResponse(postResponse) var uploadLocation = try uploadLocationFromResponse(postResponse)
let digest = Digest.hash(fromData) let digest = Digest.hash(fromData)
if chunkSizeMb == 0 {
// monolithic upload
let response = try await rawRequest(
.PUT,
uploadLocation,
headers: [
"Content-Type": "application/octet-stream",
],
parameters: ["digest": digest],
body: fromData
)
if response.status != .created {
let body = try await response.body.readTextResponse()
throw RegistryError.UnexpectedHTTPStatusCode(when: "pushing blob (PUT) to \(uploadLocation)",
code: response.status.code, details: body ?? "")
}
return digest
}
// chunked upload
var uploadedBytes = 0 var uploadedBytes = 0
let chunks = fromData.chunks(ofCount: chunkSizeMb == 0 ? fromData.count : chunkSizeMb * 1_000_000) let chunks = fromData.chunks(ofCount: chunkSizeMb == 0 ? fromData.count : chunkSizeMb * 1_000_000)
for (index, chunk) in chunks.enumerated() { for (index, chunk) in chunks.enumerated() {
+14
View File
@@ -0,0 +1,14 @@
import Foundation
enum Architecture: String, Codable {
case arm64
case amd64
}
func CurrentArchitecture() -> Architecture {
#if arch(arm64)
return .arm64
#elseif arch(x86_64)
return .amd64
#endif
}
+87
View File
@@ -0,0 +1,87 @@
import Virtualization
struct Darwin: Platform {
var ecid: VZMacMachineIdentifier
var hardwareModel: VZMacHardwareModel
init(ecid: VZMacMachineIdentifier, hardwareModel: VZMacHardwareModel) {
self.ecid = ecid
self.hardwareModel = hardwareModel
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
let encodedECID = try container.decode(String.self, forKey: .ecid)
guard let data = Data.init(base64Encoded: encodedECID) else {
throw DecodingError.dataCorruptedError(forKey: .ecid,
in: container,
debugDescription: "failed to initialize Data using the provided value")
}
guard let ecid = VZMacMachineIdentifier.init(dataRepresentation: data) else {
throw DecodingError.dataCorruptedError(forKey: .ecid,
in: container,
debugDescription: "failed to initialize VZMacMachineIdentifier using the provided value")
}
self.ecid = ecid
let encodedHardwareModel = try container.decode(String.self, forKey: .hardwareModel)
guard let data = Data.init(base64Encoded: encodedHardwareModel) else {
throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
}
guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
}
self.hardwareModel = hardwareModel
}
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(ecid.dataRepresentation.base64EncodedString(), forKey: .ecid)
try container.encode(hardwareModel.dataRepresentation.base64EncodedString(), forKey: .hardwareModel)
}
func os() -> OS {
.darwin
}
func bootLoader(nvramURL: URL) throws -> VZBootLoader {
VZMacOSBootLoader()
}
func platform(nvramURL: URL) -> VZPlatformConfiguration {
let result = VZMacPlatformConfiguration()
result.machineIdentifier = ecid
result.auxiliaryStorage = VZMacAuxiliaryStorage(contentsOf: nvramURL)
result.hardwareModel = hardwareModel
return result
}
func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
let result = VZMacGraphicsDeviceConfiguration()
if let hostMainScreen = NSScreen.main {
let vmScreenSize = NSSize(width: vmConfig.display.width, height: vmConfig.display.height)
result.displays = [
VZMacGraphicsDisplayConfiguration(for: hostMainScreen, sizeInPoints: vmScreenSize)
]
return result
}
result.displays = [
VZMacGraphicsDisplayConfiguration(
widthInPixels: vmConfig.display.width,
heightInPixels: vmConfig.display.height,
// A reasonable guess according to Apple's documentation[1]
// [1]: https://developer.apple.com/documentation/coregraphics/1456599-cgdisplayscreensize
pixelsPerInch: 72
)
]
return result
}
}
+33
View File
@@ -0,0 +1,33 @@
import Virtualization
@available(macOS 13, *)
struct Linux: Platform {
func os() -> OS {
.linux
}
func bootLoader(nvramURL: URL) throws -> VZBootLoader {
let result = VZEFIBootLoader()
result.variableStore = VZEFIVariableStore(url: nvramURL)
return result
}
func platform(nvramURL: URL) -> VZPlatformConfiguration {
VZGenericPlatformConfiguration()
}
func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration {
let result = VZVirtioGraphicsDeviceConfiguration()
result.scanouts = [
VZVirtioGraphicsScanoutConfiguration(
widthInPixels: vmConfig.display.width,
heightInPixels: vmConfig.display.height
)
]
return result
}
}
+6
View File
@@ -0,0 +1,6 @@
import Virtualization
enum OS: String, Codable {
case darwin
case linux
}
+8
View File
@@ -0,0 +1,8 @@
import Virtualization
protocol Platform: Codable {
func os() -> OS
func bootLoader(nvramURL: URL) throws -> VZBootLoader
func platform(nvramURL: URL) -> VZPlatformConfiguration
func graphicsDevice(vmConfig: VMConfig) -> VZGraphicsDeviceConfiguration
}
+11
View File
@@ -0,0 +1,11 @@
import Foundation
protocol PrunableStorage {
func prunables() throws -> [Prunable]
}
protocol Prunable {
func delete() throws
func accessDate() throws -> Date
func sizeBytes() throws -> Int
}
+1
View File
@@ -16,6 +16,7 @@ struct Root: AsyncParsableCommand {
IP.self, IP.self,
Pull.self, Pull.self,
Push.self, Push.self,
Prune.self,
Delete.self, Delete.self,
]) ])
+25
View File
@@ -0,0 +1,25 @@
import Foundation
extension URL {
func accessDate() throws -> Date {
let attrs = try resourceValues(forKeys: [.contentAccessDateKey])
return attrs.contentAccessDate!
}
func updateAccessDate(_ accessDate: Date = Date()) throws {
let attrs = try resourceValues(forKeys: [.contentAccessDateKey])
let modificationDate = attrs.contentAccessDate!
let times = [accessDate.asTimeval(), modificationDate.asTimeval()]
let ret = utimes(path, times)
if ret != 0 {
throw RuntimeError("utimes(2) failed: \(ret.explanation())")
}
}
}
extension Date {
func asTimeval() -> timeval {
timeval(tv_sec: Int(timeIntervalSince1970), tv_usec: 0)
}
}
+11
View File
@@ -0,0 +1,11 @@
import Foundation
extension URL: Prunable {
func delete() throws {
try FileManager.default.removeItem(at: self)
}
func sizeBytes() throws -> Int {
try resourceValues(forKeys: [.totalFileAllocatedSizeKey]).totalFileAllocatedSize!
}
}
+1
View File
@@ -5,6 +5,7 @@ import Dynamic
// Kudos to @saagarjha's VirtualApple for finding about _VZVirtualMachineStartOptions // Kudos to @saagarjha's VirtualApple for finding about _VZVirtualMachineStartOptions
extension VZVirtualMachine { extension VZVirtualMachine {
@available(macOS 12, *)
func start(_ recovery: Bool) async throws { func start(_ recovery: Bool) async throws {
if !recovery { if !recovery {
// just use the regular API // just use the regular API
+51 -42
View File
@@ -10,6 +10,12 @@ struct NoMainScreenFoundError: Error {
struct DownloadFailed: Error { struct DownloadFailed: Error {
} }
struct UnsupportedOSError: Error {
}
struct UnsupportedArchitectureError: Error {
}
class VM: NSObject, VZVirtualMachineDelegate, ObservableObject { class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
// Virtualization.Framework's virtual machine // Virtualization.Framework's virtual machine
@Published var virtualMachine: VZVirtualMachine @Published var virtualMachine: VZVirtualMachine
@@ -29,17 +35,20 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
withSoftnet: Bool = false, withSoftnet: Bool = false,
additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = [] additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] = []
) throws { ) throws {
let auxStorage = VZMacAuxiliaryStorage(contentsOf: vmDir.nvramURL)
name = vmDir.name name = vmDir.name
config = try VMConfig.init(fromURL: vmDir.configURL) config = try VMConfig.init(fromURL: vmDir.configURL)
if config.arch != CurrentArchitecture() {
throw UnsupportedArchitectureError()
}
// Initialize the virtual machine and its configuration // Initialize the virtual machine and its configuration
if withSoftnet { if withSoftnet {
softnet = try Softnet(vmMACAddress: config.macAddress.string) softnet = try Softnet(vmMACAddress: config.macAddress.string)
} }
let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, auxStorage: auxStorage, vmConfig: config, let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL,
nvramURL: vmDir.nvramURL, vmConfig: config,
softnet: softnet, additionalDiskAttachments: additionalDiskAttachments) softnet: softnet, additionalDiskAttachments: additionalDiskAttachments)
virtualMachine = VZVirtualMachine(configuration: configuration) virtualMachine = VZVirtualMachine(configuration: configuration)
@@ -55,14 +64,11 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
} }
} }
let expectedIPSWLocation = try IPSWCache().locationFor(image: image)
let ipswCacheFolder = Config().tartCacheDir.appendingPathComponent("IPSWs", isDirectory: true)
try FileManager.default.createDirectory(at: ipswCacheFolder, withIntermediateDirectories: true)
let expectedIPSWLocation = ipswCacheFolder.appendingPathComponent("\(image.buildVersion).ipsw", isDirectory: false)
if FileManager.default.fileExists(atPath: expectedIPSWLocation.path) { if FileManager.default.fileExists(atPath: expectedIPSWLocation.path) {
defaultLogger.appendNewLine("Using cached *.ipsw file...") defaultLogger.appendNewLine("Using cached *.ipsw file...")
try expectedIPSWLocation.updateAccessDate()
return expectedIPSWLocation return expectedIPSWLocation
} }
@@ -119,7 +125,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
} }
// Create NVRAM // Create NVRAM
let auxStorage = try VZMacAuxiliaryStorage(creatingStorageAt: vmDir.nvramURL, hardwareModel: requirements.hardwareModel) _ = try VZMacAuxiliaryStorage(creatingStorageAt: vmDir.nvramURL, hardwareModel: requirements.hardwareModel)
// Create disk // Create disk
try vmDir.resizeDisk(diskSizeGB) try vmDir.resizeDisk(diskSizeGB)
@@ -127,7 +133,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
name = vmDir.name name = vmDir.name
// Create config // Create config
config = VMConfig( config = VMConfig(
hardwareModel: requirements.hardwareModel, platform: Darwin(ecid: VZMacMachineIdentifier(), hardwareModel: requirements.hardwareModel),
cpuCountMin: requirements.minimumSupportedCPUCount, cpuCountMin: requirements.minimumSupportedCPUCount,
memorySizeMin: requirements.minimumSupportedMemorySize memorySizeMin: requirements.minimumSupportedMemorySize
) )
@@ -140,8 +146,9 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
softnet = try Softnet(vmMACAddress: config.macAddress.string) softnet = try Softnet(vmMACAddress: config.macAddress.string)
} }
let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, auxStorage: auxStorage, vmConfig: config, let configuration = try Self.craftConfiguration(diskURL: vmDir.diskURL, nvramURL: vmDir.nvramURL,
softnet: softnet, additionalDiskAttachments: additionalDiskAttachments) vmConfig: config, softnet: softnet,
additionalDiskAttachments: additionalDiskAttachments)
virtualMachine = VZVirtualMachine(configuration: configuration) virtualMachine = VZVirtualMachine(configuration: configuration)
super.init() super.init()
@@ -162,12 +169,39 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
} }
} }
@available(macOS 13, *)
static func linux(vmDir: VMDirectory, diskSizeGB: UInt16) async throws -> VM {
// Create NVRAM
_ = try VZEFIVariableStore(creatingVariableStoreAt: vmDir.nvramURL)
// Create disk
try vmDir.resizeDisk(diskSizeGB)
// Create config
let config = VMConfig(platform: Linux(), cpuCountMin: 4, memorySizeMin: 4096 * 1024 * 1024)
try config.save(toURL: vmDir.configURL)
return try VM(vmDir: vmDir)
}
func run(_ recovery: Bool) async throws { func run(_ recovery: Bool) async throws {
if let softnet = softnet { if let softnet = softnet {
try softnet.run() try softnet.run()
} }
try await virtualMachine.start(recovery) DispatchQueue.main.sync {
Task {
if #available(macOS 13, *) {
// new API introduced in Ventura
let startOptions = VZMacOSVirtualMachineStartOptions()
startOptions.startUpFromMacOSRecovery = recovery
try await virtualMachine.start(options: startOptions)
} else {
// use method that also available on Monterey
try await virtualMachine.start(recovery)
}
}
}
await withTaskCancellationHandler(operation: { await withTaskCancellationHandler(operation: {
sema.wait() sema.wait()
@@ -190,7 +224,7 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
static func craftConfiguration( static func craftConfiguration(
diskURL: URL, diskURL: URL,
auxStorage: VZMacAuxiliaryStorage, nvramURL: URL,
vmConfig: VMConfig, vmConfig: VMConfig,
softnet: Softnet? = nil, softnet: Softnet? = nil,
additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment] additionalDiskAttachments: [VZDiskImageStorageDeviceAttachment]
@@ -198,42 +232,17 @@ class VM: NSObject, VZVirtualMachineDelegate, ObservableObject {
let configuration = VZVirtualMachineConfiguration() let configuration = VZVirtualMachineConfiguration()
// Boot loader // Boot loader
configuration.bootLoader = VZMacOSBootLoader() configuration.bootLoader = try vmConfig.platform.bootLoader(nvramURL: nvramURL)
// CPU and memory // CPU and memory
configuration.cpuCount = vmConfig.cpuCount configuration.cpuCount = vmConfig.cpuCount
configuration.memorySize = vmConfig.memorySize configuration.memorySize = vmConfig.memorySize
// Platform // Platform
let platform = VZMacPlatformConfiguration() configuration.platform = vmConfig.platform.platform(nvramURL: nvramURL)
platform.machineIdentifier = vmConfig.ecid
platform.auxiliaryStorage = auxStorage
platform.hardwareModel = vmConfig.hardwareModel
configuration.platform = platform
// Display // Display
let graphicsDeviceConfiguration = VZMacGraphicsDeviceConfiguration() configuration.graphicsDevices = [vmConfig.platform.graphicsDevice(vmConfig: vmConfig)]
if let hostMainScreen = NSScreen.main {
let vmScreenSize = NSSize(
width: vmConfig.display.width,
height: vmConfig.display.height
)
graphicsDeviceConfiguration.displays = [
VZMacGraphicsDisplayConfiguration(for: hostMainScreen, sizeInPoints: vmScreenSize)
]
} else {
graphicsDeviceConfiguration.displays = [
VZMacGraphicsDisplayConfiguration(
widthInPixels: vmConfig.display.width,
heightInPixels: vmConfig.display.height,
// Reasonable guess like https://developer.apple.com/documentation/coregraphics/1456599-cgdisplayscreensize
pixelsPerInch: 72
)
]
}
configuration.graphicsDevices = [graphicsDeviceConfiguration]
// Audio // Audio
let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration() let soundDeviceConfiguration = VZVirtioSoundDeviceConfiguration()
+30 -36
View File
@@ -16,14 +16,18 @@ class LessThanMinimalResourcesError: NSObject, LocalizedError {
enum CodingKeys: String, CodingKey { enum CodingKeys: String, CodingKey {
case version case version
case ecid case os
case hardwareModel case arch
case cpuCountMin case cpuCountMin
case cpuCount case cpuCount
case memorySizeMin case memorySizeMin
case memorySize case memorySize
case macAddress case macAddress
case display case display
// macOS-specific keys
case ecid
case hardwareModel
} }
struct VMDisplayConfig: Codable { struct VMDisplayConfig: Codable {
@@ -33,25 +37,25 @@ struct VMDisplayConfig: Codable {
struct VMConfig: Codable { struct VMConfig: Codable {
var version: Int = 1 var version: Int = 1
var ecid: VZMacMachineIdentifier var os: OS
var hardwareModel: VZMacHardwareModel var arch: Architecture
var platform: Platform
var cpuCountMin: Int var cpuCountMin: Int
private(set) var cpuCount: Int private(set) var cpuCount: Int
var memorySizeMin: UInt64 var memorySizeMin: UInt64
private(set) var memorySize: UInt64 private(set) var memorySize: UInt64
var macAddress: VZMACAddress var macAddress: VZMACAddress
var display: VMDisplayConfig = VMDisplayConfig() var display: VMDisplayConfig = VMDisplayConfig()
init( init(
ecid: VZMacMachineIdentifier = VZMacMachineIdentifier(), platform: Platform,
hardwareModel: VZMacHardwareModel, cpuCountMin: Int,
cpuCountMin: Int, memorySizeMin: UInt64,
memorySizeMin: UInt64, macAddress: VZMACAddress = VZMACAddress.randomLocallyAdministered()
macAddress: VZMACAddress = VZMACAddress.randomLocallyAdministered()
) { ) {
self.ecid = ecid self.os = platform.os()
self.hardwareModel = hardwareModel self.arch = CurrentArchitecture()
self.platform = platform
self.macAddress = macAddress self.macAddress = macAddress
self.cpuCountMin = cpuCountMin self.cpuCountMin = cpuCountMin
self.memorySizeMin = memorySizeMin self.memorySizeMin = memorySizeMin
@@ -81,29 +85,18 @@ struct VMConfig: Codable {
let container = try decoder.container(keyedBy: CodingKeys.self) let container = try decoder.container(keyedBy: CodingKeys.self)
version = try container.decode(Int.self, forKey: .version) version = try container.decode(Int.self, forKey: .version)
os = try container.decodeIfPresent(OS.self, forKey: .os) ?? .darwin
let encodedECID = try container.decode(String.self, forKey: .ecid) arch = try container.decodeIfPresent(Architecture.self, forKey: .arch) ?? .arm64
guard let data = Data.init(base64Encoded: encodedECID) else { switch os {
throw DecodingError.dataCorruptedError(forKey: .ecid, case .darwin:
in: container, platform = try Darwin(from: decoder)
debugDescription: "failed to initialize Data using the provided value") case .linux:
if #available(macOS 13, *) {
platform = try Linux(from: decoder)
} else {
throw UnsupportedOSError()
}
} }
guard let ecid = VZMacMachineIdentifier.init(dataRepresentation: data) else {
throw DecodingError.dataCorruptedError(forKey: .ecid,
in: container,
debugDescription: "failed to initialize VZMacMachineIdentifier using the provided value")
}
self.ecid = ecid
let encodedHardwareModel = try container.decode(String.self, forKey: .hardwareModel)
guard let data = Data.init(base64Encoded: encodedHardwareModel) else {
throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
}
guard let hardwareModel = VZMacHardwareModel.init(dataRepresentation: data) else {
throw DecodingError.dataCorruptedError(forKey: .hardwareModel, in: container, debugDescription: "")
}
self.hardwareModel = hardwareModel
cpuCountMin = try container.decode(Int.self, forKey: .cpuCountMin) cpuCountMin = try container.decode(Int.self, forKey: .cpuCountMin)
cpuCount = try container.decode(Int.self, forKey: .cpuCount) cpuCount = try container.decode(Int.self, forKey: .cpuCount)
memorySizeMin = try container.decode(UInt64.self, forKey: .memorySizeMin) memorySizeMin = try container.decode(UInt64.self, forKey: .memorySizeMin)
@@ -125,8 +118,9 @@ struct VMConfig: Codable {
var container = encoder.container(keyedBy: CodingKeys.self) var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(version, forKey: .version) try container.encode(version, forKey: .version)
try container.encode(ecid.dataRepresentation.base64EncodedString(), forKey: .ecid) try container.encode(os, forKey: .os)
try container.encode(hardwareModel.dataRepresentation.base64EncodedString(), forKey: .hardwareModel) try container.encode(arch, forKey: .arch)
try platform.encode(to: encoder)
try container.encode(cpuCountMin, forKey: .cpuCountMin) try container.encode(cpuCountMin, forKey: .cpuCountMin)
try container.encode(cpuCount, forKey: .cpuCount) try container.encode(cpuCount, forKey: .cpuCount)
try container.encode(memorySizeMin, forKey: .memorySizeMin) try container.encode(memorySizeMin, forKey: .memorySizeMin)
+10 -8
View File
@@ -117,13 +117,15 @@ extension VMDirectory {
// Read VM's compressed disk as chunks // Read VM's compressed disk as chunks
// and sequentially upload them as blobs // and sequentially upload them as blobs
let disk = try FileHandle(forReadingFrom: diskURL) let mappedDisk = try Data(contentsOf: diskURL, options: [.alwaysMapped])
var diskReadBytes: UInt64 = 0 let mappedDiskSize = mappedDisk.count
let compressingFilter = try InputFilter<Data>(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { _ in var mappedDiskReadOffset = 0
let data = try disk.read(upToCount: Self.bufferSizeBytes) let compressingFilter = try InputFilter(.compress, using: .lz4, bufferCapacity: Self.bufferSizeBytes) { (length: Int) -> Data? in
diskReadBytes += UInt64(data?.count ?? 0) let bytesRead = min(length, mappedDiskSize - mappedDiskReadOffset)
let data = mappedDisk.subdata(in: mappedDiskReadOffset ..< mappedDiskReadOffset + bytesRead)
mappedDiskReadOffset += bytesRead
progress.completedUnitCount += Int64(data?.count ?? 0) progress.completedUnitCount = Int64(mappedDiskReadOffset)
return data return data
} }
@@ -140,12 +142,12 @@ extension VMDirectory {
layers.append(OCIManifestLayer(mediaType: Self.nvramMediaType, size: nvram.count, digest: nvramDigest)) layers.append(OCIManifestLayer(mediaType: Self.nvramMediaType, size: nvram.count, digest: nvramDigest))
// Craft a stub OCI config for Docker Hub compatibility // Craft a stub OCI config for Docker Hub compatibility
let ociConfigJSON = try OCIConfig().toJSON() let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os).toJSON()
let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb) let ociConfigDigest = try await registry.pushBlob(fromData: ociConfigJSON, chunkSizeMb: chunkSizeMb)
let manifest = OCIManifest( let manifest = OCIManifest(
config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest), config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest),
layers: layers, layers: layers,
uncompressedDiskSize: diskReadBytes uncompressedDiskSize: UInt64(mappedDiskReadOffset)
) )
// Manifest // Manifest
+25 -1
View File
@@ -7,7 +7,7 @@ struct UninitializedVMDirectoryError: Error {
struct AlreadyInitializedVMDirectoryError: Error { struct AlreadyInitializedVMDirectoryError: Error {
} }
struct VMDirectory { struct VMDirectory: Prunable {
var baseURL: URL var baseURL: URL
var configURL: URL { var configURL: URL {
@@ -20,6 +20,10 @@ struct VMDirectory {
baseURL.appendingPathComponent("nvram.bin") baseURL.appendingPathComponent("nvram.bin")
} }
var explicitlyPulledMark: URL {
baseURL.appendingPathComponent(".explicitly-pulled")
}
var name: String { var name: String {
baseURL.lastPathComponent baseURL.lastPathComponent
} }
@@ -77,4 +81,24 @@ struct VMDirectory {
try diskFileHandle.truncate(atOffset: UInt64(sizeGB) * 1000 * 1000 * 1000) try diskFileHandle.truncate(atOffset: UInt64(sizeGB) * 1000 * 1000 * 1000)
try diskFileHandle.close() try diskFileHandle.close()
} }
func delete() throws {
try FileManager.default.removeItem(at: baseURL)
}
func accessDate() throws -> Date {
try baseURL.accessDate()
}
func sizeBytes() throws -> Int {
try configURL.sizeBytes() + diskURL.sizeBytes() + nvramURL.sizeBytes()
}
func markExplicitlyPulled() {
FileManager.default.createFile(atPath: explicitlyPulledMark.path, contents: nil)
}
func isExplicitlyPulled() -> Bool {
FileManager.default.fileExists(atPath: explicitlyPulledMark.path)
}
} }
+73 -9
View File
@@ -1,6 +1,6 @@
import Foundation import Foundation
class VMStorageOCI { class VMStorageOCI: PrunableStorage {
let baseURL = Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true) let baseURL = Config().tartCacheDir.appendingPathComponent("OCIs", isDirectory: true)
private func vmURL(_ name: RemoteName) -> URL { private func vmURL(_ name: RemoteName) -> URL {
@@ -16,6 +16,8 @@ class VMStorageOCI {
try vmDir.validate() try vmDir.validate()
try vmDir.baseURL.updateAccessDate()
return vmDir return vmDir
} }
@@ -40,10 +42,48 @@ class VMStorageOCI {
func delete(_ name: RemoteName) throws { func delete(_ name: RemoteName) throws {
try FileManager.default.removeItem(at: vmURL(name)) try FileManager.default.removeItem(at: vmURL(name))
try gc()
} }
func list() throws -> [(String, VMDirectory)] { func gc() throws {
var result: [(String, VMDirectory)] = Array() var refCounts = Dictionary<URL, UInt>()
guard let enumerator = FileManager.default.enumerator(at: baseURL,
includingPropertiesForKeys: [.isSymbolicLinkKey]) else {
return
}
for case let foundURL as URL in enumerator {
let isSymlink = try foundURL.resourceValues(forKeys: [.isSymbolicLinkKey]).isSymbolicLink!
// Perform garbage collection for tag-based images
// with broken outgoing references
if isSymlink && foundURL == foundURL.resolvingSymlinksInPath() {
try FileManager.default.removeItem(at: foundURL)
continue
}
let vmDir = VMDirectory(baseURL: foundURL.resolvingSymlinksInPath())
if !vmDir.initialized {
continue
}
refCounts[vmDir.baseURL] = (refCounts[vmDir.baseURL] ?? 0) + (isSymlink ? 1 : 0)
}
// Perform garbage collection for digest-based images
// with no incoming references
for (baseURL, incRefCount) in refCounts {
let vmDir = VMDirectory(baseURL: baseURL)
if !vmDir.isExplicitlyPulled() && incRefCount == 0 {
try FileManager.default.removeItem(at: baseURL)
}
}
}
func list() throws -> [(String, VMDirectory, Bool)] {
var result: [(String, VMDirectory, Bool)] = Array()
guard let enumerator = FileManager.default.enumerator(at: baseURL, guard let enumerator = FileManager.default.enumerator(at: baseURL,
includingPropertiesForKeys: [.isSymbolicLinkKey], options: [.producesRelativePathURLs]) else { includingPropertiesForKeys: [.isSymbolicLinkKey], options: [.producesRelativePathURLs]) else {
@@ -60,28 +100,46 @@ class VMStorageOCI {
let parts = [foundURL.deletingLastPathComponent().relativePath, foundURL.lastPathComponent] let parts = [foundURL.deletingLastPathComponent().relativePath, foundURL.lastPathComponent]
var name: String var name: String
if try foundURL.resourceValues(forKeys: [.isSymbolicLinkKey]).isSymbolicLink! { let isSymlink = try foundURL.resourceValues(forKeys: [.isSymbolicLinkKey]).isSymbolicLink!
if isSymlink {
name = parts.joined(separator: ":") name = parts.joined(separator: ":")
} else { } else {
name = parts.joined(separator: "@") name = parts.joined(separator: "@")
} }
result.append((name, vmDir)) result.append((name, vmDir, isSymlink))
} }
return result return result
} }
func prunables() throws -> [Prunable] {
try list().filter { (_, _, isSymlink) in !isSymlink }.map { (_, vmDir, _) in vmDir }
}
func pull(_ name: RemoteName, registry: Registry) async throws { func pull(_ name: RemoteName, registry: Registry) async throws {
defaultLogger.appendNewLine("pulling manifest...") defaultLogger.appendNewLine("pulling manifest...")
let (manifest, _) = try await registry.pullManifest(reference: name.reference.value) let (manifest, manifestData) = try await registry.pullManifest(reference: name.reference.value)
var digestName = RemoteName(host: name.host, namespace: name.namespace, let digestName = RemoteName(host: name.host, namespace: name.namespace,
reference: Reference(digest: try manifest.digest())) reference: Reference(digest: Digest.hash(manifestData)))
if !exists(digestName) { if !exists(digestName) {
let tmpVMDir = try VMDirectory.temporary() let tmpVMDir = try VMDirectory.temporary()
// Try to reclaim some cache space if we know the VM size in advance
if let uncompressedDiskSize = manifest.uncompressedDiskSize() {
let requiredCapacityBytes = UInt64(uncompressedDiskSize + 128 * 1024 * 1024)
let attrs = try tmpVMDir.baseURL.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey])
let availableCapacityBytes = UInt64(attrs.volumeAvailableCapacityForImportantUsage!)
if availableCapacityBytes < requiredCapacityBytes {
try Prune.pruneReclaim(reclaimBytes: requiredCapacityBytes - availableCapacityBytes)
}
}
try await withTaskCancellationHandler(operation: { try await withTaskCancellationHandler(operation: {
try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest) try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest)
try move(digestName, from: tmpVMDir) try move(digestName, from: tmpVMDir)
@@ -93,8 +151,12 @@ class VMStorageOCI {
} }
if name != digestName { if name != digestName {
// Overwrite the old symbolic link // Create new or overwrite the old symbolic link
try link(from: digestName, to: name) try link(from: digestName, to: name)
} else {
// Ensure that images pulled by content digest
// are excluded from garbage collection
VMDirectory(baseURL: vmURL(name)).markExplicitlyPulled()
} }
} }
@@ -104,6 +166,8 @@ class VMStorageOCI {
} }
try FileManager.default.createSymbolicLink(at: vmURL(to), withDestinationURL: vmURL(from)) try FileManager.default.createSymbolicLink(at: vmURL(to), withDestinationURL: vmURL(from))
try gc()
} }
} }
+2 -1
View File
@@ -10,7 +10,8 @@ class ScreenSharingVNC: VNC {
} }
func waitForURL() async throws -> URL { func waitForURL() async throws -> URL {
let ip = try await IP.resolveIP(vmConfig, secondsToWait: 60) let vmMACAddress = MACAddress(fromString: vmConfig.macAddress.string)!
let ip = try await IP.resolveIP(vmMACAddress, secondsToWait: 60)
if let ip = ip { if let ip = ip {
return URL(string: "vnc://\(ip)")! return URL(string: "vnc://\(ip)")!
+22
View File
@@ -0,0 +1,22 @@
import XCTest
@testable import tart
final class URLAccessDateTests: XCTestCase {
func testGetAndSetAccessTime() throws {
// Create a temporary file
let tmpDir = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true)
var tmpFile = tmpDir.appendingPathComponent(UUID().uuidString)
FileManager.default.createFile(atPath: tmpFile.path, contents: nil)
// Ensure it's access date is different than our desired access date
let arbitraryDate = Date.init(year: 2008, month: 09, day: 28, hour: 23, minute: 15)
XCTAssertNotEqual(arbitraryDate, try tmpFile.accessDate())
// Set our desired access date for a file
try tmpFile.updateAccessDate(arbitraryDate)
// Ensure the access date has changed to our value
tmpFile.removeCachedResourceValue(forKey: .contentAccessDateKey)
XCTAssertEqual(arbitraryDate, try tmpFile.accessDate())
}
}