From 1d3aa5ac81fab4277a3cd3db3c1de0d7be3421ce Mon Sep 17 00:00:00 2001 From: Nikolay Edigaryev Date: Thu, 6 Apr 2023 18:20:04 +0400 Subject: [PATCH] tart push: allow pushing OCI VMs from the cache too (#465) * tart push: allow pushing OCI VMs from the cache too * Check for RemoteName earlier * Refactored pushing of OCI images under new tag (#466) * Refactored pushing of OCI images under new tag * Fixed compilation --------- Co-authored-by: Fedor Korotkov --- Sources/tart/Commands/Push.swift | 55 ++++++++++++++++++++++++------ Sources/tart/VMDirectory.swift | 4 +-- Sources/tart/VMStorageHelper.swift | 3 ++ Sources/tart/VMStorageLocal.swift | 2 +- Sources/tart/VMStorageOCI.swift | 12 ++++++- 5 files changed, 61 insertions(+), 15 deletions(-) diff --git a/Sources/tart/Commands/Push.swift b/Sources/tart/Commands/Push.swift index 8a6571b..622e7e4 100644 --- a/Sources/tart/Commands/Push.swift +++ b/Sources/tart/Commands/Push.swift @@ -6,7 +6,7 @@ import Compression struct Push: AsyncParsableCommand { static var configuration = CommandConfiguration(abstract: "Push a VM to a registry") - @Argument(help: "local VM name") + @Argument(help: "local or remote VM name") var localName: String @Argument(help: "remote VM name(s)") @@ -28,7 +28,8 @@ struct Push: AsyncParsableCommand { var populateCache: Bool = false func run() async throws { - let localVMDir = try VMStorageLocal().open(localName) + let ociStorage = VMStorageOCI() + let localVMDir = try VMStorageHelper.open(localName) // Parse remote names supplied by the user let remoteNames = try remoteNames.map{ @@ -53,23 +54,55 @@ struct Push: AsyncParsableCommand { defaultLogger.appendNewLine("pushing \(localName) to " + "\(registryIdentifier.host)/\(registryIdentifier.namespace)\(remoteNamesForRegistry.referenceNames())...") - let pushedRemoteName = try await localVMDir.pushToRegistry( - registry: registry, - references: remoteNamesForRegistry.map{ $0.reference.value }, - chunkSizeMb: chunkSize - ) + let references = remoteNamesForRegistry.map{ $0.reference.value } - // Populate the local cache (if requested) + let pushedRemoteName: RemoteName + // If we're pushing a local OCI VM, check if points to an already existing registry manifest + // and if so, only upload manifests (without config, disk and NVRAM) to the user-specified references + if let remoteName = try? RemoteName(localName) { + pushedRemoteName = try await lightweightPushToRegistry( + registry: registry, + remoteName: remoteName, + references: references + ) + } else { + pushedRemoteName = try await localVMDir.pushToRegistry( + registry: registry, + references: references, + chunkSizeMb: chunkSize + ) + // Populate the local cache (if requested) + if populateCache { + let expectedPushedVMDir = try ociStorage.create(pushedRemoteName) + try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false) + } + } + + // link the rest remote names if populateCache { - let ociStorage = VMStorageOCI() - let expectedPushedVMDir = try ociStorage.create(pushedRemoteName) - try localVMDir.clone(to: expectedPushedVMDir, generateMAC: false) for remoteName in remoteNamesForRegistry { try ociStorage.link(from: remoteName, to: pushedRemoteName) } } } } + + func lightweightPushToRegistry(registry: Registry, remoteName: RemoteName, references: [String]) async throws -> RemoteName { + // Is the local OCI VM already present in the registry? + let digest = try VMStorageOCI().digest(remoteName) + + let (remoteManifest, _) = try await registry.pullManifest(reference: digest) + + // Overwrite registry's references with the retrieved manifest + for reference in references { + defaultLogger.appendNewLine("pushing manifest for \(reference)...") + + _ = try await registry.pushManifest(reference: reference, manifest: remoteManifest) + } + + return RemoteName(host: registry.baseURL.host!, namespace: registry.namespace, + reference: Reference(digest: digest)) + } } extension Collection where Element == RemoteName { diff --git a/Sources/tart/VMDirectory.swift b/Sources/tart/VMDirectory.swift index 4bd01da..4abf572 100644 --- a/Sources/tart/VMDirectory.swift +++ b/Sources/tart/VMDirectory.swift @@ -55,9 +55,9 @@ struct VMDirectory: Prunable { try? FileManager.default.removeItem(at: nvramURL) } - func validate() throws { + func validate(userFriendlyName: String) throws { if !FileManager.default.fileExists(atPath: baseURL.path) { - throw RuntimeError.VMDoesNotExist(name: baseURL.lastPathComponent) + throw RuntimeError.VMDoesNotExist(name: userFriendlyName) } if !initialized { diff --git a/Sources/tart/VMStorageHelper.swift b/Sources/tart/VMStorageHelper.swift index ed65cc5..3b5ec96 100644 --- a/Sources/tart/VMStorageHelper.swift +++ b/Sources/tart/VMStorageHelper.swift @@ -57,6 +57,7 @@ enum RuntimeError : Error { case ExportFailed(_ message: String) case ImportFailed(_ message: String) case SoftnetFailed(_ message: String) + case OCIStorageError(_ message: String) } protocol HasExitCode { @@ -98,6 +99,8 @@ extension RuntimeError : CustomStringConvertible { return "VM import failed: \(message)" case .SoftnetFailed(let message): return "Softnet failed: \(message)" + case .OCIStorageError(let message): + return "OCI storage error: \(message)" } } } diff --git a/Sources/tart/VMStorageLocal.swift b/Sources/tart/VMStorageLocal.swift index 98e588f..531decf 100644 --- a/Sources/tart/VMStorageLocal.swift +++ b/Sources/tart/VMStorageLocal.swift @@ -14,7 +14,7 @@ class VMStorageLocal { func open(_ name: String) throws -> VMDirectory { let vmDir = VMDirectory(baseURL: vmURL(name)) - try vmDir.validate() + try vmDir.validate(userFriendlyName: name) return vmDir } diff --git a/Sources/tart/VMStorageOCI.swift b/Sources/tart/VMStorageOCI.swift index f126a9e..49e9052 100644 --- a/Sources/tart/VMStorageOCI.swift +++ b/Sources/tart/VMStorageOCI.swift @@ -16,10 +16,20 @@ class VMStorageOCI: PrunableStorage { VMDirectory(baseURL: vmURL(name)).initialized } + func digest(_ name: RemoteName) throws -> String { + let digest = vmURL(name).resolvingSymlinksInPath().lastPathComponent + + if !digest.starts(with: "sha256:") { + throw RuntimeError.OCIStorageError("\(name) is not a digest and doesn't point to a digest") + } + + return digest + } + func open(_ name: RemoteName) throws -> VMDirectory { let vmDir = VMDirectory(baseURL: vmURL(name)) - try vmDir.validate() + try vmDir.validate(userFriendlyName: name.description) try vmDir.baseURL.updateAccessDate()