diff --git a/benchmark/README.md b/benchmark/README.md index a307058..944299c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -22,6 +22,32 @@ You can also enable the debugging output to diagnose issues: go run cmd/main.go fio --debug ``` +To compare an empty Tart home with the same pull after its immutable base has +been prewarmed, provide a standalone remote base image and a stacked image built +from it. For example, create and push a stacked child of the public Tahoe base: + +```shell +BASE_IMAGE=ghcr.io/cirruslabs/macos-tahoe-base:latest +STACKED_IMAGE=ghcr.io/your-org/macos-tahoe-stacked:latest + +tart clone --stacked "$BASE_IMAGE" macos-tahoe-stacked +tart push macos-tahoe-stacked "$STACKED_IMAGE" +``` + +Then benchmark that pair: + +```shell +go run cmd/main.go stacked-oci \ + --base-image "$BASE_IMAGE" \ + --image "$STACKED_IMAGE" +``` + +The command first performs an unmeasured pull to warm registry, CDN, and +filesystem caches. It then uses disposable `TART_HOME` directories for both +measured scenarios. The prewarmed scenario keeps the VM created by +`tart clone --stacked` alive while pulling the child, so the shared immutable +base layer remains referenced and available for reuse. + ## Results ### Mar 27, 2024 diff --git a/benchmark/internal/command/root.go b/benchmark/internal/command/root.go index 77a33bc..e93d4bf 100644 --- a/benchmark/internal/command/root.go +++ b/benchmark/internal/command/root.go @@ -2,6 +2,7 @@ package command import ( "github.com/cirruslabs/tart/benchmark/internal/command/fio" + "github.com/cirruslabs/tart/benchmark/internal/command/stackedoci" "github.com/cirruslabs/tart/benchmark/internal/command/xcode" "github.com/spf13/cobra" ) @@ -15,6 +16,7 @@ func NewCommand() *cobra.Command { cmd.AddCommand( fio.NewCommand(), + stackedoci.NewCommand(), xcode.NewCommand(), ) diff --git a/benchmark/internal/command/stackedoci/stackedoci.go b/benchmark/internal/command/stackedoci/stackedoci.go new file mode 100644 index 0000000..dcb8238 --- /dev/null +++ b/benchmark/internal/command/stackedoci/stackedoci.go @@ -0,0 +1,161 @@ +package stackedoci + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/gosuri/uitable" + "github.com/spf13/cobra" + "go.uber.org/zap" + "go.uber.org/zap/zapio" +) + +var ( + debug bool + baseImage string + stackedImage string + insecure bool + concurrency uint +) + +func NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "stacked-oci", + Short: "benchmark empty and prewarmed Tart homes for stacked OCI pulls", + Long: "Warm the registry once, then compare an empty Tart home with one whose " + + "immutable base has already been materialized by tart clone --stacked. " + + "Every scenario uses a disposable TART_HOME and leaves the user's Tart home untouched.", + RunE: run, + } + + cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging") + cmd.Flags().StringVar(&baseImage, "base-image", "", "remote flat OCI image used as the stacked image's base") + cmd.Flags().StringVar(&stackedImage, "image", "", "remote stacked OCI image to pull and clone") + cmd.Flags().BoolVar(&insecure, "insecure", false, "connect to the OCI registry via insecure HTTP") + cmd.Flags().UintVar(&concurrency, "concurrency", 4, "network concurrency passed to tart pull and clone") + _ = cmd.MarkFlagRequired("base-image") + _ = cmd.MarkFlagRequired("image") + + return cmd +} + +func run(cmd *cobra.Command, _ []string) error { + if concurrency < 1 { + return fmt.Errorf("concurrency cannot be less than 1") + } + + config := zap.NewProductionConfig() + if debug { + config.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } + logger, err := config.Build() + if err != nil { + return err + } + defer func() { _ = logger.Sync() }() + + warmupHome, err := os.MkdirTemp("", "tart-stacked-oci-warmup-*") + if err != nil { + return err + } + defer os.RemoveAll(warmupHome) + + emptyHome, err := os.MkdirTemp("", "tart-stacked-oci-empty-*") + if err != nil { + return err + } + defer os.RemoveAll(emptyHome) + + warmHome, err := os.MkdirTemp("", "tart-stacked-oci-warm-*") + if err != nil { + return err + } + defer os.RemoveAll(warmHome) + + table := uitable.New() + table.AddRow("Scenario", "Operation", "Time") + + // Warm registry, CDN, and filesystem caches before either measured + // scenario so their difference reflects Tart's local base reuse. + if _, err := timedTart(cmd.Context(), logger, warmupHome, pullArguments(stackedImage)...); err != nil { + return fmt.Errorf("registry warmup failed: %w", err) + } + if err := os.RemoveAll(warmupHome); err != nil { + return fmt.Errorf("removing registry warmup home: %w", err) + } + + duration, err := timedTart(cmd.Context(), logger, emptyHome, pullArguments(stackedImage)...) + if err != nil { + return fmt.Errorf("empty-home stacked pull failed: %w", err) + } + table.AddRow("empty", "pull stacked image", duration) + + duration, err = timedTart(cmd.Context(), logger, emptyHome, "clone", stackedImage, "empty-clone") + if err != nil { + return fmt.Errorf("empty-home stacked clone failed: %w", err) + } + table.AddRow("empty", "clone stacked image", duration) + if err := os.RemoveAll(emptyHome); err != nil { + return fmt.Errorf("removing empty home: %w", err) + } + + duration, err = timedTart(cmd.Context(), logger, warmHome, cloneBaseArguments(baseImage)...) + if err != nil { + return fmt.Errorf("base prewarm failed: %w", err) + } + table.AddRow("prewarmed", "clone --stacked base image", duration) + + duration, err = timedTart(cmd.Context(), logger, warmHome, pullArguments(stackedImage)...) + if err != nil { + return fmt.Errorf("prewarmed stacked pull failed: %w", err) + } + table.AddRow("prewarmed", "pull stacked image", duration) + + duration, err = timedTart(cmd.Context(), logger, warmHome, "clone", stackedImage, "warm-clone") + if err != nil { + return fmt.Errorf("prewarmed stacked clone failed: %w", err) + } + table.AddRow("prewarmed", "clone stacked image", duration) + + fmt.Println(table.String()) + return nil +} + +func pullArguments(image string) []string { + args := []string{"pull", "--concurrency", fmt.Sprint(concurrency)} + if insecure { + args = append(args, "--insecure") + } + return append(args, image) +} + +func cloneBaseArguments(image string) []string { + args := []string{"clone", "--stacked", "--concurrency", fmt.Sprint(concurrency)} + if insecure { + args = append(args, "--insecure") + } + return append(args, image, "prewarmed-base") +} + +func timedTart( + ctx context.Context, + logger *zap.Logger, + tartHome string, + args ...string, +) (time.Duration, error) { + logger.Sugar().Debugf("TART_HOME=%s tart %s", tartHome, strings.Join(args, " ")) + start := time.Now() + + command := exec.CommandContext(ctx, "tart", args...) + command.Env = append(os.Environ(), "TART_HOME="+tartHome) + loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel} + command.Stdout = loggerWriter + command.Stderr = loggerWriter + + err := command.Run() + return time.Since(start).Round(time.Millisecond), err +} diff --git a/docs/faq.md b/docs/faq.md index 77c254a..9c692f8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -231,6 +231,28 @@ export TART_NO_AUTO_PRUNE= TART_NO_AUTO_PRUNE= tart pull ... ``` +## Stacked disk images + +On macOS 27 or newer, `tart clone --stacked` can create a VM from a remote, +standalone macOS OCI image whose writes are stored in a private ASIF overlay while +its source disk remains a shared read-only base: + +```shell +tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base:latest macos-build +``` + +Running and pushing `macos-build` preserves that disk relationship. Pulling +another image from the same lineage only downloads immutable disk files that +are not already present in Tart's cache. For a stopped stacked VM, +`tart set --disk-size` grows its private writable overlay without changing the +base; a subsequent push records the new guest-visible disk size. + +`tart pull` can cache a stacked image without assembling its disk. Clone, run, +import, and export require a Tart build with DiskImageKit support and macOS 27 +or newer. Existing standalone raw and ASIF images continue to work on older +hosts. Keep published lineages shallow when possible: every additional parent +overlay adds another ASIF file to validate and assemble at run time. + ## Disk resizing Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images have the `cloud-initramfs-growroot` package installed that runs on boot) and on the rest of the distributions by running the `growpart` or `resize2fs` commands. diff --git a/docs/quick-start.md b/docs/quick-start.md index eeed28a..325f4d3 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -265,3 +265,15 @@ tart clone acme.io/remoteorg/name:latest my-local-vm-name ``` If the specified image is not already present, this invocation calls the `tart pull` implicitly before cloning. + +### Creating a Stacked Disk + +On macOS 27 or newer, use `--stacked` to create a VM that keeps a remote standalone +macOS OCI image as an immutable base and stores only its own writes separately: + +```bash +tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base my-local-vm-name +``` + +Pushing this VM preserves the disk relationship. Pulling another image from the +same lineage reuses immutable disk files that are already in Tart's cache. diff --git a/integration-tests/test_stacked_oci.py b/integration-tests/test_stacked_oci.py new file mode 100644 index 0000000..43d16a0 --- /dev/null +++ b/integration-tests/test_stacked_oci.py @@ -0,0 +1,94 @@ +import os +import platform +import subprocess +import uuid +from typing import Optional + +import pytest +from paramiko.client import AutoAddPolicy, SSHClient + + +def _macos_major_version() -> int: + version = platform.mac_ver()[0] + return int(version.split(".", maxsplit=1)[0]) if version else 0 + + +def _shutdown_vm(tart, vm_name: str, guest_command: Optional[str] = None) -> None: + tart_run_process = tart.run_async(["run", "--no-graphics", vm_name]) + client = None + + try: + stdout, _ = tart.run(["ip", vm_name, "--wait", "180"]) + client = SSHClient() + client.set_missing_host_key_policy(AutoAddPolicy) + client.connect(stdout.strip(), username="admin", password="admin") + if guest_command: + _, command_stdout, command_stderr = client.exec_command(guest_command) + assert command_stdout.channel.recv_exit_status() == 0, command_stderr.read().decode() + client.exec_command("sudo shutdown -h now") + + tart_run_process.wait(timeout=180) + assert tart_run_process.returncode == 0 + finally: + if client is not None: + client.close() + if tart_run_process.poll() is None: + tart_run_process.terminate() + try: + tart_run_process.wait(timeout=30) + except subprocess.TimeoutExpired: + tart_run_process.kill() + tart_run_process.wait(timeout=30) + + +@pytest.mark.skipif( + _macos_major_version() < 27, + reason="stacked disk images require DiskImageKit on macOS 27 or newer", +) +def test_stacked_oci_round_trip_and_boot(tart, docker_registry): + suffix = str(uuid.uuid4()) + source_image = os.environ.get( + "TART_STACKED_INTEGRATION_BASE_IMAGE", + "ghcr.io/cirruslabs/macos-tahoe-base:latest", + ) + source_clone_args = ["clone"] + if os.environ.get("TART_STACKED_INTEGRATION_BASE_INSECURE") == "1": + source_clone_args.append("--insecure") + standalone_vm = f"stacked-base-{suffix}" + stacked_vm = f"stacked-child-{suffix}" + restored_vm = f"stacked-restored-{suffix}" + base_remote = docker_registry.remote_name(f"stacked-base-{suffix}") + child_remote = docker_registry.remote_name(f"stacked-child-{suffix}") + + try: + # Publish a normal Tart image, then start a stacked lineage from that + # remote image. The base remains a normal flat OCI image. + tart.run(source_clone_args + [source_image, standalone_vm]) + tart.run(["push", "--insecure", standalone_vm, base_remote]) + tart.run(["clone", "--insecure", "--stacked", base_remote, stacked_vm]) + + stacked_path = os.path.join(tart.home(), "vms", stacked_vm) + assert os.path.isfile(os.path.join(stacked_path, "overlay.asif")) + assert os.path.isfile(os.path.join(stacked_path, "manifest.json")) + assert not os.path.exists(os.path.join(stacked_path, "disk.img")) + + # Boot once so the top ASIF overlay contains real guest writes, then + # exercise stacked push, pull, clone, assembly, and a second boot. + _shutdown_vm(tart, stacked_vm, "touch ~/stacked-round-trip-marker") + tart.run(["push", "--insecure", stacked_vm, child_remote]) + tart.run(["delete", stacked_vm]) + tart.run(["pull", "--insecure", child_remote]) + tart.run(["clone", child_remote, restored_vm]) + + restored_path = os.path.join(tart.home(), "vms", restored_vm) + assert os.path.isfile(os.path.join(restored_path, "overlay.asif")) + assert os.path.isfile(os.path.join(restored_path, "manifest.json")) + assert not os.path.exists(os.path.join(restored_path, "disk.img")) + + _shutdown_vm(tart, restored_vm, "test -f ~/stacked-round-trip-marker") + finally: + for vm_name in (restored_vm, stacked_vm, standalone_vm, child_remote, base_remote): + try: + tart.run(["delete", vm_name]) + except Exception: + pass