mirror of
https://github.com/cirruslabs/tart.git
synced 2026-10-17 11:25:33 +02:00
Add stacked disk integration coverage and documentation (#1316)
This commit is contained in:
Yibo Zhuang
@@ -22,6 +22,32 @@ You can also enable the debugging output to diagnose issues:
|
||||
go run cmd/main.go fio --debug
|
||||
```
|
||||
|
||||
To compare an empty Tart home with the same pull after its immutable base has
|
||||
been prewarmed, provide a standalone remote base image and a stacked image built
|
||||
from it. For example, create and push a stacked child of the public Tahoe base:
|
||||
|
||||
```shell
|
||||
BASE_IMAGE=ghcr.io/cirruslabs/macos-tahoe-base:latest
|
||||
STACKED_IMAGE=ghcr.io/your-org/macos-tahoe-stacked:latest
|
||||
|
||||
tart clone --stacked "$BASE_IMAGE" macos-tahoe-stacked
|
||||
tart push macos-tahoe-stacked "$STACKED_IMAGE"
|
||||
```
|
||||
|
||||
Then benchmark that pair:
|
||||
|
||||
```shell
|
||||
go run cmd/main.go stacked-oci \
|
||||
--base-image "$BASE_IMAGE" \
|
||||
--image "$STACKED_IMAGE"
|
||||
```
|
||||
|
||||
The command first performs an unmeasured pull to warm registry, CDN, and
|
||||
filesystem caches. It then uses disposable `TART_HOME` directories for both
|
||||
measured scenarios. The prewarmed scenario keeps the VM created by
|
||||
`tart clone --stacked` alive while pulling the child, so the shared immutable
|
||||
base layer remains referenced and available for reuse.
|
||||
|
||||
## Results
|
||||
|
||||
### Mar 27, 2024
|
||||
|
||||
@@ -2,6 +2,7 @@ package command
|
||||
|
||||
import (
|
||||
"github.com/cirruslabs/tart/benchmark/internal/command/fio"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/command/stackedoci"
|
||||
"github.com/cirruslabs/tart/benchmark/internal/command/xcode"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -15,6 +16,7 @@ func NewCommand() *cobra.Command {
|
||||
|
||||
cmd.AddCommand(
|
||||
fio.NewCommand(),
|
||||
stackedoci.NewCommand(),
|
||||
xcode.NewCommand(),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package stackedoci
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gosuri/uitable"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapio"
|
||||
)
|
||||
|
||||
var (
|
||||
debug bool
|
||||
baseImage string
|
||||
stackedImage string
|
||||
insecure bool
|
||||
concurrency uint
|
||||
)
|
||||
|
||||
func NewCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "stacked-oci",
|
||||
Short: "benchmark empty and prewarmed Tart homes for stacked OCI pulls",
|
||||
Long: "Warm the registry once, then compare an empty Tart home with one whose " +
|
||||
"immutable base has already been materialized by tart clone --stacked. " +
|
||||
"Every scenario uses a disposable TART_HOME and leaves the user's Tart home untouched.",
|
||||
RunE: run,
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging")
|
||||
cmd.Flags().StringVar(&baseImage, "base-image", "", "remote flat OCI image used as the stacked image's base")
|
||||
cmd.Flags().StringVar(&stackedImage, "image", "", "remote stacked OCI image to pull and clone")
|
||||
cmd.Flags().BoolVar(&insecure, "insecure", false, "connect to the OCI registry via insecure HTTP")
|
||||
cmd.Flags().UintVar(&concurrency, "concurrency", 4, "network concurrency passed to tart pull and clone")
|
||||
_ = cmd.MarkFlagRequired("base-image")
|
||||
_ = cmd.MarkFlagRequired("image")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, _ []string) error {
|
||||
if concurrency < 1 {
|
||||
return fmt.Errorf("concurrency cannot be less than 1")
|
||||
}
|
||||
|
||||
config := zap.NewProductionConfig()
|
||||
if debug {
|
||||
config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
|
||||
}
|
||||
logger, err := config.Build()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = logger.Sync() }()
|
||||
|
||||
warmupHome, err := os.MkdirTemp("", "tart-stacked-oci-warmup-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(warmupHome)
|
||||
|
||||
emptyHome, err := os.MkdirTemp("", "tart-stacked-oci-empty-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(emptyHome)
|
||||
|
||||
warmHome, err := os.MkdirTemp("", "tart-stacked-oci-warm-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(warmHome)
|
||||
|
||||
table := uitable.New()
|
||||
table.AddRow("Scenario", "Operation", "Time")
|
||||
|
||||
// Warm registry, CDN, and filesystem caches before either measured
|
||||
// scenario so their difference reflects Tart's local base reuse.
|
||||
if _, err := timedTart(cmd.Context(), logger, warmupHome, pullArguments(stackedImage)...); err != nil {
|
||||
return fmt.Errorf("registry warmup failed: %w", err)
|
||||
}
|
||||
if err := os.RemoveAll(warmupHome); err != nil {
|
||||
return fmt.Errorf("removing registry warmup home: %w", err)
|
||||
}
|
||||
|
||||
duration, err := timedTart(cmd.Context(), logger, emptyHome, pullArguments(stackedImage)...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("empty-home stacked pull failed: %w", err)
|
||||
}
|
||||
table.AddRow("empty", "pull stacked image", duration)
|
||||
|
||||
duration, err = timedTart(cmd.Context(), logger, emptyHome, "clone", stackedImage, "empty-clone")
|
||||
if err != nil {
|
||||
return fmt.Errorf("empty-home stacked clone failed: %w", err)
|
||||
}
|
||||
table.AddRow("empty", "clone stacked image", duration)
|
||||
if err := os.RemoveAll(emptyHome); err != nil {
|
||||
return fmt.Errorf("removing empty home: %w", err)
|
||||
}
|
||||
|
||||
duration, err = timedTart(cmd.Context(), logger, warmHome, cloneBaseArguments(baseImage)...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("base prewarm failed: %w", err)
|
||||
}
|
||||
table.AddRow("prewarmed", "clone --stacked base image", duration)
|
||||
|
||||
duration, err = timedTart(cmd.Context(), logger, warmHome, pullArguments(stackedImage)...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prewarmed stacked pull failed: %w", err)
|
||||
}
|
||||
table.AddRow("prewarmed", "pull stacked image", duration)
|
||||
|
||||
duration, err = timedTart(cmd.Context(), logger, warmHome, "clone", stackedImage, "warm-clone")
|
||||
if err != nil {
|
||||
return fmt.Errorf("prewarmed stacked clone failed: %w", err)
|
||||
}
|
||||
table.AddRow("prewarmed", "clone stacked image", duration)
|
||||
|
||||
fmt.Println(table.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func pullArguments(image string) []string {
|
||||
args := []string{"pull", "--concurrency", fmt.Sprint(concurrency)}
|
||||
if insecure {
|
||||
args = append(args, "--insecure")
|
||||
}
|
||||
return append(args, image)
|
||||
}
|
||||
|
||||
func cloneBaseArguments(image string) []string {
|
||||
args := []string{"clone", "--stacked", "--concurrency", fmt.Sprint(concurrency)}
|
||||
if insecure {
|
||||
args = append(args, "--insecure")
|
||||
}
|
||||
return append(args, image, "prewarmed-base")
|
||||
}
|
||||
|
||||
func timedTart(
|
||||
ctx context.Context,
|
||||
logger *zap.Logger,
|
||||
tartHome string,
|
||||
args ...string,
|
||||
) (time.Duration, error) {
|
||||
logger.Sugar().Debugf("TART_HOME=%s tart %s", tartHome, strings.Join(args, " "))
|
||||
start := time.Now()
|
||||
|
||||
command := exec.CommandContext(ctx, "tart", args...)
|
||||
command.Env = append(os.Environ(), "TART_HOME="+tartHome)
|
||||
loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel}
|
||||
command.Stdout = loggerWriter
|
||||
command.Stderr = loggerWriter
|
||||
|
||||
err := command.Run()
|
||||
return time.Since(start).Round(time.Millisecond), err
|
||||
}
|
||||
+22
@@ -231,6 +231,28 @@ export TART_NO_AUTO_PRUNE=
|
||||
TART_NO_AUTO_PRUNE= tart pull ...
|
||||
```
|
||||
|
||||
## Stacked disk images
|
||||
|
||||
On macOS 27 or newer, `tart clone --stacked` can create a VM from a remote,
|
||||
standalone macOS OCI image whose writes are stored in a private ASIF overlay while
|
||||
its source disk remains a shared read-only base:
|
||||
|
||||
```shell
|
||||
tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base:latest macos-build
|
||||
```
|
||||
|
||||
Running and pushing `macos-build` preserves that disk relationship. Pulling
|
||||
another image from the same lineage only downloads immutable disk files that
|
||||
are not already present in Tart's cache. For a stopped stacked VM,
|
||||
`tart set --disk-size` grows its private writable overlay without changing the
|
||||
base; a subsequent push records the new guest-visible disk size.
|
||||
|
||||
`tart pull` can cache a stacked image without assembling its disk. Clone, run,
|
||||
import, and export require a Tart build with DiskImageKit support and macOS 27
|
||||
or newer. Existing standalone raw and ASIF images continue to work on older
|
||||
hosts. Keep published lineages shallow when possible: every additional parent
|
||||
overlay adds another ASIF file to validate and assemble at run time.
|
||||
|
||||
## Disk resizing
|
||||
|
||||
Disk resizing works on most cloud-ready Linux distributions out-of-the box (e.g. Ubuntu Cloud Images have the `cloud-initramfs-growroot` package installed that runs on boot) and on the rest of the distributions by running the `growpart` or `resize2fs` commands.
|
||||
|
||||
@@ -265,3 +265,15 @@ tart clone acme.io/remoteorg/name:latest my-local-vm-name
|
||||
```
|
||||
|
||||
If the specified image is not already present, this invocation calls the `tart pull` implicitly before cloning.
|
||||
|
||||
### Creating a Stacked Disk
|
||||
|
||||
On macOS 27 or newer, use `--stacked` to create a VM that keeps a remote standalone
|
||||
macOS OCI image as an immutable base and stores only its own writes separately:
|
||||
|
||||
```bash
|
||||
tart clone --stacked ghcr.io/cirruslabs/macos-tahoe-base my-local-vm-name
|
||||
```
|
||||
|
||||
Pushing this VM preserves the disk relationship. Pulling another image from the
|
||||
same lineage reuses immutable disk files that are already in Tart's cache.
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
from paramiko.client import AutoAddPolicy, SSHClient
|
||||
|
||||
|
||||
def _macos_major_version() -> int:
|
||||
version = platform.mac_ver()[0]
|
||||
return int(version.split(".", maxsplit=1)[0]) if version else 0
|
||||
|
||||
|
||||
def _shutdown_vm(tart, vm_name: str, guest_command: Optional[str] = None) -> None:
|
||||
tart_run_process = tart.run_async(["run", "--no-graphics", vm_name])
|
||||
client = None
|
||||
|
||||
try:
|
||||
stdout, _ = tart.run(["ip", vm_name, "--wait", "180"])
|
||||
client = SSHClient()
|
||||
client.set_missing_host_key_policy(AutoAddPolicy)
|
||||
client.connect(stdout.strip(), username="admin", password="admin")
|
||||
if guest_command:
|
||||
_, command_stdout, command_stderr = client.exec_command(guest_command)
|
||||
assert command_stdout.channel.recv_exit_status() == 0, command_stderr.read().decode()
|
||||
client.exec_command("sudo shutdown -h now")
|
||||
|
||||
tart_run_process.wait(timeout=180)
|
||||
assert tart_run_process.returncode == 0
|
||||
finally:
|
||||
if client is not None:
|
||||
client.close()
|
||||
if tart_run_process.poll() is None:
|
||||
tart_run_process.terminate()
|
||||
try:
|
||||
tart_run_process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
tart_run_process.kill()
|
||||
tart_run_process.wait(timeout=30)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_macos_major_version() < 27,
|
||||
reason="stacked disk images require DiskImageKit on macOS 27 or newer",
|
||||
)
|
||||
def test_stacked_oci_round_trip_and_boot(tart, docker_registry):
|
||||
suffix = str(uuid.uuid4())
|
||||
source_image = os.environ.get(
|
||||
"TART_STACKED_INTEGRATION_BASE_IMAGE",
|
||||
"ghcr.io/cirruslabs/macos-tahoe-base:latest",
|
||||
)
|
||||
source_clone_args = ["clone"]
|
||||
if os.environ.get("TART_STACKED_INTEGRATION_BASE_INSECURE") == "1":
|
||||
source_clone_args.append("--insecure")
|
||||
standalone_vm = f"stacked-base-{suffix}"
|
||||
stacked_vm = f"stacked-child-{suffix}"
|
||||
restored_vm = f"stacked-restored-{suffix}"
|
||||
base_remote = docker_registry.remote_name(f"stacked-base-{suffix}")
|
||||
child_remote = docker_registry.remote_name(f"stacked-child-{suffix}")
|
||||
|
||||
try:
|
||||
# Publish a normal Tart image, then start a stacked lineage from that
|
||||
# remote image. The base remains a normal flat OCI image.
|
||||
tart.run(source_clone_args + [source_image, standalone_vm])
|
||||
tart.run(["push", "--insecure", standalone_vm, base_remote])
|
||||
tart.run(["clone", "--insecure", "--stacked", base_remote, stacked_vm])
|
||||
|
||||
stacked_path = os.path.join(tart.home(), "vms", stacked_vm)
|
||||
assert os.path.isfile(os.path.join(stacked_path, "overlay.asif"))
|
||||
assert os.path.isfile(os.path.join(stacked_path, "manifest.json"))
|
||||
assert not os.path.exists(os.path.join(stacked_path, "disk.img"))
|
||||
|
||||
# Boot once so the top ASIF overlay contains real guest writes, then
|
||||
# exercise stacked push, pull, clone, assembly, and a second boot.
|
||||
_shutdown_vm(tart, stacked_vm, "touch ~/stacked-round-trip-marker")
|
||||
tart.run(["push", "--insecure", stacked_vm, child_remote])
|
||||
tart.run(["delete", stacked_vm])
|
||||
tart.run(["pull", "--insecure", child_remote])
|
||||
tart.run(["clone", child_remote, restored_vm])
|
||||
|
||||
restored_path = os.path.join(tart.home(), "vms", restored_vm)
|
||||
assert os.path.isfile(os.path.join(restored_path, "overlay.asif"))
|
||||
assert os.path.isfile(os.path.join(restored_path, "manifest.json"))
|
||||
assert not os.path.exists(os.path.join(restored_path, "disk.img"))
|
||||
|
||||
_shutdown_vm(tart, restored_vm, "test -f ~/stacked-round-trip-marker")
|
||||
finally:
|
||||
for vm_name in (restored_vm, stacked_vm, standalone_vm, child_remote, base_remote):
|
||||
try:
|
||||
tart.run(["delete", vm_name])
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user