feat: --bake-images — pre-pulled multi-arch release-image store

.gitignore

@@ -35,3 +35,4 @@ go.work.sum
 cloudflare-token
 pull-secret
 pull-secret*.txt
+.gstack/

ROADMAP.md

@@ -28,24 +28,24 @@ mounted with `context=container_file_t` — see the Task 13 notes in the spec),
       on darwin). Needs a vmnet bridged-mode story before LAN-reachable
       clusters work on Mac.
 
-## Image baking (`--bake-images`) — branch `rtalur-bake-images`
-
-Complete and green on Linux (libvirt): opt-in flag, multi-arch store baked via
-skopeo + virt-make-fs, attached read-only, wired into live-ISO ignition and a
-master MachineConfig.
-
-- [ ] **macOS integration** (deliberately deferred to the on-hardware phase;
-      whichever branch merges second closes this):
-      - merge the baked-store wiring into the HTTP-served ignition in
-        `publish-pxe-assets` (the macOS equivalent of the live-ISO merge in
-        `embed-ignition-iso`);
-      - build the store without libguestfs: `mke2fs -d` (brew `e2fsprogs`)
-        instead of `virt-make-fs`, raw image instead of qcow2 (vfkit's
-        virtio-blk takes raw);
-      - a vfkit-side `ImportDisk` equivalent so `create-master-vms` can attach
-        the per-cluster store copy.
-- [ ] **End-to-end validation on Linux**: a real `--bake-images` install that
-      confirms the node pulls platform images from the store, not quay.io.
+## Image baking (`--bake-images`) — branch `rtalur-bake-images-macos`
+
+Code-complete on both backends (see `docs/dev/image-baking.md`). Linux:
+skopeo + virt-make-fs → qcow2 into the libvirt pool. macOS: skopeo +
+`mke2fs -d` → raw image, APFS-cloned per cluster, merged into the HTTP-served
+ignition. `make check` green; darwin pipeline validated under `--simulate` and
+the mke2fs pack validated against real e2fsprogs.
+
+Branch note: `rtalur-bake-images` is the same feature rebased onto `main`
+(without the macOS backend); `rtalur-bake-images-macos` stacks it on
+`rtalur-macos-backend` and adds the macOS integration. Whichever merge order
+is chosen, keep only one of the two bake branches.
+
+- [ ] **End-to-end validation on a real cluster** (either OS): a
+      `--bake-images` install confirming CRI-O serves release images from the
+      store, not quay.io, in both bootstrap and post-pivot phases. Needs
+      ~40+ GB free disk for the multi-arch store — did not fit the dev Mac
+      alongside the existing cluster.
 
 ## Later phases (per project vision)
 

app/cluster_test.go

@@ -92,6 +92,67 @@ func newTestEnv(t *testing.T) (*config.Config, interfaces.Deps, *fakes.Bundle) {
 	return cfg, deps, bundle
 }
 
+// TestCreateCluster_BakeImages exercises the --bake-images path end-to-end
+// with fakes: the bake stage builds the store, the ignition gets the
+// MachineConfig + live-ISO merge, and the master gets a read-only store disk.
+func TestCreateCluster_BakeImages(t *testing.T) {
+	cfg, deps, bundle := newTestEnv(t)
+	mgr := app.NewClusterManager(cfg, deps)
+
+	c := newTestCluster("baked")
+	c.BakeImages = true
+	if err := mgr.Create(context.Background(), c); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+
+	if got := len(bundle.ImageBaker.Baked); got != 1 {
+		t.Fatalf("ImageBaker.Bake calls: got %d want 1", got)
+	}
+	if !bundle.Installer.WroteImageStoreManifest {
+		t.Error("expected WriteImageStoreManifest to be called")
+	}
+	// Both boot-media paths wire the store into the bootstrap ignition: Linux
+	// merges into the live-ISO ignition (embed-ignition-iso), macOS into the
+	// HTTP-served ignition (publish-pxe-assets).
+	if !bundle.Installer.MergedImageStoreIgnition {
+		t.Error("expected MergeImageStoreIntoLiveISOIgnition to be called")
+	}
+	if got := len(bundle.VM.ImportedDisks); got != 1 {
+		t.Fatalf("ImportDisk calls: got %d want 1", got)
+	}
+	// The master VM must carry exactly one read-only, shareable extra disk.
+	if len(bundle.VM.Created) != 1 {
+		t.Fatalf("VMs created: got %d want 1", len(bundle.VM.Created))
+	}
+	disks := bundle.VM.Created[0].ExtraDisks
+	if len(disks) != 1 {
+		t.Fatalf("master extra disks: got %d want 1", len(disks))
+	}
+	if !disks[0].ReadOnly || !disks[0].Shareable {
+		t.Errorf("store disk must be read-only + shareable, got %+v", disks[0])
+	}
+}
+
+// TestCreateCluster_NoBakeByDefault confirms baking is opt-in: a default
+// cluster touches none of the bake machinery.
+func TestCreateCluster_NoBakeByDefault(t *testing.T) {
+	cfg, deps, bundle := newTestEnv(t)
+	mgr := app.NewClusterManager(cfg, deps)
+
+	if err := mgr.Create(context.Background(), newTestCluster("plain")); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	if got := len(bundle.ImageBaker.Baked); got != 0 {
+		t.Errorf("expected no bakes, got %d", got)
+	}
+	if bundle.Installer.WroteImageStoreManifest || bundle.Installer.MergedImageStoreIgnition {
+		t.Error("bake-store installer methods should not be called without --bake-images")
+	}
+	if len(bundle.VM.Created) == 1 && len(bundle.VM.Created[0].ExtraDisks) != 0 {
+		t.Errorf("master should have no extra disks without --bake-images")
+	}
+}
+
 // TestCreateCluster_HappyPath walks the full stage list with all-fake deps
 // and asserts each interface saw the expected call.
 func TestCreateCluster_HappyPath(t *testing.T) {
@@ -185,6 +246,7 @@ func TestCreateCluster_HappyPath(t *testing.T) {
 		"ensure-cluster-dir",
 		"download-binaries",
 		"download-rhcos",
+		"bake-image-store",
 		"generate-ssh-key",
 		"generate-ignition",
 		bootMediaStageName(),

app/deps.go

@@ -52,6 +52,7 @@ func baseDeps(cfg *config.Config, hostIP string) (interfaces.Deps, interfaces.Co
 		Cmd:        cmd,
 		Download:   dl,
 		Installer:  openshift.NewOpenShiftInstaller(cmd),
+		ImageBaker: openshift.NewOpenShiftImageBaker(cmd),
 		Files:      files,
 		CSR:        csr.NewOCCSRApprover(cmd),
 		Hostname:   host.NewSSHHostnameInjector(cmd),

app/deps_test.go

@@ -17,8 +17,8 @@ func TestNewDarwinDeps_WiresMacProviders(t *testing.T) {
 	if deps.VM == nil || deps.Net == nil {
 		t.Fatal("darwin deps must wire VM and Net")
 	}
-	if deps.Installer == nil || deps.Files == nil {
-		t.Fatal("darwin deps must wire the shared deps (installer, files)")
+	if deps.Installer == nil || deps.ImageBaker == nil || deps.Files == nil {
+		t.Fatal("darwin deps must wire the shared deps (installer, image baker, files)")
 	}
 	// The vfkit VMManager treats ImportISO as a no-op (libvirt would shell out).
 	if _, err := deps.VM.ImportISO(context.Background(), "p", "v", "/tmp/x"); err != nil {

app/manager.go

@@ -19,6 +19,7 @@ import (
 	"github.com/TheEasyShift/easyshift/providers/openshift"
 	"github.com/TheEasyShift/easyshift/stages/allocatenetwork"
 	"github.com/TheEasyShift/easyshift/stages/applytlscerts"
+	"github.com/TheEasyShift/easyshift/stages/bakeimagestore"
 	"github.com/TheEasyShift/easyshift/stages/createmastervms"
 	"github.com/TheEasyShift/easyshift/stages/createnetwork"
 	"github.com/TheEasyShift/easyshift/stages/downloadbinaries"
@@ -68,7 +69,7 @@ func (cm *ClusterManager) buildStages() []interfaces.Stage {
 	// (kernel/initrd/rootfs + ignition over HTTP) for vfkit's Linux bootloader.
 	bootMedia := interfaces.Stage(embedignitioniso.New(d.Installer, d.VM))
 	if runtime.GOOS == "darwin" {
-		bootMedia = publishpxeassets.New(d.Files)
+		bootMedia = publishpxeassets.New(d.Files, d.Installer)
 	}
 	return []interfaces.Stage{
 		registercluster.New(),
@@ -77,6 +78,7 @@ func (cm *ClusterManager) buildStages() []interfaces.Stage {
 		ensureclusterdir.New(),
 		downloadbinaries.New(d.Download, d.Cmd, d.Host),
 		downloadrhcos.New(d.Installer, d.Download),
+		bakeimagestore.New(d.ImageBaker, d.Host),
 		generatesshkey.New(d.Cmd, d.Host),
 		generateignition.New(d.Installer, d.DNS, d.Host),
 		bootMedia,

cmd/easyshift/main.go

@@ -141,6 +141,7 @@ func newCreateCommand(mgr **app.ClusterManager, simBundle **fakes.Bundle, cfgp *
 		tlsEmail    string
 		tlsStaging  bool
 		magicDNS    string
+		bakeImages  bool
 	)
 
 	cmd := &cobra.Command{
@@ -170,6 +171,7 @@ func newCreateCommand(mgr **app.ClusterManager, simBundle **fakes.Bundle, cfgp *
 				TLSEmail:     tlsEmail,
 				TLSStaging:   tlsStaging,
 				MagicDNS:     magicDNS,
+				BakeImages:   bakeImages,
 			}
 			// In a bridge-mode simulation there is no real node, so pretend it
 			// came up on its reserved IP — otherwise the verify-master-ip stage
@@ -238,6 +240,10 @@ func newCreateCommand(mgr **app.ClusterManager, simBundle **fakes.Bundle, cfgp *
 		"Wildcard DNS service so cluster names resolve to the master IP with no records to manage: "+
 			"'auto' (NAT -> sslip.io, bridge -> off), 'sslip.io', 'nip.io', or 'off'. "+
 			"Mutually exclusive with --dns-provider.")
+	cmd.Flags().BoolVar(&bakeImages, "bake-images", false,
+		"Pre-pull the entire OCP release payload into a read-only disk attached to the master so "+
+			"the install never reaches quay.io for platform images. Built once per version (multi-arch: "+
+			"amd64 + aarch64), shared across clusters. Needs skopeo + virt-make-fs on PATH.")
 
 	_ = cmd.MarkFlagRequired("name")
 	return cmd

config/config.go

@@ -88,6 +88,16 @@ const (
 	ClusterStateRunning  = "running"
 	ClusterStateStopped  = "stopped"
 	ClusterStateError    = "error"
+
+	// BakedImagesLabel is the ext4 filesystem label of the read-only disk that
+	// carries the pre-pulled release payload (a CRI-O "additional image store").
+	// Both the live-ISO ignition and the installed node's MachineConfig mount
+	// the disk by this label, so device ordering on the VM is irrelevant.
+	BakedImagesLabel = "baked-images"
+	// BakedImagesMountPath is where the baked store disk is mounted on the node
+	// and the path added to CRI-O's additionalimagestores. CRI-O then serves
+	// release images from here instead of pulling them from quay.io.
+	BakedImagesMountPath = "/var/lib/baked-images"
 )
 
 // Config is the global on-disk configuration. There is no singleton; callers
@@ -169,6 +179,14 @@ type ClusterConfig struct {
 	// --magic-dns flag's "auto"/"off" by the manager before any stage runs.
 	MagicDNS string `json:"magicDNS,omitempty"`
 
+	// BakeImages, when true, pre-pulls the entire OCP release payload into a
+	// read-only disk attached to the master so the install never reaches
+	// quay.io for platform images. The store is built once per OCP version,
+	// shared across clusters, and is multi-arch (every supported release arch),
+	// so an amd64 or aarch64 (Rosetta) node finds its images locally. See
+	// docs/dev/image-baking.md.
+	BakeImages bool `json:"bakeImages,omitempty"`
+
 	NetworkSubnet string   `json:"networkSubnet"`
 	IPAddresses   []string `json:"ipAddresses"`
 	MACAddresses  []string `json:"macAddresses"`

config/paths.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 )
 
@@ -31,6 +32,53 @@ func RHCOSCacheDir(configDir, version string) string {
 	return filepath.Join(configDir, "rhcos", version)
 }
 
+// ImageStoreCacheDir is the per-version cache root for the baked image store.
+// It holds the CRI-O overlay container store (built once with skopeo) and the
+// packed qcow2 produced from it. Shared across clusters of the same version.
+func ImageStoreCacheDir(configDir, version string) string {
+	return filepath.Join(configDir, "imagestore", version)
+}
+
+// ImageStoreOverlayDir is the CRI-O container storage graphroot that skopeo
+// copies release images into, inside the per-version cache.
+func ImageStoreOverlayDir(configDir, version string) string {
+	return filepath.Join(ImageStoreCacheDir(configDir, version), "store")
+}
+
+// ImageStoreDiskPath is the packed read-only disk (an ext4 filesystem labeled
+// BakedImagesLabel containing the overlay store) for the given version. Built
+// once on the host; a per-cluster copy is attached to the master. Linux packs
+// a qcow2 (libvirt vol-upload); macOS packs a raw image (vfkit's virtio-blk
+// takes raw), hence the per-OS extension.
+func ImageStoreDiskPath(configDir, version string) string {
+	return filepath.Join(ImageStoreCacheDir(configDir, version), "store."+imageStoreDiskExt())
+}
+
+// ImageStoreVolName is the per-cluster name for the attached baked-image-store
+// disk: a libvirt pool volume on Linux, a file in the vfkit state dir on
+// macOS. Per-cluster (not shared) so cluster delete — which removes all of a
+// domain's storage — never strands another cluster.
+func ImageStoreVolName(name string) string {
+	return "easyshift-" + name + "-imagestore." + imageStoreDiskExt()
+}
+
+func imageStoreDiskExt() string {
+	if runtime.GOOS == "darwin" {
+		return "img"
+	}
+	return "qcow2"
+}
+
+// MKE2FSCandidates are the locations probed for mke2fs on macOS (used to pack
+// the baked image store; Homebrew's e2fsprogs is keg-only, so its sbin is
+// normally not on PATH). Bare names are resolved via PATH, absolute paths via
+// stat.
+var MKE2FSCandidates = []string{
+	"mke2fs",
+	"/opt/homebrew/opt/e2fsprogs/sbin/mke2fs",
+	"/usr/local/opt/e2fsprogs/sbin/mke2fs",
+}
+
 // ClusterDNSNames returns the DNS names a bridge-mode cluster needs, all of
 // which must resolve to the master IP. The wildcard *.apps is probed via a
 // synthetic console hostname because a literal "*" lookup isn't valid DNS.

docs/README.md

@@ -24,5 +24,6 @@ A good reading order: **installation → configuration → usage**, then
 | [architecture.md](dev/architecture.md) | Package layering and the staged-installer model |
 | [stages.md](dev/stages.md) | The stage contract; adding or changing a stage |
 | [providers.md](dev/providers.md) | Interfaces, provider implementations, and fakes |
+| [image-baking.md](dev/image-baking.md) | `--bake-images`: pre-pulling the release payload into a read-only store |
 | [testing.md](dev/testing.md) | Make targets, fakes, and `--simulate` |
 | [contributing.md](dev/contributing.md) | Workflow, commit conventions, PR flow |

docs/dev/image-baking.md

@@ -0,0 +1,99 @@
+# Baked image store (`--bake-images`)
+
+A fresh single-node OpenShift install pulls the entire release payload — the
+release image plus hundreds of component images, several GB — from `quay.io`,
+twice: once during the live-ISO **bootstrap** phase and again on the installed
+node as the operators roll out **post-pivot**. On a dev box building up to three
+clusters that is the dominant chunk of wall-clock and bandwidth.
+
+`--bake-images` pre-pulls that payload into a read-only disk attached to the
+master, so CRI-O serves platform images locally and never reaches `quay.io`.
+This is the same mechanism Red Hat's `factory-precaching-cli` uses for Telco/ZTP
+factory installs, hand-rolled to fit easyshift's stage pipeline and no-root
+contract.
+
+## How it works
+
+The store is a CRI-O **additional image store**: a read-only container store
+that CRI-O layers underneath its writable one. When the node asks for a release
+image, CRI-O finds it locally and skips the pull. Images are stored under their
+**original names** (`quay.io/...@sha256:...`), so the digests the release
+references match with no `imageDigestMirrors` / ICSP config needed.
+
+### Build (host side) — `stages/bakeimagestore`
+
+Built once per OCP version, cached at
+`~/.config/easyshift/imagestore/<version>/`, shared across clusters (rollback is
+a no-op, like the binaries cache). `providers/openshift.OpenShiftImageBaker`:
+
+1. `oc adm release info --pullspecs -o json <release-image>` for **every**
+   supported arch enumerates the component pullspecs.
+2. `skopeo copy --all` copies each into an overlay container store
+   (`store/`). `--all` keeps every manifest-list entry.
+3. The store is packed into a labeled ext4 disk image (`PackCommand`):
+   - **Linux**: `virt-make-fs --type=ext4 --label=baked-images --format=qcow2`
+     → `store.qcow2` (rootless via libguestfs; qcow2 because a per-cluster
+     copy is uploaded into the libvirt pool).
+   - **macOS**: `mke2fs -t ext4 -L baked-images -d <store>` → `store.img`
+     (raw — vfkit's virtio-blk takes raw). No libguestfs exists on macOS;
+     `mke2fs -d` populates the fs from a directory without mounting. The
+     image is sized from the overlay contents (+10% and 1 GiB headroom).
+     Homebrew's e2fsprogs is keg-only, so its keg sbin paths are probed
+     (`config.MKE2FSCandidates`).
+
+Preflight requires `skopeo` plus the per-OS packer: `virt-make-fs`
+(guestfs-tools / libguestfs-tools) on Linux, `mke2fs`
+(`brew install e2fsprogs`) on macOS.
+
+### Multi-arch
+
+The store is **multi-arch**. `SupportedReleaseArches` (`x86_64`, `aarch64`) are
+each enumerated from their arch-specific release image
+(`ocp-release:<version>-<arch>`) and unioned; an arch whose release image
+doesn't exist for the version is skipped. One store therefore serves an amd64
+node, an aarch64 node, and amd64 workloads run on aarch64 via Rosetta. The
+RHCOS live-ISO arch is still selected separately (see
+`providers/openshift.coreOSArch`) — baking does not change which ISO boots.
+
+### Attach + wire (node side)
+
+- `stages/createmastervms` attaches a **per-cluster** copy of the store via
+  `ImportDisk`: on Linux uploaded into the libvirt pool (read-only +
+  shareable; per cluster so `virsh undefine --remove-all-storage` on delete
+  never strands another cluster), on macOS APFS-cloned into the vfkit state
+  dir (vfkit has no read-only virtio-blk, so the per-cluster copy is the
+  isolation; `Delete` removes the clone).
+- The disk is mounted by label (`/dev/disk/by-label/baked-images` →
+  `/var/lib/baked-images`) and registered with CRI-O via
+  `additionalimagestores` in a `storage.conf` drop-in.
+- That wiring is applied in **both** install phases:
+  - **post-pivot:** a master `MachineConfig` dropped into the install dir's
+    `openshift/` (`Installer.WriteImageStoreManifest`) so it is rendered into
+    the node's ignition and present from first boot, before CRI-O pulls
+    operators.
+  - **bootstrap:** the same file + mount unit merged into the bootstrap
+    ignition (`Installer.MergeImageStoreIntoLiveISOIgnition`) — on Linux into
+    `bootstrap-in-place-for-live-iso.ign` before the ISO is embedded
+    (`embed-ignition-iso`), on macOS into the HTTP-served `config.ign`
+    (`publish-pxe-assets`).
+
+Renderers live in `providers/openshift/baker.go` (`RenderStorageConfDropin`,
+`RenderMountUnit`, `RenderMachineConfig`, `MergeBakedStoreIntoIgnition`) and are
+unit-tested in `baker_test.go`.
+
+## Verification boundary
+
+The pipeline, the rendered artifacts, and the `--simulate` trace are covered by
+unit + app tests. What needs a **real cluster** to confirm:
+
+- CRI-O actually resolves release images from the mounted store (no `quay.io`
+  pull) in both phases.
+- `create single-node-ignition-config` picks up the MachineConfig dropped in
+  `openshift/` (the documented additional-manifest path; verify it is rendered
+  into the node ignition).
+- The mount unit ordering (`Before=crio.service`, `RequiredBy=crio.service`)
+  makes the store available before CRI-O starts.
+- Rootless `skopeo` overlay + `virt-make-fs` produce a store CRI-O accepts.
+
+Measure install time with and without `--bake-images` on first run (cold) and on
+the second/third cluster of the same version (warm cache) to quantify the win.