From 9b0cc1d93aada196e9b7282b584b11e5ee469734 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Fri, 19 Jun 2026 10:45:34 -0400 Subject: [PATCH 1/4] feat: wire --bake-images through config, interfaces, and stages Add the plumbing for opt-in image baking (--bake-images): the CLI flag, ClusterConfig.BakeImages, the ImageBaker interface + BakeSpec, store path helpers, the bake-image-store stage slot in the manager, ignition hooks (MachineConfig manifest + live-ISO merge), the ExtraDisk VM-spec support with libvirt ImportDisk, and fakes + app-level tests. The concrete baker and stage land in the follow-up commit. Also ignore the local .gstack/ tool-state directory. Signed-off-by: Raghavendra Talur Assisted-by: Claude Code/claude-fable-5 --- .gitignore | 1 + app/cluster_test.go | 62 ++++++++++++++++ app/deps.go | 1 + app/deps_test.go | 4 +- app/manager.go | 2 + cmd/easyshift/main.go | 6 ++ config/config.go | 18 +++++ config/paths.go | 28 ++++++++ docs/README.md | 1 + interfaces/deps.go | 1 + interfaces/interfaces.go | 52 ++++++++++++++ providers/fakes/fakes.go | 119 ++++++++++++++++++++++++++----- providers/libvirt/libvirt.go | 35 +++++++++ providers/openshift/installer.go | 34 +++++++++ stages/createmastervms/stage.go | 46 +++++++++++- stages/embedignitioniso/stage.go | 8 +++ stages/generateignition/stage.go | 9 +++ 17 files changed, 407 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index de0ee0b..5b59733 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ go.work.sum cloudflare-token pull-secret pull-secret*.txt +.gstack/ diff --git a/app/cluster_test.go b/app/cluster_test.go index 3959452..3909679 100644 --- a/app/cluster_test.go +++ b/app/cluster_test.go @@ -92,6 +92,67 @@ func newTestEnv(t *testing.T) (*config.Config, interfaces.Deps, *fakes.Bundle) { return cfg, deps, bundle } +// TestCreateCluster_BakeImages exercises the --bake-images path end-to-end +// with fakes: the bake stage builds the store, the ignition gets the +// MachineConfig + live-ISO merge, and the master gets a read-only store disk. +func TestCreateCluster_BakeImages(t *testing.T) { + cfg, deps, bundle := newTestEnv(t) + mgr := app.NewClusterManager(cfg, deps) + + c := newTestCluster("baked") + c.BakeImages = true + if err := mgr.Create(context.Background(), c); err != nil { + t.Fatalf("Create: %v", err) + } + + if got := len(bundle.ImageBaker.Baked); got != 1 { + t.Fatalf("ImageBaker.Bake calls: got %d want 1", got) + } + if !bundle.Installer.WroteImageStoreManifest { + t.Error("expected WriteImageStoreManifest to be called") + } + // Merging the image store into the *live ISO* ignition is part of the + // Linux boot-media path; macOS serves ignition over HTTP (publish-pxe-assets), + // where the equivalent merge is wired in the on-hardware phase. + if runtime.GOOS != "darwin" && !bundle.Installer.MergedImageStoreIgnition { + t.Error("expected MergeImageStoreIntoLiveISOIgnition to be called") + } + if got := len(bundle.VM.ImportedDisks); got != 1 { + t.Fatalf("ImportDisk calls: got %d want 1", got) + } + // The master VM must carry exactly one read-only, shareable extra disk. + if len(bundle.VM.Created) != 1 { + t.Fatalf("VMs created: got %d want 1", len(bundle.VM.Created)) + } + disks := bundle.VM.Created[0].ExtraDisks + if len(disks) != 1 { + t.Fatalf("master extra disks: got %d want 1", len(disks)) + } + if !disks[0].ReadOnly || !disks[0].Shareable { + t.Errorf("store disk must be read-only + shareable, got %+v", disks[0]) + } +} + +// TestCreateCluster_NoBakeByDefault confirms baking is opt-in: a default +// cluster touches none of the bake machinery. +func TestCreateCluster_NoBakeByDefault(t *testing.T) { + cfg, deps, bundle := newTestEnv(t) + mgr := app.NewClusterManager(cfg, deps) + + if err := mgr.Create(context.Background(), newTestCluster("plain")); err != nil { + t.Fatalf("Create: %v", err) + } + if got := len(bundle.ImageBaker.Baked); got != 0 { + t.Errorf("expected no bakes, got %d", got) + } + if bundle.Installer.WroteImageStoreManifest || bundle.Installer.MergedImageStoreIgnition { + t.Error("bake-store installer methods should not be called without --bake-images") + } + if len(bundle.VM.Created) == 1 && len(bundle.VM.Created[0].ExtraDisks) != 0 { + t.Errorf("master should have no extra disks without --bake-images") + } +} + // TestCreateCluster_HappyPath walks the full stage list with all-fake deps // and asserts each interface saw the expected call. func TestCreateCluster_HappyPath(t *testing.T) { @@ -185,6 +246,7 @@ func TestCreateCluster_HappyPath(t *testing.T) { "ensure-cluster-dir", "download-binaries", "download-rhcos", + "bake-image-store", "generate-ssh-key", "generate-ignition", bootMediaStageName(), diff --git a/app/deps.go b/app/deps.go index d9bd755..9a37b4c 100644 --- a/app/deps.go +++ b/app/deps.go @@ -52,6 +52,7 @@ func baseDeps(cfg *config.Config, hostIP string) (interfaces.Deps, interfaces.Co Cmd: cmd, Download: dl, Installer: openshift.NewOpenShiftInstaller(cmd), + ImageBaker: openshift.NewOpenShiftImageBaker(cmd), Files: files, CSR: csr.NewOCCSRApprover(cmd), Hostname: host.NewSSHHostnameInjector(cmd), diff --git a/app/deps_test.go b/app/deps_test.go index bbfc19b..fae4e48 100644 --- a/app/deps_test.go +++ b/app/deps_test.go @@ -17,8 +17,8 @@ func TestNewDarwinDeps_WiresMacProviders(t *testing.T) { if deps.VM == nil || deps.Net == nil { t.Fatal("darwin deps must wire VM and Net") } - if deps.Installer == nil || deps.Files == nil { - t.Fatal("darwin deps must wire the shared deps (installer, files)") + if deps.Installer == nil || deps.ImageBaker == nil || deps.Files == nil { + t.Fatal("darwin deps must wire the shared deps (installer, image baker, files)") } // The vfkit VMManager treats ImportISO as a no-op (libvirt would shell out). if _, err := deps.VM.ImportISO(context.Background(), "p", "v", "/tmp/x"); err != nil { diff --git a/app/manager.go b/app/manager.go index c1e159a..df32235 100644 --- a/app/manager.go +++ b/app/manager.go @@ -19,6 +19,7 @@ import ( "github.com/TheEasyShift/easyshift/providers/openshift" "github.com/TheEasyShift/easyshift/stages/allocatenetwork" "github.com/TheEasyShift/easyshift/stages/applytlscerts" + "github.com/TheEasyShift/easyshift/stages/bakeimagestore" "github.com/TheEasyShift/easyshift/stages/createmastervms" "github.com/TheEasyShift/easyshift/stages/createnetwork" "github.com/TheEasyShift/easyshift/stages/downloadbinaries" @@ -77,6 +78,7 @@ func (cm *ClusterManager) buildStages() []interfaces.Stage { ensureclusterdir.New(), downloadbinaries.New(d.Download, d.Cmd, d.Host), downloadrhcos.New(d.Installer, d.Download), + bakeimagestore.New(d.ImageBaker, d.Host), generatesshkey.New(d.Cmd, d.Host), generateignition.New(d.Installer, d.DNS, d.Host), bootMedia, diff --git a/cmd/easyshift/main.go b/cmd/easyshift/main.go index c5caa72..146c6a5 100644 --- a/cmd/easyshift/main.go +++ b/cmd/easyshift/main.go @@ -141,6 +141,7 @@ func newCreateCommand(mgr **app.ClusterManager, simBundle **fakes.Bundle, cfgp * tlsEmail string tlsStaging bool magicDNS string + bakeImages bool ) cmd := &cobra.Command{ @@ -170,6 +171,7 @@ func newCreateCommand(mgr **app.ClusterManager, simBundle **fakes.Bundle, cfgp * TLSEmail: tlsEmail, TLSStaging: tlsStaging, MagicDNS: magicDNS, + BakeImages: bakeImages, } // In a bridge-mode simulation there is no real node, so pretend it // came up on its reserved IP — otherwise the verify-master-ip stage @@ -238,6 +240,10 @@ func newCreateCommand(mgr **app.ClusterManager, simBundle **fakes.Bundle, cfgp * "Wildcard DNS service so cluster names resolve to the master IP with no records to manage: "+ "'auto' (NAT -> sslip.io, bridge -> off), 'sslip.io', 'nip.io', or 'off'. "+ "Mutually exclusive with --dns-provider.") + cmd.Flags().BoolVar(&bakeImages, "bake-images", false, + "Pre-pull the entire OCP release payload into a read-only disk attached to the master so "+ + "the install never reaches quay.io for platform images. Built once per version (multi-arch: "+ + "amd64 + aarch64), shared across clusters. Needs skopeo + virt-make-fs on PATH.") _ = cmd.MarkFlagRequired("name") return cmd diff --git a/config/config.go b/config/config.go index 3482673..858ce57 100644 --- a/config/config.go +++ b/config/config.go @@ -88,6 +88,16 @@ const ( ClusterStateRunning = "running" ClusterStateStopped = "stopped" ClusterStateError = "error" + + // BakedImagesLabel is the ext4 filesystem label of the read-only disk that + // carries the pre-pulled release payload (a CRI-O "additional image store"). + // Both the live-ISO ignition and the installed node's MachineConfig mount + // the disk by this label, so device ordering on the VM is irrelevant. + BakedImagesLabel = "baked-images" + // BakedImagesMountPath is where the baked store disk is mounted on the node + // and the path added to CRI-O's additionalimagestores. CRI-O then serves + // release images from here instead of pulling them from quay.io. + BakedImagesMountPath = "/var/lib/baked-images" ) // Config is the global on-disk configuration. There is no singleton; callers @@ -169,6 +179,14 @@ type ClusterConfig struct { // --magic-dns flag's "auto"/"off" by the manager before any stage runs. MagicDNS string `json:"magicDNS,omitempty"` + // BakeImages, when true, pre-pulls the entire OCP release payload into a + // read-only disk attached to the master so the install never reaches + // quay.io for platform images. The store is built once per OCP version, + // shared across clusters, and is multi-arch (every supported release arch), + // so an amd64 or aarch64 (Rosetta) node finds its images locally. See + // docs/dev/image-baking.md. + BakeImages bool `json:"bakeImages,omitempty"` + NetworkSubnet string `json:"networkSubnet"` IPAddresses []string `json:"ipAddresses"` MACAddresses []string `json:"macAddresses"` diff --git a/config/paths.go b/config/paths.go index e81805a..6b5a90d 100644 --- a/config/paths.go +++ b/config/paths.go @@ -31,6 +31,34 @@ func RHCOSCacheDir(configDir, version string) string { return filepath.Join(configDir, "rhcos", version) } +// ImageStoreCacheDir is the per-version cache root for the baked image store. +// It holds the CRI-O overlay container store (built once with skopeo) and the +// packed qcow2 produced from it. Shared across clusters of the same version. +func ImageStoreCacheDir(configDir, version string) string { + return filepath.Join(configDir, "imagestore", version) +} + +// ImageStoreOverlayDir is the CRI-O container storage graphroot that skopeo +// copies release images into, inside the per-version cache. +func ImageStoreOverlayDir(configDir, version string) string { + return filepath.Join(ImageStoreCacheDir(configDir, version), "store") +} + +// ImageStoreQcowPath is the packed read-only qcow2 (an ext4 filesystem labeled +// BakedImagesLabel containing the overlay store) for the given version. Built +// once on the host; a per-cluster copy is uploaded into the libvirt pool and +// attached to the master. +func ImageStoreQcowPath(configDir, version string) string { + return filepath.Join(ImageStoreCacheDir(configDir, version), "store.qcow2") +} + +// ImageStoreVolName is the per-cluster libvirt pool volume name for the +// attached baked-image-store disk. Per-cluster (not shared) so cluster delete +// — which removes all of a domain's storage — never strands another cluster. +func ImageStoreVolName(name string) string { + return "easyshift-" + name + "-imagestore.qcow2" +} + // ClusterDNSNames returns the DNS names a bridge-mode cluster needs, all of // which must resolve to the master IP. The wildcard *.apps is probed via a // synthetic console hostname because a literal "*" lookup isn't valid DNS. diff --git a/docs/README.md b/docs/README.md index 1c1245c..989fe77 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,5 +24,6 @@ A good reading order: **installation → configuration → usage**, then | [architecture.md](dev/architecture.md) | Package layering and the staged-installer model | | [stages.md](dev/stages.md) | The stage contract; adding or changing a stage | | [providers.md](dev/providers.md) | Interfaces, provider implementations, and fakes | +| [image-baking.md](dev/image-baking.md) | `--bake-images`: pre-pulling the release payload into a read-only store | | [testing.md](dev/testing.md) | Make targets, fakes, and `--simulate` | | [contributing.md](dev/contributing.md) | Workflow, commit conventions, PR flow | diff --git a/interfaces/deps.go b/interfaces/deps.go index f1fd904..0fe33b9 100644 --- a/interfaces/deps.go +++ b/interfaces/deps.go @@ -11,6 +11,7 @@ type Deps struct { VM VMManager Net NetworkProvisioner Installer Installer + ImageBaker ImageBaker Files FileServer CSR CSRApprover Hostname HostnameInjector diff --git a/interfaces/interfaces.go b/interfaces/interfaces.go index 11ddb36..8f51720 100644 --- a/interfaces/interfaces.go +++ b/interfaces/interfaces.go @@ -44,6 +44,17 @@ type VMSpec struct { // (--bootloader linux); empty on the libvirt backend. KernelPath string InitrdPath string + // ExtraDisks are additional disks attached after the primary OS disk, in + // order. Used to attach the read-only baked image-store disk. + ExtraDisks []ExtraDisk +} + +// ExtraDisk is a secondary disk attached to a VM by path (a libvirt pool +// volume path). ReadOnly + Shareable suit a store cloned to several VMs. +type ExtraDisk struct { + Path string + ReadOnly bool + Shareable bool } // VMManager abstracts libvirt VM lifecycle plus the storage helpers needed to @@ -59,6 +70,10 @@ type VMManager interface { // ImportISO uploads localPath into the named storage pool as volName and // returns the pool volume path for use as VMSpec.BootISO. ImportISO(ctx context.Context, pool, volName, localPath string) (string, error) + // ImportDisk uploads a qcow2 image at localPath into the named storage pool + // as volName (format qcow2) and returns the pool volume path for use as an + // ExtraDisk. Used for the baked image-store disk. + ImportDisk(ctx context.Context, pool, volName, localPath string) (string, error) // RemoveISO deletes a volume previously created by ImportISO. RemoveISO(ctx context.Context, pool, volName string) error // CheckAccess probes that the libvirt endpoint is reachable (libvirtd up, @@ -173,12 +188,21 @@ type InstallerSpec struct { type Installer interface { WriteInstallConfig(ctx context.Context, spec InstallerSpec) error CreateIgnitionConfigs(ctx context.Context, spec InstallerSpec) error + // WriteImageStoreManifest drops a MachineConfig into the install dir's + // openshift/ so the next ignition render wires CRI-O to the baked image + // store (mounted read-only on the installed node). Must run before + // CreateSingleNodeIgnition, which loads any manifests already on disk. + WriteImageStoreManifest(ctx context.Context, spec InstallerSpec) error CreateSingleNodeIgnition(ctx context.Context, spec InstallerSpec) error // WriteRosettaManifest drops a MachineConfig into the install dir that // mounts the vfkit rosetta virtiofs share and registers the x86-64 // binfmt_misc handler, so the installed node runs amd64 binaries via // Apple Rosetta. Called on macOS hosts before CreateSingleNodeIgnition. WriteRosettaManifest(ctx context.Context, spec InstallerSpec) error + // MergeImageStoreIntoLiveISOIgnition rewrites the live-ISO ignition at + // ignitionPath to mount the baked store and register it with CRI-O, so the + // bootstrap (live-ISO) phase also serves images locally. Operates in place. + MergeImageStoreIntoLiveISOIgnition(ctx context.Context, spec InstallerSpec, ignitionPath string) error EmbedIgnitionInISO(ctx context.Context, spec InstallerSpec, isoPath, ignitionPath, outputPath string) error // EmbedNetworkKeyfileInISO embeds a NetworkManager keyfile into the live // ISO (coreos-installer iso network embed) so the node applies static @@ -203,6 +227,34 @@ type CoreOSLivePXE struct { RootfsURL string } +// BakeSpec carries everything ImageBaker.Bake needs for one OCP version. The +// store is multi-arch: the baker enumerates every supported release arch, so a +// single store serves amd64 and aarch64 (Rosetta) nodes alike. +type BakeSpec struct { + // Version is the resolved OCP version (e.g. "4.21.0"). + Version string + // OCBinaryPath is the `oc` for this version (used for `oc adm release info`). + OCBinaryPath string + // PullSecretPath is the on-disk registry auth file passed to oc + skopeo. + PullSecretPath string + // OverlayDir is the CRI-O graphroot skopeo copies images into. + OverlayDir string + // OutputQcowPath is the packed, labeled qcow2 to produce from OverlayDir. + OutputQcowPath string +} + +// ImageBaker pre-pulls the OCP release payload into a read-only disk image so +// installs serve platform images locally instead of pulling from quay.io. +type ImageBaker interface { + // Ready reports whether the packed store for this spec already exists, so + // the bake stage can skip the (expensive) rebuild on resume. + Ready(spec BakeSpec) (bool, error) + // Bake enumerates the release payload for every supported arch, copies each + // image into the overlay store, and packs it into OutputQcowPath. Must + // tolerate retry after a partial run. + Bake(ctx context.Context, spec BakeSpec) error +} + // FileServer abstracts the HTTP server that hosts ignition + RHCOS files. type FileServer interface { Start(ctx context.Context) error diff --git a/providers/fakes/fakes.go b/providers/fakes/fakes.go index bcbb657..5e7538d 100644 --- a/providers/fakes/fakes.go +++ b/providers/fakes/fakes.go @@ -134,15 +134,16 @@ func (d *Downloader) Download(_ context.Context, url, destPath string) error { // VMManager is a fake interfaces.VMManager. Created VMs are tracked in Created // and considered running until Stop/Delete is called. type VMManager struct { - mu sync.Mutex - Created []interfaces.VMSpec - Started []string - Stopped []string - Deleted []string - ImportedISOs []string // volNames passed to ImportISO - RemovedISOs []string // volNames passed to RemoveISO - running map[string]bool - Err error + mu sync.Mutex + Created []interfaces.VMSpec + Started []string + Stopped []string + Deleted []string + ImportedISOs []string // volNames passed to ImportISO + ImportedDisks []string // volNames passed to ImportDisk + RemovedISOs []string // volNames passed to RemoveISO + running map[string]bool + Err error // CheckAccessErr, if set, is returned by CheckAccess (simulates libvirt // being unreachable). CheckAccessErr error @@ -222,6 +223,17 @@ func (v *VMManager) ImportISO(_ context.Context, _, volName, _ string) (string, return "/var/lib/libvirt/images/" + volName, nil } +// ImportDisk records volName and returns a deterministic fake pool path. +func (v *VMManager) ImportDisk(_ context.Context, _, volName, _ string) (string, error) { + v.mu.Lock() + defer v.mu.Unlock() + v.ImportedDisks = append(v.ImportedDisks, volName) + if v.Err != nil { + return "", v.Err + } + return "/var/lib/libvirt/images/" + volName, nil +} + // RemoveISO records volName. func (v *VMManager) RemoveISO(_ context.Context, _, volName string) error { v.mu.Lock() @@ -307,13 +319,15 @@ func (n *NetworkProvisioner) ResetNetwork(_ context.Context, network string) err // the most recent method call so tests can assert on the resolved binary // paths the stages produced. type Installer struct { - mu sync.Mutex - WroteInstallConfig bool - CreatedIgnitions bool - CreatedSingleNodeIgn bool - WroteRosettaManifest bool - EmbeddedISO bool - EmbeddedNetwork bool + mu sync.Mutex + WroteInstallConfig bool + CreatedIgnitions bool + CreatedSingleNodeIgn bool + WroteRosettaManifest bool + WroteImageStoreManifest bool + MergedImageStoreIgnition bool + EmbeddedISO bool + EmbeddedNetwork bool // LastNetworkKeyfile is the keyfile path passed to the most recent // EmbedNetworkKeyfileInISO call (empty if never called). LastNetworkKeyfile string @@ -357,6 +371,14 @@ func (i *Installer) WriteRosettaManifest(_ context.Context, spec interfaces.Inst return i.Err } +func (i *Installer) WriteImageStoreManifest(_ context.Context, spec interfaces.InstallerSpec) error { + i.mu.Lock() + defer i.mu.Unlock() + i.WroteImageStoreManifest = true + i.record(spec) + return i.Err +} + func (i *Installer) CreateSingleNodeIgnition(_ context.Context, spec interfaces.InstallerSpec) error { i.mu.Lock() defer i.mu.Unlock() @@ -376,6 +398,14 @@ func (i *Installer) CreateSingleNodeIgnition(_ context.Context, spec interfaces. return nil } +func (i *Installer) MergeImageStoreIntoLiveISOIgnition(_ context.Context, spec interfaces.InstallerSpec, _ string) error { + i.mu.Lock() + defer i.mu.Unlock() + i.MergedImageStoreIgnition = true + i.record(spec) + return i.Err +} + func (i *Installer) EmbedIgnitionInISO(_ context.Context, spec interfaces.InstallerSpec, _, _, _ string) error { i.mu.Lock() defer i.mu.Unlock() @@ -437,6 +467,42 @@ func (i *Installer) CoreOSLivePXEURLs(_ context.Context, spec interfaces.Install }, nil } +// ImageBaker is a fake interfaces.ImageBaker. It records bake specs and +// reports the store as not-ready until a Bake has run (so the stage exercises +// the build path), unless ReadyResult is forced. +type ImageBaker struct { + mu sync.Mutex + Baked []interfaces.BakeSpec + ReadyForced *bool // when non-nil, Ready returns this regardless of Baked + Err error +} + +// Ready returns ReadyForced when set, else whether a Bake has been recorded. +func (b *ImageBaker) Ready(spec interfaces.BakeSpec) (bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.ReadyForced != nil { + return *b.ReadyForced, b.Err + } + for _, s := range b.Baked { + if s.OutputQcowPath == spec.OutputQcowPath { + return true, b.Err + } + } + return false, b.Err +} + +// Bake records the spec. +func (b *ImageBaker) Bake(_ context.Context, spec interfaces.BakeSpec) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.Err != nil { + return b.Err + } + b.Baked = append(b.Baked, spec) + return nil +} + // FileServer is a fake interfaces.FileServer. type FileServer struct { mu sync.Mutex @@ -787,6 +853,7 @@ func All() (interfaces.Deps, *Bundle) { VM: &VMManager{}, Net: &NetworkProvisioner{}, Installer: &Installer{}, + ImageBaker: &ImageBaker{}, Files: &FileServer{Root: fakeHTTPRoot(), URL: "http://fake:9393"}, CSR: &CSRApprover{}, Hostname: &HostnameInjector{}, @@ -804,6 +871,7 @@ func All() (interfaces.Deps, *Bundle) { VM: b.VM, Net: b.Net, Installer: b.Installer, + ImageBaker: b.ImageBaker, Files: b.Files, CSR: b.CSR, Hostname: b.Hostname, @@ -834,6 +902,9 @@ func (b *Bundle) WriteTrace(w io.Writer) { for _, v := range b.VM.Created { fmt.Fprintf(w, " %s ram=%dMiB vcpus=%d disk=%dGiB net=%q boot-iso=%q\n", v.Name, v.MemoryMiB, v.VCPUs, v.DiskSizeGiB, v.NetworkArg, v.BootISO) + for _, d := range v.ExtraDisks { + fmt.Fprintf(w, " + extra-disk %s readonly=%t shareable=%t\n", d.Path, d.ReadOnly, d.Shareable) + } } } if len(b.VM.Started) > 0 { @@ -848,6 +919,15 @@ func (b *Bundle) WriteTrace(w io.Writer) { if len(b.VM.ImportedISOs) > 0 { fmt.Fprintf(w, "\nISOs imported to libvirt pool: %v\n", b.VM.ImportedISOs) } + if len(b.VM.ImportedDisks) > 0 { + fmt.Fprintf(w, "\nDisks imported to libvirt pool: %v\n", b.VM.ImportedDisks) + } + if len(b.ImageBaker.Baked) > 0 { + fmt.Fprintf(w, "\nImage stores baked (%d):\n", len(b.ImageBaker.Baked)) + for _, s := range b.ImageBaker.Baked { + fmt.Fprintf(w, " version=%s -> %s\n", s.Version, s.OutputQcowPath) + } + } if len(b.Net.Ensured) > 0 { fmt.Fprintf(w, "\nShared NAT network ensured (%d):\n", len(b.Net.Ensured)) @@ -909,9 +989,15 @@ func (b *Bundle) WriteTrace(w io.Writer) { if b.Installer.WroteRosettaManifest { fmt.Fprintln(w, " - WriteRosettaManifest (macOS amd64-via-Rosetta)") } + if b.Installer.WroteImageStoreManifest { + fmt.Fprintln(w, " - WriteImageStoreManifest (baked image store)") + } if b.Installer.CreatedSingleNodeIgn { fmt.Fprintln(w, " - CreateSingleNodeIgnition") } + if b.Installer.MergedImageStoreIgnition { + fmt.Fprintln(w, " - MergeImageStoreIntoLiveISOIgnition (baked image store)") + } if b.Installer.EmbeddedISO { fmt.Fprintln(w, " - EmbedIgnitionInISO") } @@ -956,6 +1042,7 @@ type Bundle struct { VM *VMManager Net *NetworkProvisioner Installer *Installer + ImageBaker *ImageBaker Files *FileServer CSR *CSRApprover Hostname *HostnameInjector diff --git a/providers/libvirt/libvirt.go b/providers/libvirt/libvirt.go index 0f2fb17..e9427cb 100644 --- a/providers/libvirt/libvirt.go +++ b/providers/libvirt/libvirt.go @@ -71,6 +71,16 @@ func (m *LibvirtVMManager) Create(ctx context.Context, spec interfaces.VMSpec) e "--boot", "hd,cdrom", "--noautoconsole", } + for _, d := range spec.ExtraDisks { + disk := fmt.Sprintf("path=%s,bus=virtio", d.Path) + if d.ReadOnly { + disk += ",readonly=on" + } + if d.Shareable { + disk += ",shareable=on" + } + args = append(args, "--disk", disk) + } if spec.BootISO != "" { args = append(args, "--cdrom", spec.BootISO) } else { @@ -176,6 +186,31 @@ func (m *LibvirtVMManager) ImportISO(ctx context.Context, pool, volName, localPa return strings.TrimSpace(string(out)), nil } +// ImportDisk uploads a qcow2 image into the named storage pool as volName +// (preserving the qcow2 format so its virtual size, not file size, governs the +// volume) and returns the resulting pool volume path. Idempotent: a stale +// volume from a prior attempt is dropped first. Used for the baked image store. +func (m *LibvirtVMManager) ImportDisk(ctx context.Context, pool, volName, localPath string) (string, error) { + fi, err := os.Stat(localPath) + if err != nil { + return "", fmt.Errorf("stat disk %s: %w", localPath, err) + } + _, _ = m.virsh(ctx, "vol-delete", "--pool", pool, volName) + + if _, err := m.virsh(ctx, "vol-create-as", pool, volName, + strconv.FormatInt(fi.Size(), 10), "--format", "qcow2"); err != nil { + return "", fmt.Errorf("vol-create-as %s: %w", volName, err) + } + if _, err := m.virsh(ctx, "vol-upload", "--pool", pool, volName, localPath); err != nil { + return "", fmt.Errorf("vol-upload %s: %w", volName, err) + } + out, err := m.virsh(ctx, "vol-path", "--pool", pool, volName) + if err != nil { + return "", fmt.Errorf("vol-path %s: %w", volName, err) + } + return strings.TrimSpace(string(out)), nil +} + // RemoveISO deletes a volume created by ImportISO. Missing volumes are not // an error (best-effort cleanup during rollback). func (m *LibvirtVMManager) RemoveISO(ctx context.Context, pool, volName string) error { diff --git a/providers/openshift/installer.go b/providers/openshift/installer.go index 54718d8..80b552c 100644 --- a/providers/openshift/installer.go +++ b/providers/openshift/installer.go @@ -145,6 +145,40 @@ func (i *OpenShiftInstaller) WriteRosettaManifest(_ context.Context, spec interf return nil } +// WriteImageStoreManifest drops the baked-store MachineConfig into the install +// dir's openshift/ directory. `create single-node-ignition-config` loads any +// manifests already present there, so the storage.conf drop-in + read-only +// mount unit are rendered into the installed node's ignition (present from +// first boot, before CRI-O pulls the operator payload). +func (i *OpenShiftInstaller) WriteImageStoreManifest(_ context.Context, spec interfaces.InstallerSpec) error { + dir := filepath.Join(spec.ClusterDir, "openshift") + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create openshift manifests dir: %w", err) + } + path := filepath.Join(dir, MachineConfigName+".yaml") + if err := os.WriteFile(path, []byte(RenderMachineConfig()), 0o644); err != nil { + return fmt.Errorf("write image-store MachineConfig: %w", err) + } + return nil +} + +// MergeImageStoreIntoLiveISOIgnition rewrites the live-ISO ignition in place so +// the bootstrap phase mounts the baked store and registers it with CRI-O. +func (i *OpenShiftInstaller) MergeImageStoreIntoLiveISOIgnition(_ context.Context, _ interfaces.InstallerSpec, ignitionPath string) error { + data, err := os.ReadFile(ignitionPath) + if err != nil { + return fmt.Errorf("read live-iso ignition: %w", err) + } + merged, err := MergeBakedStoreIntoIgnition(data) + if err != nil { + return err + } + if err := os.WriteFile(ignitionPath, merged, 0o600); err != nil { + return fmt.Errorf("write merged live-iso ignition: %w", err) + } + return nil +} + // CreateSingleNodeIgnition runs `openshift-install create single-node-ignition-config`. func (i *OpenShiftInstaller) CreateSingleNodeIgnition(ctx context.Context, spec interfaces.InstallerSpec) error { if _, err := i.cmd.Run(ctx, spec.InstallerPath, "create", "single-node-ignition-config", "--dir", spec.ClusterDir); err != nil { diff --git a/stages/createmastervms/stage.go b/stages/createmastervms/stage.go index 6a00c09..e262f35 100644 --- a/stages/createmastervms/stage.go +++ b/stages/createmastervms/stage.go @@ -5,6 +5,7 @@ package createmastervms import ( "context" "fmt" + "os" "runtime" "github.com/TheEasyShift/easyshift/config" @@ -55,9 +56,15 @@ func (s *Stage) Preflight(ctx context.Context, sc *interfaces.StageContext) erro return fmt.Errorf("query disk space at %s: %w", sc.Config.ConfigDir, err) } need := uint64(sc.Cluster.MasterDiskGB) * 1024 * 1024 * 1024 + // Baking attaches a per-cluster copy of the store qcow2; count it. + if sc.Cluster.BakeImages { + if fi, err := os.Stat(config.ImageStoreQcowPath(sc.Config.ConfigDir, sc.Cluster.OCPVersion)); err == nil { + need += uint64(fi.Size()) + } + } if avail < need { - return fmt.Errorf("insufficient disk under %s: have %d GiB, need %d GiB for master disk", - sc.Config.ConfigDir, avail>>30, sc.Cluster.MasterDiskGB) + return fmt.Errorf("insufficient disk under %s: have %d GiB, need %d GiB for master disk%s", + sc.Config.ConfigDir, avail>>30, need>>30, bakeNote(sc.Cluster.BakeImages)) } if sc.Cluster.NetworkMode == config.NetworkModeBridge { br, err := s.host.InspectBridge(sc.Cluster.Bridge) @@ -101,6 +108,14 @@ func (s *Stage) createMasterVM(ctx context.Context, sc *interfaces.StageContext, role := fmt.Sprintf("master-%d", index) vmName := fmt.Sprintf("%s-%s", role, c.Name) mac := macFor(c, role) + var extraDisks []interfaces.ExtraDisk + if c.BakeImages { + disk, err := s.attachImageStore(ctx, sc, vmName) + if err != nil { + return err + } + extraDisks = append(extraDisks, disk) + } spec := interfaces.VMSpec{ Name: vmName, MemoryMiB: c.MasterRAM, @@ -108,6 +123,7 @@ func (s *Stage) createMasterVM(ctx context.Context, sc *interfaces.StageContext, DiskSizeGiB: c.MasterDiskGB, StoragePool: c.StoragePool, MAC: mac, + ExtraDisks: extraDisks, } if runtime.GOOS == "darwin" { // vfkit install phase: direct-kernel boot of the live PXE assets with @@ -123,6 +139,32 @@ func (s *Stage) createMasterVM(ctx context.Context, sc *interfaces.StageContext, return s.vm.Create(ctx, spec) } +// attachImageStore uploads the cached, multi-arch baked store qcow2 into the +// pool as a per-cluster volume (so cluster delete, which removes all of a +// domain's storage, never strands another cluster) and returns it as a +// read-only, shareable extra disk. The node mounts it by label and points +// CRI-O's additionalimagestores at it. +func (s *Stage) attachImageStore(ctx context.Context, sc *interfaces.StageContext, vmName string) (interfaces.ExtraDisk, error) { + // ImportDisk stats the source and returns a clear error if the bake-image- + // store stage never produced it, so no extra guard is needed here (and the + // raw stat would wrongly fail under --simulate, where no real file exists). + cached := config.ImageStoreQcowPath(sc.Config.ConfigDir, sc.Cluster.OCPVersion) + volPath, err := s.vm.ImportDisk(ctx, sc.Cluster.StoragePool, config.ImageStoreVolName(vmName), cached) + if err != nil { + return interfaces.ExtraDisk{}, fmt.Errorf("import baked image store into pool: %w", err) + } + return interfaces.ExtraDisk{Path: volPath, ReadOnly: true, Shareable: true}, nil +} + +// bakeNote annotates the disk-space error when the baked image store inflates +// the requirement, so the number isn't surprising. +func bakeNote(baking bool) string { + if baking { + return " (incl. baked image store)" + } + return "" +} + func macFor(c *config.ClusterConfig, role string) string { for i, mac := range c.MACAddresses { if i < c.MasterCount && role == fmt.Sprintf("master-%d", i) { diff --git a/stages/embedignitioniso/stage.go b/stages/embedignitioniso/stage.go index 6664677..6f9b2cf 100644 --- a/stages/embedignitioniso/stage.go +++ b/stages/embedignitioniso/stage.go @@ -34,6 +34,14 @@ func (s *Stage) Apply(ctx context.Context, sc *interfaces.StageContext) error { srcISO := sc.RHCOSLiveISOPath() ignition := filepath.Join(sc.ClusterDir(), "bootstrap-in-place-for-live-iso.ign") local := sc.MasterISOPath() + // When baking, also wire the live-ISO (bootstrap) phase to the read-only + // store so the temporary control plane serves images locally too. Done on + // the generated ignition before it is embedded into the ISO. + if sc.Cluster.BakeImages { + if err := s.installer.MergeImageStoreIntoLiveISOIgnition(ctx, sc.InstallerSpec(), ignition); err != nil { + return err + } + } if err := s.installer.EmbedIgnitionInISO(ctx, sc.InstallerSpec(), srcISO, ignition, local); err != nil { return err } diff --git a/stages/generateignition/stage.go b/stages/generateignition/stage.go index 3fcd0c9..fb1d54f 100644 --- a/stages/generateignition/stage.go +++ b/stages/generateignition/stage.go @@ -75,6 +75,14 @@ func (s *Stage) Apply(ctx context.Context, sc *interfaces.StageContext) error { return err } } + // Drop the baked-store MachineConfig before rendering ignition, so the + // installed node mounts the store and CRI-O reads it from first boot. The + // manifest must exist before CreateSingleNodeIgnition, which loads it. + if sc.Cluster.BakeImages { + if err := s.installer.WriteImageStoreManifest(ctx, spec); err != nil { + return err + } + } return s.installer.CreateSingleNodeIgnition(ctx, spec) } @@ -86,6 +94,7 @@ func (*Stage) Rollback(_ context.Context, sc *interfaces.StageContext) error { "worker.ign", "bootstrap.ign", "metadata.json", + filepath.Join("openshift", "99-master-baked-image-store.yaml"), } { _ = os.Remove(filepath.Join(sc.ClusterDir(), name)) } From f63e6dda9f88f60e77ff6797f6187f8b7cbf73f4 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Sun, 28 Jun 2026 01:33:55 -0400 Subject: [PATCH 2/4] feat: complete --bake-images with the image baker + stage The wiring for --bake-images already landed (interfaces, deps, stages, config, CLI flag), but the concrete pieces it references were untracked. Add them so the tree builds on its own: - providers/openshift/baker.go: OpenShiftImageBaker. Enumerates the union of every supported release arch's pullspecs (amd64 + aarch64) so a baked store is multi-arch-correct, skopeo copy --all into a rootless overlay store, then virt-make-fs packs it into a labeled ext4 qcow2. Pure renderers for the storage.conf drop-in, mount unit, and master MachineConfig, plus live-ISO ignition merge. - stages/bakeimagestore: the bake-image-store stage. No-op unless the cluster opts in; preflights pull secret + skopeo/virt-make-fs on PATH. - docs/dev/image-baking.md: design + operator notes. Baking stays opt-in (--bake-images, default off). Assisted-by: Claude Code/claude-opus-4-8 Signed-off-by: Raghavendra Talur --- docs/dev/image-baking.md | 87 ++++++++ providers/openshift/baker.go | 331 ++++++++++++++++++++++++++++++ providers/openshift/baker_test.go | 165 +++++++++++++++ stages/bakeimagestore/stage.go | 80 ++++++++ 4 files changed, 663 insertions(+) create mode 100644 docs/dev/image-baking.md create mode 100644 providers/openshift/baker.go create mode 100644 providers/openshift/baker_test.go create mode 100644 stages/bakeimagestore/stage.go diff --git a/docs/dev/image-baking.md b/docs/dev/image-baking.md new file mode 100644 index 0000000..940a1b9 --- /dev/null +++ b/docs/dev/image-baking.md @@ -0,0 +1,87 @@ +# Baked image store (`--bake-images`) + +A fresh single-node OpenShift install pulls the entire release payload — the +release image plus hundreds of component images, several GB — from `quay.io`, +twice: once during the live-ISO **bootstrap** phase and again on the installed +node as the operators roll out **post-pivot**. On a dev box building up to three +clusters that is the dominant chunk of wall-clock and bandwidth. + +`--bake-images` pre-pulls that payload into a read-only disk attached to the +master, so CRI-O serves platform images locally and never reaches `quay.io`. +This is the same mechanism Red Hat's `factory-precaching-cli` uses for Telco/ZTP +factory installs, hand-rolled to fit easyshift's stage pipeline and no-root +contract. + +## How it works + +The store is a CRI-O **additional image store**: a read-only container store +that CRI-O layers underneath its writable one. When the node asks for a release +image, CRI-O finds it locally and skips the pull. Images are stored under their +**original names** (`quay.io/...@sha256:...`), so the digests the release +references match with no `imageDigestMirrors` / ICSP config needed. + +### Build (host side) — `stages/bakeimagestore` + +Built once per OCP version, cached at +`~/.config/easyshift/imagestore//`, shared across clusters (rollback is +a no-op, like the binaries cache). `providers/openshift.OpenShiftImageBaker`: + +1. `oc adm release info --pullspecs -o json ` for **every** + supported arch enumerates the component pullspecs. +2. `skopeo copy --all` copies each into an overlay container store + (`store/`). `--all` keeps every manifest-list entry. +3. `virt-make-fs --type=ext4 --label=baked-images --format=qcow2` packs the + store into `store.qcow2`. `virt-make-fs` runs rootless via libguestfs, so no + root is required. + +Needs `skopeo` and `virt-make-fs` (guestfs-tools / libguestfs-tools) on PATH; +the bake stage preflights both. + +### Multi-arch + +The store is **multi-arch**. `SupportedReleaseArches` (`x86_64`, `aarch64`) are +each enumerated from their arch-specific release image +(`ocp-release:-`) and unioned; an arch whose release image +doesn't exist for the version is skipped. One store therefore serves an amd64 +node, an aarch64 node, and amd64 workloads run on aarch64 via Rosetta. The +RHCOS live-ISO arch is still selected separately (see +`providers/openshift.coreOSArch`) — baking does not change which ISO boots. + +### Attach + wire (node side) + +- `stages/createmastervms` uploads a **per-cluster** copy of `store.qcow2` into + the libvirt pool (`ImportDisk`) and attaches it read-only + shareable. Per + cluster — not shared — so `virsh undefine --remove-all-storage` on delete + never strands another cluster. +- The disk is mounted by label (`/dev/disk/by-label/baked-images` → + `/var/lib/baked-images`) and registered with CRI-O via + `additionalimagestores` in a `storage.conf` drop-in. +- That wiring is applied in **both** install phases: + - **post-pivot:** a master `MachineConfig` dropped into the install dir's + `openshift/` (`Installer.WriteImageStoreManifest`) so it is rendered into + the node's ignition and present from first boot, before CRI-O pulls + operators. + - **bootstrap:** the same file + mount unit merged into + `bootstrap-in-place-for-live-iso.ign` + (`Installer.MergeImageStoreIntoLiveISOIgnition`) before the ISO is embedded. + +Renderers live in `providers/openshift/baker.go` (`RenderStorageConfDropin`, +`RenderMountUnit`, `RenderMachineConfig`, `MergeBakedStoreIntoIgnition`) and are +unit-tested in `baker_test.go`. + +## Verification boundary + +The pipeline, the rendered artifacts, and the `--simulate` trace are covered by +unit + app tests. What needs a **real cluster** to confirm: + +- CRI-O actually resolves release images from the mounted store (no `quay.io` + pull) in both phases. +- `create single-node-ignition-config` picks up the MachineConfig dropped in + `openshift/` (the documented additional-manifest path; verify it is rendered + into the node ignition). +- The mount unit ordering (`Before=crio.service`, `RequiredBy=crio.service`) + makes the store available before CRI-O starts. +- Rootless `skopeo` overlay + `virt-make-fs` produce a store CRI-O accepts. + +Measure install time with and without `--bake-images` on first run (cold) and on +the second/third cluster of the same version (warm cache) to quantify the win. diff --git a/providers/openshift/baker.go b/providers/openshift/baker.go new file mode 100644 index 0000000..d7f80e9 --- /dev/null +++ b/providers/openshift/baker.go @@ -0,0 +1,331 @@ +package openshift + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/TheEasyShift/easyshift/config" + "github.com/TheEasyShift/easyshift/interfaces" +) + +// SupportedReleaseArches are the OCP release architectures easyshift bakes into +// the (multi-arch) image store. A version that doesn't publish one of these is +// skipped at bake time, so a single-arch release still works. amd64 and +// aarch64 cover Apple-silicon hosts running arm64 OCP plus Rosetta-run amd64 +// workloads. Names match the `ocp-release:-` tag suffix. +var SupportedReleaseArches = []string{"x86_64", "aarch64"} + +// releaseImageRepo is the public repository of tagged OCP release images. +const releaseImageRepo = "quay.io/openshift-release-dev/ocp-release" + +// ReleaseImageURL returns the release image ref for an OCP version + arch, +// e.g. quay.io/openshift-release-dev/ocp-release:4.21.0-x86_64. +func ReleaseImageURL(version, arch string) string { + return fmt.Sprintf("%s:%s-%s", releaseImageRepo, version, arch) +} + +// OpenShiftImageBaker implements interfaces.ImageBaker by shelling out to `oc` +// (release enumeration), `skopeo` (copy into a CRI-O overlay store), and +// `virt-make-fs` (pack the store into a labeled qcow2 — rootless via +// libguestfs). It holds no per-bake state; everything comes via BakeSpec. +type OpenShiftImageBaker struct { + cmd interfaces.CommandRunner +} + +// NewOpenShiftImageBaker returns an ImageBaker backed by cmd. +func NewOpenShiftImageBaker(cmd interfaces.CommandRunner) *OpenShiftImageBaker { + return &OpenShiftImageBaker{cmd: cmd} +} + +// Ready reports whether the packed qcow2 already exists and is non-empty, so a +// resumed create skips the multi-GB rebuild. +func (b *OpenShiftImageBaker) Ready(spec interfaces.BakeSpec) (bool, error) { + fi, err := os.Stat(spec.OutputQcowPath) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + return fi.Size() > 0, nil +} + +// Bake enumerates the release payload for every supported arch, copies each +// image into the overlay store under their ORIGINAL names (so the digests the +// release references match with no mirror config needed), then packs the store +// into a labeled qcow2. Idempotent: skopeo skips images already present and the +// qcow2 is rebuilt from the (now-complete) store each run. +func (b *OpenShiftImageBaker) Bake(ctx context.Context, spec interfaces.BakeSpec) error { + images, bakedArches, err := b.enumerate(ctx, spec) + if err != nil { + return err + } + if len(bakedArches) == 0 { + return fmt.Errorf("no OCP release image found for version %s in any supported arch %v", spec.Version, SupportedReleaseArches) + } + + if err := os.MkdirAll(spec.OverlayDir, 0o755); err != nil { + return fmt.Errorf("create overlay store dir: %w", err) + } + runRoot := filepath.Join(filepath.Dir(spec.OverlayDir), "run") + if err := os.MkdirAll(runRoot, 0o755); err != nil { + return fmt.Errorf("create overlay run dir: %w", err) + } + + for _, img := range images { + dst := fmt.Sprintf("containers-storage:[overlay@%s+%s]%s", spec.OverlayDir, runRoot, img) + if _, err := b.cmd.Run(ctx, "skopeo", "copy", + "--all", // copy every manifest-list entry so all arches land in the store + "--authfile", spec.PullSecretPath, + "--retry-times", "3", + "docker://"+img, dst, + ); err != nil { + return fmt.Errorf("skopeo copy %s: %w", img, err) + } + } + + // Pack the store into a fresh labeled ext4 qcow2. virt-make-fs runs rootless + // (libguestfs supermin), so this stays within easyshift's no-root contract. + _ = os.Remove(spec.OutputQcowPath) + if _, err := b.cmd.Run(ctx, "virt-make-fs", + "--type=ext4", + "--label="+config.BakedImagesLabel, + "--format=qcow2", + "--size=+1G", // headroom over the store contents for fs metadata + spec.OverlayDir, spec.OutputQcowPath, + ); err != nil { + return fmt.Errorf("virt-make-fs pack image store: %w", err) + } + return nil +} + +// enumerate returns the de-duplicated union of release payload pullspecs across +// every supported arch whose release image exists, plus the list of arches that +// contributed. Each arch's release image is included alongside its components. +func (b *OpenShiftImageBaker) enumerate(ctx context.Context, spec interfaces.BakeSpec) (images, bakedArches []string, err error) { + seen := map[string]bool{} + add := func(ref string) { + if ref != "" && !seen[ref] { + seen[ref] = true + images = append(images, ref) + } + } + for _, arch := range SupportedReleaseArches { + relImg := ReleaseImageURL(spec.Version, arch) + out, infoErr := b.cmd.Run(ctx, spec.OCBinaryPath, "adm", "release", "info", + relImg, "--registry-config", spec.PullSecretPath, "-o", "json") + if infoErr != nil { + // Treat as "this arch isn't published for this version" and move on; + // a genuine auth/network failure surfaces when no arch succeeds. + continue + } + specs, parseErr := parseReleasePullspecs(out) + if parseErr != nil { + return nil, nil, fmt.Errorf("parse release info for %s: %w", arch, parseErr) + } + add(relImg) + for _, s := range specs { + add(s) + } + bakedArches = append(bakedArches, arch) + } + return images, bakedArches, nil +} + +// releaseInfo is the subset of `oc adm release info -o json` we read: the +// embedded image stream whose tags point at every component pullspec by digest. +type releaseInfo struct { + References struct { + Spec struct { + Tags []struct { + From struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"from"` + } `json:"tags"` + } `json:"spec"` + } `json:"references"` +} + +// parseReleasePullspecs extracts component pullspecs (DockerImage refs) from the +// JSON emitted by `oc adm release info -o json`. +func parseReleasePullspecs(data []byte) ([]string, error) { + var ri releaseInfo + if err := json.Unmarshal(data, &ri); err != nil { + return nil, fmt.Errorf("unmarshal release info: %w", err) + } + var out []string + for _, t := range ri.References.Spec.Tags { + if t.From.Name != "" { + out = append(out, t.From.Name) + } + } + return out, nil +} + +// --- CRI-O wiring renderers (pure) -------------------------------------- + +// storageConfDropinPath is where the additional-image-store drop-in lands. The +// containers/storage library merges *.conf from this dir over storage.conf. +const storageConfDropinPath = "/etc/containers/storage.conf.d/10-baked-images.conf" + +// RenderStorageConfDropin returns the storage.conf drop-in that registers the +// baked store as a read-only CRI-O additional image store. +func RenderStorageConfDropin() string { + return fmt.Sprintf(`[storage] +[storage.options] +additionalimagestores = [ + "%s", +] +`, config.BakedImagesMountPath) +} + +// RenderMountUnit returns the systemd .mount unit that mounts the baked store +// disk read-only before CRI-O starts. Returned name is the systemd-escaped unit +// file name matching BakedImagesMountPath. +func RenderMountUnit() (name, contents string) { + contents = fmt.Sprintf(`[Unit] +Description=Baked OCP image store (read-only CRI-O additional image store) +Before=crio.service +After=local-fs.target + +[Mount] +What=/dev/disk/by-label/%s +Where=%s +Type=ext4 +Options=ro,nofail + +[Install] +RequiredBy=crio.service +`, config.BakedImagesLabel, config.BakedImagesMountPath) + return systemdEscapePath(config.BakedImagesMountPath) + ".mount", contents +} + +// MachineConfigName is the role-scoped MachineConfig that applies the baked +// store wiring to the installed node (post-pivot, the long operator-rollout +// tail). 99- prefix orders it after the rendered base config. +const MachineConfigName = "99-master-baked-image-store" + +// RenderMachineConfig returns the master MachineConfig YAML that ships the +// storage.conf drop-in + mount unit to the installed node. Dropped into the +// install dir's openshift/ so `create single-node-ignition-config` renders it +// into the node's ignition — present from first boot, before CRI-O pulls +// operators. +func RenderMachineConfig() string { + unitName, unitContents := RenderMountUnit() + storageB64 := base64.StdEncoding.EncodeToString([]byte(RenderStorageConfDropin())) + return fmt.Sprintf(`apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: %s +spec: + config: + ignition: + version: 3.2.0 + storage: + files: + - path: %s + mode: 420 + overwrite: true + contents: + source: data:text/plain;base64,%s + systemd: + units: + - name: %s + enabled: true + contents: | +%s +`, MachineConfigName, storageConfDropinPath, storageB64, unitName, indent(unitContents, " ")) +} + +// MergeBakedStoreIntoIgnition adds the storage.conf drop-in file and the mount +// unit to a raw Ignition config (the bootstrap-in-place-for-live-iso.ign), so +// the baked store is also used during the live-ISO bootstrap phase. It edits +// the JSON structurally to preserve whatever the installer emitted. +func MergeBakedStoreIntoIgnition(ignitionJSON []byte) ([]byte, error) { + var cfg map[string]any + if err := json.Unmarshal(ignitionJSON, &cfg); err != nil { + return nil, fmt.Errorf("parse live-iso ignition: %w", err) + } + + storage, _ := cfg["storage"].(map[string]any) + if storage == nil { + storage = map[string]any{} + } + files, _ := storage["files"].([]any) + files = append(files, map[string]any{ + "path": storageConfDropinPath, + "mode": 420, + "overwrite": true, + "contents": map[string]any{ + "source": "data:text/plain;base64," + base64.StdEncoding.EncodeToString([]byte(RenderStorageConfDropin())), + }, + }) + storage["files"] = files + cfg["storage"] = storage + + systemd, _ := cfg["systemd"].(map[string]any) + if systemd == nil { + systemd = map[string]any{} + } + units, _ := systemd["units"].([]any) + unitName, unitContents := RenderMountUnit() + units = append(units, map[string]any{ + "name": unitName, + "enabled": true, + "contents": unitContents, + }) + systemd["units"] = units + cfg["systemd"] = systemd + + out, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("re-marshal live-iso ignition: %w", err) + } + return out, nil +} + +// indent prefixes every non-empty line of s with prefix (for embedding a +// multi-line unit file under a YAML block scalar). +func indent(s, prefix string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i, ln := range lines { + if ln != "" { + lines[i] = prefix + ln + } + } + return strings.Join(lines, "\n") +} + +// systemdEscapePath converts an absolute mount path to the systemd unit-name +// stem (the equivalent of `systemd-escape -p`): leading/trailing slashes +// dropped, "/" → "-", and any byte outside [A-Za-z0-9_.] (notably "-") +// rendered as \xNN. A leading "." becomes \x2e. +func systemdEscapePath(p string) string { + p = strings.Trim(p, "/") + if p == "" { + return "-" + } + var b strings.Builder + for i := 0; i < len(p); i++ { + c := p[i] + switch { + case c == '/': + b.WriteByte('-') + case c == '.' && i == 0: + b.WriteString(`\x2e`) + case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '_': + b.WriteByte(c) + default: + fmt.Fprintf(&b, `\x%02x`, c) + } + } + return b.String() +} diff --git a/providers/openshift/baker_test.go b/providers/openshift/baker_test.go new file mode 100644 index 0000000..fd5a1ff --- /dev/null +++ b/providers/openshift/baker_test.go @@ -0,0 +1,165 @@ +package openshift + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestReleaseImageURL(t *testing.T) { + got := ReleaseImageURL("4.21.0", "x86_64") + want := "quay.io/openshift-release-dev/ocp-release:4.21.0-x86_64" + if got != want { + t.Fatalf("ReleaseImageURL: got %q want %q", got, want) + } + if got := ReleaseImageURL("4.21.0", "aarch64"); !strings.HasSuffix(got, "-aarch64") { + t.Fatalf("aarch64 URL missing arch suffix: %q", got) + } +} + +func TestParseReleasePullspecs(t *testing.T) { + in := []byte(`{ + "references": { + "spec": { + "tags": [ + {"name": "cli", "from": {"kind": "DockerImage", "name": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:aaa"}}, + {"name": "etcd", "from": {"kind": "DockerImage", "name": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bbb"}}, + {"name": "broken", "from": {"kind": "DockerImage", "name": ""}} + ] + } + } + }`) + got, err := parseReleasePullspecs(in) + if err != nil { + t.Fatalf("parseReleasePullspecs: %v", err) + } + if len(got) != 2 { + t.Fatalf("want 2 pullspecs (empty skipped), got %d: %v", len(got), got) + } + if !strings.HasSuffix(got[0], "sha256:aaa") || !strings.HasSuffix(got[1], "sha256:bbb") { + t.Fatalf("unexpected pullspecs: %v", got) + } +} + +func TestSystemdEscapePath(t *testing.T) { + // Matches `systemd-escape -p /var/lib/baked-images`: '/' -> '-', '-' -> \x2d. + if got, want := systemdEscapePath("/var/lib/baked-images"), `var-lib-baked\x2dimages`; got != want { + t.Fatalf("systemdEscapePath: got %q want %q", got, want) + } + if got := systemdEscapePath("/"); got != "-" { + t.Fatalf("root path: got %q want %q", got, "-") + } +} + +func TestRenderMountUnit(t *testing.T) { + name, contents := RenderMountUnit() + if name != `var-lib-baked\x2dimages.mount` { + t.Fatalf("mount unit name: got %q", name) + } + for _, want := range []string{ + "What=/dev/disk/by-label/baked-images", + "Where=/var/lib/baked-images", + "Options=ro,nofail", + "Before=crio.service", + "RequiredBy=crio.service", + } { + if !strings.Contains(contents, want) { + t.Errorf("mount unit missing %q\n%s", want, contents) + } + } +} + +func TestRenderStorageConfDropin(t *testing.T) { + got := RenderStorageConfDropin() + if !strings.Contains(got, "additionalimagestores") || !strings.Contains(got, "/var/lib/baked-images") { + t.Fatalf("storage.conf drop-in missing key/path:\n%s", got) + } +} + +func TestRenderMachineConfig(t *testing.T) { + mc := RenderMachineConfig() + for _, want := range []string{ + "kind: MachineConfig", + "machineconfiguration.openshift.io/role: master", + MachineConfigName, + "/etc/containers/storage.conf.d/10-baked-images.conf", + "data:text/plain;base64,", + `var-lib-baked\x2dimages.mount`, + } { + if !strings.Contains(mc, want) { + t.Errorf("MachineConfig missing %q", want) + } + } +} + +func TestMergeBakedStoreIntoIgnition(t *testing.T) { + // A minimal ignition with a pre-existing file + unit, to prove we append. + base := []byte(`{ + "ignition": {"version": "3.2.0"}, + "storage": {"files": [{"path": "/etc/existing"}]}, + "systemd": {"units": [{"name": "existing.service"}]} + }`) + out, err := MergeBakedStoreIntoIgnition(base) + if err != nil { + t.Fatalf("merge: %v", err) + } + var cfg struct { + Storage struct { + Files []struct { + Path string `json:"path"` + Contents struct { + Source string `json:"source"` + } `json:"contents"` + } `json:"files"` + } `json:"storage"` + Systemd struct { + Units []struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + } `json:"units"` + } `json:"systemd"` + } + if err := json.Unmarshal(out, &cfg); err != nil { + t.Fatalf("unmarshal merged: %v\n%s", err, out) + } + if len(cfg.Storage.Files) != 2 { + t.Fatalf("want 2 files (existing + dropin), got %d", len(cfg.Storage.Files)) + } + if len(cfg.Systemd.Units) != 2 { + t.Fatalf("want 2 units (existing + mount), got %d", len(cfg.Systemd.Units)) + } + var foundDropin, foundMount bool + for _, f := range cfg.Storage.Files { + if f.Path == storageConfDropinPath { + foundDropin = true + if !strings.HasPrefix(f.Contents.Source, "data:text/plain;base64,") { + t.Errorf("dropin source not a data URL: %q", f.Contents.Source) + } + } + } + for _, u := range cfg.Systemd.Units { + if u.Name == `var-lib-baked\x2dimages.mount` { + foundMount = true + if !u.Enabled { + t.Errorf("mount unit not enabled") + } + } + } + if !foundDropin || !foundMount { + t.Fatalf("merged ignition missing dropin=%t mount=%t", foundDropin, foundMount) + } +} + +func TestMergeBakedStoreIntoIgnition_EmptyConfig(t *testing.T) { + // No storage/systemd keys present: merge must create them. + out, err := MergeBakedStoreIntoIgnition([]byte(`{"ignition":{"version":"3.2.0"}}`)) + if err != nil { + t.Fatalf("merge: %v", err) + } + // The unit name's backslash is JSON-escaped in raw output; assert on the + // dropin path and the mount unit body, which survive verbatim. + if !strings.Contains(string(out), storageConfDropinPath) || + !strings.Contains(string(out), "Where=/var/lib/baked-images") { + t.Fatalf("merge into empty config dropped entries:\n%s", out) + } +} diff --git a/stages/bakeimagestore/stage.go b/stages/bakeimagestore/stage.go new file mode 100644 index 0000000..3440624 --- /dev/null +++ b/stages/bakeimagestore/stage.go @@ -0,0 +1,80 @@ +// Package bakeimagestore pre-pulls the OCP release payload into a read-only, +// multi-arch disk image so the install serves platform images locally instead +// of pulling them from quay.io. The store is built once per OCP version and +// shared across clusters; this stage is a no-op unless the cluster opted in +// with BakeImages. +package bakeimagestore + +import ( + "context" + "fmt" + "os" + + "github.com/TheEasyShift/easyshift/config" + "github.com/TheEasyShift/easyshift/interfaces" +) + +// Stage builds the per-version baked image store. +type Stage struct { + baker interfaces.ImageBaker + host interfaces.HostInspector +} + +// New returns the bake-image-store stage. +func New(baker interfaces.ImageBaker, host interfaces.HostInspector) *Stage { + return &Stage{baker: baker, host: host} +} + +func (*Stage) Name() string { return "bake-image-store" } + +// Preflight checks the bake tooling is present (only when baking is enabled and +// the store isn't already built). skopeo copies images into a rootless CRI-O +// overlay store; virt-make-fs (guestfs-tools) packs it into a labeled qcow2. +func (s *Stage) Preflight(_ context.Context, sc *interfaces.StageContext) error { + if !sc.Cluster.BakeImages { + return nil + } + if ready, err := s.baker.Ready(s.spec(sc)); err == nil && ready { + return nil + } + if err := config.ValidatePullSecretJSON(sc.Config.ConfigDir); err != nil { + return err + } + for _, tool := range []string{"skopeo", "virt-make-fs"} { + if err := s.host.LookPath(tool); err != nil { + return fmt.Errorf("--bake-images needs %q on PATH: %w\n hint: install skopeo and guestfs-tools (Fedora/RHEL) or skopeo + libguestfs-tools (Debian/Ubuntu)", tool, err) + } + } + return nil +} + +func (s *Stage) Apply(ctx context.Context, sc *interfaces.StageContext) error { + if !sc.Cluster.BakeImages { + return nil + } + spec := s.spec(sc) + if ready, err := s.baker.Ready(spec); err != nil { + return fmt.Errorf("probe baked image store: %w", err) + } else if ready { + return nil + } + if err := os.MkdirAll(config.ImageStoreCacheDir(sc.Config.ConfigDir, sc.Cluster.OCPVersion), 0o755); err != nil { + return err + } + return s.baker.Bake(ctx, spec) +} + +// Rollback is a no-op: the store is a per-version cache shared across clusters, +// like the binaries cache. Deleting one cluster must not evict it. +func (*Stage) Rollback(_ context.Context, _ *interfaces.StageContext) error { return nil } + +func (s *Stage) spec(sc *interfaces.StageContext) interfaces.BakeSpec { + cfgDir, version := sc.Config.ConfigDir, sc.Cluster.OCPVersion + return interfaces.BakeSpec{ + Version: version, + OCBinaryPath: sc.OCBinaryPath(), + PullSecretPath: config.PullSecretPath(cfgDir), + OverlayDir: config.ImageStoreOverlayDir(cfgDir, version), + OutputQcowPath: config.ImageStoreQcowPath(cfgDir, version), + } +} From acbc49a75e83d1b66fb96353986503fc1c129ca5 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 25 Aug 2026 01:33:04 -0400 Subject: [PATCH 3/4] macos: complete --bake-images on the vfkit backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bake feature deferred its macOS half to the on-hardware phase. Wire it up end to end: - publish-pxe-assets merges the baked-store mount + CRI-O drop-in into the HTTP-served bootstrap ignition — the macOS equivalent of embed-ignition-iso's live-ISO merge on Linux. - The baker packs with mke2fs -d on darwin (no libguestfs on macOS; Homebrew e2fsprogs, keg paths probed) into a raw image — vfkit's virtio-blk takes raw, not qcow2 — sized from the overlay contents. ImageStoreDiskPath/ImageStoreVolName pick the extension per OS. - vfkit ImportDisk clones the cached store per cluster (APFS clonefile, plain-copy fallback) into the state dir; ExtraDisks attach after the primary disk in both boot phases; Delete removes the clone. vfkit has no read-only virtio-blk, so the per-cluster copy is the isolation. - bake-image-store preflight requires skopeo + mke2fs on darwin with a brew hint (skopeo + virt-make-fs unchanged on Linux). Validated on hardware: mke2fs -d pack produces a clean labeled ext4 image; the full darwin pipeline runs under --simulate. A real baked install needs ~40+ GB free disk — tracked in ROADMAP.md. Assisted-by: Claude Code/claude-fable-5 Signed-off-by: Raghavendra Talur --- app/cluster_test.go | 8 +-- app/manager.go | 2 +- config/paths.go | 38 +++++++++--- interfaces/interfaces.go | 7 ++- providers/fakes/fakes.go | 4 +- providers/openshift/baker.go | 85 +++++++++++++++++++++++---- providers/openshift/baker_test.go | 24 ++++++++ providers/vfkit/vfkit.go | 70 ++++++++++++++++++---- providers/vfkit/vfkit_test.go | 53 ++++++++++++++++- stages/bakeimagestore/stage.go | 41 +++++++++++-- stages/createmastervms/stage.go | 4 +- stages/publishpxeassets/stage.go | 23 ++++++-- stages/publishpxeassets/stage_test.go | 57 +++++++++++++++++- 13 files changed, 362 insertions(+), 54 deletions(-) diff --git a/app/cluster_test.go b/app/cluster_test.go index 3909679..f466409 100644 --- a/app/cluster_test.go +++ b/app/cluster_test.go @@ -111,10 +111,10 @@ func TestCreateCluster_BakeImages(t *testing.T) { if !bundle.Installer.WroteImageStoreManifest { t.Error("expected WriteImageStoreManifest to be called") } - // Merging the image store into the *live ISO* ignition is part of the - // Linux boot-media path; macOS serves ignition over HTTP (publish-pxe-assets), - // where the equivalent merge is wired in the on-hardware phase. - if runtime.GOOS != "darwin" && !bundle.Installer.MergedImageStoreIgnition { + // Both boot-media paths wire the store into the bootstrap ignition: Linux + // merges into the live-ISO ignition (embed-ignition-iso), macOS into the + // HTTP-served ignition (publish-pxe-assets). + if !bundle.Installer.MergedImageStoreIgnition { t.Error("expected MergeImageStoreIntoLiveISOIgnition to be called") } if got := len(bundle.VM.ImportedDisks); got != 1 { diff --git a/app/manager.go b/app/manager.go index df32235..87a8984 100644 --- a/app/manager.go +++ b/app/manager.go @@ -69,7 +69,7 @@ func (cm *ClusterManager) buildStages() []interfaces.Stage { // (kernel/initrd/rootfs + ignition over HTTP) for vfkit's Linux bootloader. bootMedia := interfaces.Stage(embedignitioniso.New(d.Installer, d.VM)) if runtime.GOOS == "darwin" { - bootMedia = publishpxeassets.New(d.Files) + bootMedia = publishpxeassets.New(d.Files, d.Installer) } return []interfaces.Stage{ registercluster.New(), diff --git a/config/paths.go b/config/paths.go index 6b5a90d..27eb939 100644 --- a/config/paths.go +++ b/config/paths.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" ) @@ -44,19 +45,38 @@ func ImageStoreOverlayDir(configDir, version string) string { return filepath.Join(ImageStoreCacheDir(configDir, version), "store") } -// ImageStoreQcowPath is the packed read-only qcow2 (an ext4 filesystem labeled +// ImageStoreDiskPath is the packed read-only disk (an ext4 filesystem labeled // BakedImagesLabel containing the overlay store) for the given version. Built -// once on the host; a per-cluster copy is uploaded into the libvirt pool and -// attached to the master. -func ImageStoreQcowPath(configDir, version string) string { - return filepath.Join(ImageStoreCacheDir(configDir, version), "store.qcow2") +// once on the host; a per-cluster copy is attached to the master. Linux packs +// a qcow2 (libvirt vol-upload); macOS packs a raw image (vfkit's virtio-blk +// takes raw), hence the per-OS extension. +func ImageStoreDiskPath(configDir, version string) string { + return filepath.Join(ImageStoreCacheDir(configDir, version), "store."+imageStoreDiskExt()) } -// ImageStoreVolName is the per-cluster libvirt pool volume name for the -// attached baked-image-store disk. Per-cluster (not shared) so cluster delete -// — which removes all of a domain's storage — never strands another cluster. +// ImageStoreVolName is the per-cluster name for the attached baked-image-store +// disk: a libvirt pool volume on Linux, a file in the vfkit state dir on +// macOS. Per-cluster (not shared) so cluster delete — which removes all of a +// domain's storage — never strands another cluster. func ImageStoreVolName(name string) string { - return "easyshift-" + name + "-imagestore.qcow2" + return "easyshift-" + name + "-imagestore." + imageStoreDiskExt() +} + +func imageStoreDiskExt() string { + if runtime.GOOS == "darwin" { + return "img" + } + return "qcow2" +} + +// MKE2FSCandidates are the locations probed for mke2fs on macOS (used to pack +// the baked image store; Homebrew's e2fsprogs is keg-only, so its sbin is +// normally not on PATH). Bare names are resolved via PATH, absolute paths via +// stat. +var MKE2FSCandidates = []string{ + "mke2fs", + "/opt/homebrew/opt/e2fsprogs/sbin/mke2fs", + "/usr/local/opt/e2fsprogs/sbin/mke2fs", } // ClusterDNSNames returns the DNS names a bridge-mode cluster needs, all of diff --git a/interfaces/interfaces.go b/interfaces/interfaces.go index 8f51720..083fccf 100644 --- a/interfaces/interfaces.go +++ b/interfaces/interfaces.go @@ -239,8 +239,9 @@ type BakeSpec struct { PullSecretPath string // OverlayDir is the CRI-O graphroot skopeo copies images into. OverlayDir string - // OutputQcowPath is the packed, labeled qcow2 to produce from OverlayDir. - OutputQcowPath string + // OutputDiskPath is the packed, labeled disk image to produce from + // OverlayDir (qcow2 on Linux, raw on macOS — see ImageStoreDiskPath). + OutputDiskPath string } // ImageBaker pre-pulls the OCP release payload into a read-only disk image so @@ -250,7 +251,7 @@ type ImageBaker interface { // the bake stage can skip the (expensive) rebuild on resume. Ready(spec BakeSpec) (bool, error) // Bake enumerates the release payload for every supported arch, copies each - // image into the overlay store, and packs it into OutputQcowPath. Must + // image into the overlay store, and packs it into OutputDiskPath. Must // tolerate retry after a partial run. Bake(ctx context.Context, spec BakeSpec) error } diff --git a/providers/fakes/fakes.go b/providers/fakes/fakes.go index 5e7538d..b7c6c4e 100644 --- a/providers/fakes/fakes.go +++ b/providers/fakes/fakes.go @@ -485,7 +485,7 @@ func (b *ImageBaker) Ready(spec interfaces.BakeSpec) (bool, error) { return *b.ReadyForced, b.Err } for _, s := range b.Baked { - if s.OutputQcowPath == spec.OutputQcowPath { + if s.OutputDiskPath == spec.OutputDiskPath { return true, b.Err } } @@ -925,7 +925,7 @@ func (b *Bundle) WriteTrace(w io.Writer) { if len(b.ImageBaker.Baked) > 0 { fmt.Fprintf(w, "\nImage stores baked (%d):\n", len(b.ImageBaker.Baked)) for _, s := range b.ImageBaker.Baked { - fmt.Fprintf(w, " version=%s -> %s\n", s.Version, s.OutputQcowPath) + fmt.Fprintf(w, " version=%s -> %s\n", s.Version, s.OutputDiskPath) } } diff --git a/providers/openshift/baker.go b/providers/openshift/baker.go index d7f80e9..617aa76 100644 --- a/providers/openshift/baker.go +++ b/providers/openshift/baker.go @@ -5,8 +5,11 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io/fs" "os" + "os/exec" "path/filepath" + "runtime" "strings" "github.com/TheEasyShift/easyshift/config" @@ -45,7 +48,7 @@ func NewOpenShiftImageBaker(cmd interfaces.CommandRunner) *OpenShiftImageBaker { // Ready reports whether the packed qcow2 already exists and is non-empty, so a // resumed create skips the multi-GB rebuild. func (b *OpenShiftImageBaker) Ready(spec interfaces.BakeSpec) (bool, error) { - fi, err := os.Stat(spec.OutputQcowPath) + fi, err := os.Stat(spec.OutputDiskPath) if os.IsNotExist(err) { return false, nil } @@ -89,19 +92,81 @@ func (b *OpenShiftImageBaker) Bake(ctx context.Context, spec interfaces.BakeSpec } } - // Pack the store into a fresh labeled ext4 qcow2. virt-make-fs runs rootless - // (libguestfs supermin), so this stays within easyshift's no-root contract. - _ = os.Remove(spec.OutputQcowPath) - if _, err := b.cmd.Run(ctx, "virt-make-fs", + // Pack the store into a fresh labeled ext4 disk image. Both packers run + // rootless, staying within easyshift's no-root contract. + _ = os.Remove(spec.OutputDiskPath) + sizeMiB, err := packSizeMiB(spec.OverlayDir) + if err != nil { + return fmt.Errorf("size image store: %w", err) + } + tool, args := PackCommand(runtime.GOOS, spec.OverlayDir, spec.OutputDiskPath, sizeMiB) + if _, err := b.cmd.Run(ctx, tool, args...); err != nil { + return fmt.Errorf("%s pack image store: %w", tool, err) + } + return nil +} + +// PackCommand returns the command that packs overlayDir into a labeled ext4 +// disk image at outPath. Linux uses virt-make-fs (libguestfs supermin, qcow2 +// for the libvirt pool) and sizes the fs itself; macOS has no libguestfs, so +// mke2fs -d populates a raw image (vfkit's virtio-blk takes raw) at an +// explicit size of sizeMiB. +func PackCommand(goos, overlayDir, outPath string, sizeMiB int64) (string, []string) { + if goos == "darwin" { + return ResolveMke2fs(), []string{ + "-q", + "-t", "ext4", + "-L", config.BakedImagesLabel, + "-d", overlayDir, + "-m", "0", // no root-reserved blocks on a read-only store + outPath, + fmt.Sprintf("%dm", sizeMiB), + } + } + return "virt-make-fs", []string{ "--type=ext4", - "--label="+config.BakedImagesLabel, + "--label=" + config.BakedImagesLabel, "--format=qcow2", "--size=+1G", // headroom over the store contents for fs metadata - spec.OverlayDir, spec.OutputQcowPath, - ); err != nil { - return fmt.Errorf("virt-make-fs pack image store: %w", err) + overlayDir, outPath, } - return nil +} + +// ResolveMke2fs returns the first usable mke2fs candidate (Homebrew's +// e2fsprogs is keg-only, so its sbin is normally off PATH). Falls back to the +// bare name so the eventual exec error names the missing tool. +func ResolveMke2fs() string { + for _, c := range config.MKE2FSCandidates { + if strings.Contains(c, "/") { + if _, err := os.Stat(c); err == nil { + return c + } + continue + } + if p, err := exec.LookPath(c); err == nil { + return p + } + } + return "mke2fs" +} + +// packSizeMiB walks dir and returns its content size plus fs-metadata headroom +// (10% + 1 GiB), in MiB — the explicit size mke2fs needs. +func packSizeMiB(dir string) (int64, error) { + var bytes int64 + err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if info, err := d.Info(); err == nil && d.Type().IsRegular() { + bytes += info.Size() + } + return nil + }) + if err != nil { + return 0, err + } + return (bytes+bytes/10)/(1024*1024) + 1024, nil } // enumerate returns the de-duplicated union of release payload pullspecs across diff --git a/providers/openshift/baker_test.go b/providers/openshift/baker_test.go index fd5a1ff..a727b11 100644 --- a/providers/openshift/baker_test.go +++ b/providers/openshift/baker_test.go @@ -163,3 +163,27 @@ func TestMergeBakedStoreIntoIgnition_EmptyConfig(t *testing.T) { t.Fatalf("merge into empty config dropped entries:\n%s", out) } } + +func TestPackCommand(t *testing.T) { + name, args := PackCommand("linux", "/cache/store", "/cache/store.qcow2", 4096) + if name != "virt-make-fs" { + t.Errorf("linux pack tool: got %q want virt-make-fs", name) + } + joined := strings.Join(args, " ") + for _, want := range []string{"--type=ext4", "--label=baked-images", "--format=qcow2", "/cache/store", "/cache/store.qcow2"} { + if !strings.Contains(joined, want) { + t.Errorf("linux pack args missing %q: %v", want, args) + } + } + + name, args = PackCommand("darwin", "/cache/store", "/cache/store.img", 4096) + if !strings.Contains(name, "mke2fs") { + t.Errorf("darwin pack tool: got %q want mke2fs", name) + } + joined = strings.Join(args, " ") + for _, want := range []string{"-t ext4", "-L baked-images", "-d /cache/store", "/cache/store.img", "4096m"} { + if !strings.Contains(joined, want) { + t.Errorf("darwin pack args missing %q: %v", want, args) + } + } +} diff --git a/providers/vfkit/vfkit.go b/providers/vfkit/vfkit.go index 2a49efd..61c48a7 100644 --- a/providers/vfkit/vfkit.go +++ b/providers/vfkit/vfkit.go @@ -22,6 +22,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -33,6 +34,7 @@ import ( "github.com/sirupsen/logrus" + "github.com/TheEasyShift/easyshift/config" "github.com/TheEasyShift/easyshift/interfaces" ) @@ -289,11 +291,21 @@ func (m *VMManager) buildArgs(name string, ls launchSpec, phase string) []string "--cpus", strconv.Itoa(s.VCPUs), "--memory", strconv.Itoa(s.MemoryMiB), "--device", "virtio-blk,path=" + ls.DiskPath, - "--device", "virtio-net,unixSocketPath=" + m.sockPath(name) + ",mac=" + s.MAC, + } + // Extra disks (the baked image store) attach after the primary disk in both + // phases: the live install mounts the store by label during bootstrap, the + // run phase for the node's lifetime. vfkit's virtio-blk has no read-only + // option; the guest mounts ro and each cluster gets its own (APFS-cloned) + // copy, so a stray write can't corrupt another cluster's store. + for _, d := range s.ExtraDisks { + args = append(args, "--device", "virtio-blk,path="+d.Path) + } + args = append(args, + "--device", "virtio-net,unixSocketPath="+m.sockPath(name)+",mac="+s.MAC, "--device", "rosetta,mountTag=rosetta", - "--device", "virtio-serial,logFilePath=" + m.consolePath(name), + "--device", "virtio-serial,logFilePath="+m.consolePath(name), "--pidfile", m.pidPath(name), - } + ) switch phase { case phaseRun: args = append(args, "--bootloader", "efi,variable-store="+m.efiPath(name)+",create") @@ -340,21 +352,59 @@ func (m *VMManager) Stop(_ context.Context, name string) error { return nil } -// Delete stops the VM and removes its state dir. +// Delete stops the VM and removes its state dir, plus the imported +// image-store copy (which lives beside — not inside — the VM dir because +// ImportDisk runs before Create). func (m *VMManager) Delete(ctx context.Context, name string) error { _ = m.Stop(ctx, name) + _ = os.Remove(filepath.Join(m.stateDir, config.ImageStoreVolName(name))) return os.RemoveAll(m.vmDir(name)) } // CheckAccess: vfkit presence is verified in preflight via LookPath. func (m *VMManager) CheckAccess(_ context.Context) error { return nil } -// ImportISO / ImportDisk / RemoveISO / StoragePoolActive are libvirt -// storage-pool concepts with no vfkit analog (boot uses PXE assets over HTTP). -func (m *VMManager) ImportISO(_ context.Context, _, _, _ string) (string, error) { return "", nil } -func (m *VMManager) ImportDisk(_ context.Context, _, _, _ string) (string, error) { return "", nil } -func (m *VMManager) RemoveISO(_ context.Context, _, _ string) error { return nil } -func (m *VMManager) StoragePoolActive(_ context.Context, _ string) error { return nil } +// ImportISO / RemoveISO / StoragePoolActive are libvirt storage-pool concepts +// with no vfkit analog (boot uses PXE assets over HTTP). +func (m *VMManager) ImportISO(_ context.Context, _, _, _ string) (string, error) { return "", nil } +func (m *VMManager) RemoveISO(_ context.Context, _, _ string) error { return nil } +func (m *VMManager) StoragePoolActive(_ context.Context, _ string) error { return nil } + +// ImportDisk copies localPath into the state dir as volName ("pool" has no +// vfkit meaning) and returns that path for use as an ExtraDisk. APFS clonefile +// (cp -c) makes the per-cluster copy instant and space-free until modified; a +// plain copy is the fallback for non-APFS volumes. The copy is removed by +// Delete of the VM whose name volName embeds (config.ImageStoreVolName). +func (m *VMManager) ImportDisk(_ context.Context, _, volName, localPath string) (string, error) { + if _, err := os.Stat(localPath); err != nil { + return "", fmt.Errorf("vfkit: import disk: source %s: %w (did the bake-image-store stage run?)", localPath, err) + } + dst := filepath.Join(m.stateDir, volName) + _ = os.Remove(dst) + if out, err := exec.Command("cp", "-c", localPath, dst).CombinedOutput(); err != nil { + if copyErr := copyFile(localPath, dst); copyErr != nil { + return "", fmt.Errorf("vfkit: import disk: clone failed (%v: %s) and copy failed: %w", err, strings.TrimSpace(string(out)), copyErr) + } + } + return dst, nil +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { _ = out.Close() }() + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Close() +} // createDisk creates a sparse raw disk image of sizeGiB if it doesn't exist. func (m *VMManager) createDisk(path string, sizeGiB int) error { diff --git a/providers/vfkit/vfkit_test.go b/providers/vfkit/vfkit_test.go index 35aef4f..c1a9d05 100644 --- a/providers/vfkit/vfkit_test.go +++ b/providers/vfkit/vfkit_test.go @@ -2,9 +2,12 @@ package vfkit import ( "context" + "os" + "path/filepath" "strings" "testing" + "github.com/TheEasyShift/easyshift/config" "github.com/TheEasyShift/easyshift/interfaces" ) @@ -129,10 +132,54 @@ func TestISONoops(t *testing.T) { if _, err := m.ImportISO(context.Background(), "p", "v", "/tmp/x"); err != nil { t.Errorf("ImportISO no-op: %v", err) } - if _, err := m.ImportDisk(context.Background(), "p", "v", "/tmp/x"); err != nil { - t.Errorf("ImportDisk no-op: %v", err) - } if err := m.StoragePoolActive(context.Background(), "p"); err != nil { t.Errorf("StoragePoolActive no-op: %v", err) } } + +func TestBuildArgs_ExtraDisks(t *testing.T) { + m := newMgr(t) + ls := installSpec() + ls.DiskPath = "/d/disk.img" + ls.Spec.ExtraDisks = []interfaces.ExtraDisk{{Path: "/cache/store.img", ReadOnly: true, Shareable: true}} + for _, phase := range []string{phaseInstall, phaseRun} { + args := m.buildArgs("master-0-demo", ls, phase) + joined := strings.Join(args, " ") + if !strings.Contains(joined, "virtio-blk,path=/cache/store.img") { + t.Errorf("%s phase: extra disk not attached: %v", phase, args) + } + // The store must come after the primary OS disk so the guest's + // by-label mount is unambiguous and /dev/vda stays the install disk. + if strings.Index(joined, "path=/d/disk.img") > strings.Index(joined, "path=/cache/store.img") { + t.Errorf("%s phase: extra disk attached before primary: %v", phase, args) + } + } +} + +func TestImportDiskAndDeleteCleanup(t *testing.T) { + m := newMgr(t) + src := filepath.Join(t.TempDir(), "store.img") + if err := os.WriteFile(src, []byte("STORE"), 0o644); err != nil { + t.Fatal(err) + } + vol := config.ImageStoreVolName("master-0-demo") + got, err := m.ImportDisk(context.Background(), "ignored-pool", vol, src) + if err != nil { + t.Fatalf("ImportDisk: %v", err) + } + data, err := os.ReadFile(got) + if err != nil || string(data) != "STORE" { + t.Fatalf("imported disk unreadable at %q: %v", got, err) + } + // Missing source must error clearly (the bake stage never produced it). + if _, err := m.ImportDisk(context.Background(), "p", vol, src+".missing"); err == nil { + t.Error("ImportDisk with missing source: expected error") + } + // Delete must remove the imported store copy along with the VM dir. + if err := m.Delete(context.Background(), "master-0-demo"); err != nil { + t.Fatalf("Delete: %v", err) + } + if _, err := os.Stat(got); !os.IsNotExist(err) { + t.Errorf("imported disk %q survived Delete", got) + } +} diff --git a/stages/bakeimagestore/stage.go b/stages/bakeimagestore/stage.go index 3440624..3ce9d00 100644 --- a/stages/bakeimagestore/stage.go +++ b/stages/bakeimagestore/stage.go @@ -9,6 +9,8 @@ import ( "context" "fmt" "os" + "runtime" + "strings" "github.com/TheEasyShift/easyshift/config" "github.com/TheEasyShift/easyshift/interfaces" @@ -40,14 +42,45 @@ func (s *Stage) Preflight(_ context.Context, sc *interfaces.StageContext) error if err := config.ValidatePullSecretJSON(sc.Config.ConfigDir); err != nil { return err } - for _, tool := range []string{"skopeo", "virt-make-fs"} { - if err := s.host.LookPath(tool); err != nil { - return fmt.Errorf("--bake-images needs %q on PATH: %w\n hint: install skopeo and guestfs-tools (Fedora/RHEL) or skopeo + libguestfs-tools (Debian/Ubuntu)", tool, err) + if err := s.host.LookPath("skopeo"); err != nil { + return fmt.Errorf("--bake-images needs %q on PATH: %w\n hint: %s", "skopeo", err, bakeToolHint()) + } + if runtime.GOOS == "darwin" { + // The packer is mke2fs (no libguestfs on macOS); Homebrew's e2fsprogs + // is keg-only, so also probe its known keg locations. + if !mke2fsPresent(s.host) { + return fmt.Errorf("--bake-images needs mke2fs (none of %v found)\n hint: %s", config.MKE2FSCandidates, bakeToolHint()) } + return nil + } + if err := s.host.LookPath("virt-make-fs"); err != nil { + return fmt.Errorf("--bake-images needs %q on PATH: %w\n hint: %s", "virt-make-fs", err, bakeToolHint()) } return nil } +func mke2fsPresent(host interfaces.HostInspector) bool { + for _, c := range config.MKE2FSCandidates { + if strings.Contains(c, "/") { + if _, err := os.Stat(c); err == nil { + return true + } + continue + } + if err := host.LookPath(c); err == nil { + return true + } + } + return false +} + +func bakeToolHint() string { + if runtime.GOOS == "darwin" { + return "brew install skopeo e2fsprogs" + } + return "install skopeo and guestfs-tools (Fedora/RHEL) or skopeo + libguestfs-tools (Debian/Ubuntu)" +} + func (s *Stage) Apply(ctx context.Context, sc *interfaces.StageContext) error { if !sc.Cluster.BakeImages { return nil @@ -75,6 +108,6 @@ func (s *Stage) spec(sc *interfaces.StageContext) interfaces.BakeSpec { OCBinaryPath: sc.OCBinaryPath(), PullSecretPath: config.PullSecretPath(cfgDir), OverlayDir: config.ImageStoreOverlayDir(cfgDir, version), - OutputQcowPath: config.ImageStoreQcowPath(cfgDir, version), + OutputDiskPath: config.ImageStoreDiskPath(cfgDir, version), } } diff --git a/stages/createmastervms/stage.go b/stages/createmastervms/stage.go index e262f35..2d06a92 100644 --- a/stages/createmastervms/stage.go +++ b/stages/createmastervms/stage.go @@ -58,7 +58,7 @@ func (s *Stage) Preflight(ctx context.Context, sc *interfaces.StageContext) erro need := uint64(sc.Cluster.MasterDiskGB) * 1024 * 1024 * 1024 // Baking attaches a per-cluster copy of the store qcow2; count it. if sc.Cluster.BakeImages { - if fi, err := os.Stat(config.ImageStoreQcowPath(sc.Config.ConfigDir, sc.Cluster.OCPVersion)); err == nil { + if fi, err := os.Stat(config.ImageStoreDiskPath(sc.Config.ConfigDir, sc.Cluster.OCPVersion)); err == nil { need += uint64(fi.Size()) } } @@ -148,7 +148,7 @@ func (s *Stage) attachImageStore(ctx context.Context, sc *interfaces.StageContex // ImportDisk stats the source and returns a clear error if the bake-image- // store stage never produced it, so no extra guard is needed here (and the // raw stat would wrongly fail under --simulate, where no real file exists). - cached := config.ImageStoreQcowPath(sc.Config.ConfigDir, sc.Cluster.OCPVersion) + cached := config.ImageStoreDiskPath(sc.Config.ConfigDir, sc.Cluster.OCPVersion) volPath, err := s.vm.ImportDisk(ctx, sc.Cluster.StoragePool, config.ImageStoreVolName(vmName), cached) if err != nil { return interfaces.ExtraDisk{}, fmt.Errorf("import baked image store into pool: %w", err) diff --git a/stages/publishpxeassets/stage.go b/stages/publishpxeassets/stage.go index 78011f2..64cd451 100644 --- a/stages/publishpxeassets/stage.go +++ b/stages/publishpxeassets/stage.go @@ -30,11 +30,15 @@ import ( // Stage publishes PXE-style boot assets. type Stage struct { - files interfaces.FileServer + files interfaces.FileServer + installer interfaces.Installer } -// New returns the publish-pxe-assets stage. -func New(files interfaces.FileServer) *Stage { return &Stage{files: files} } +// New returns the publish-pxe-assets stage. installer is used only for the +// --bake-images ignition merge. +func New(files interfaces.FileServer, installer interfaces.Installer) *Stage { + return &Stage{files: files, installer: installer} +} func (*Stage) Name() string { return "publish-pxe-assets" } @@ -51,7 +55,7 @@ func KernelCmdline(baseURL, cluster string) string { // Apply copies the RHCOS rootfs and the SNO ignition (with a static-network // keyfile injected to pin the master IP) into //, and // records the install-phase cmdline on the cluster for create-master-vms. -func (s *Stage) Apply(_ context.Context, sc *interfaces.StageContext) error { +func (s *Stage) Apply(ctx context.Context, sc *interfaces.StageContext) error { cluster := sc.Cluster.Name dstDir := filepath.Join(s.files.RootDir(), cluster) if err := os.MkdirAll(dstDir, 0o755); err != nil { @@ -69,9 +73,18 @@ func (s *Stage) Apply(_ context.Context, sc *interfaces.StageContext) error { if err != nil { return fmt.Errorf("inject static network: %w", err) } - if err := os.WriteFile(filepath.Join(dstDir, "config.ign"), merged, 0o600); err != nil { + ignPath := filepath.Join(dstDir, "config.ign") + if err := os.WriteFile(ignPath, merged, 0o600); err != nil { return fmt.Errorf("publish ignition: %w", err) } + // Baking: wire the baked store into the *served* ignition — the macOS + // equivalent of embed-ignition-iso's live-ISO merge on Linux — so the + // bootstrap phase mounts the store and CRI-O serves images locally. + if sc.Cluster.BakeImages { + if err := s.installer.MergeImageStoreIntoLiveISOIgnition(ctx, sc.InstallerSpec(), ignPath); err != nil { + return fmt.Errorf("merge image store into served ignition: %w", err) + } + } sc.Cluster.InstallKernelCmdline = KernelCmdline(s.files.BaseURL(), cluster) return nil diff --git a/stages/publishpxeassets/stage_test.go b/stages/publishpxeassets/stage_test.go index c4954d2..f1ca809 100644 --- a/stages/publishpxeassets/stage_test.go +++ b/stages/publishpxeassets/stage_test.go @@ -1,14 +1,20 @@ package publishpxeassets_test import ( + "context" + "os" + "path/filepath" "strings" "testing" + "github.com/TheEasyShift/easyshift/config" + "github.com/TheEasyShift/easyshift/interfaces" + "github.com/TheEasyShift/easyshift/providers/fakes" "github.com/TheEasyShift/easyshift/stages/publishpxeassets" ) func TestName(t *testing.T) { - if got := publishpxeassets.New(nil).Name(); got != "publish-pxe-assets" { + if got := publishpxeassets.New(nil, nil).Name(); got != "publish-pxe-assets" { t.Errorf("unexpected stage name %q", got) } } @@ -22,3 +28,52 @@ func TestKernelCmdline(t *testing.T) { t.Errorf("cmdline missing rootfs url: %q", cmdline) } } + +// TestApplyMergesImageStoreWhenBaking asserts the served ignition gets the +// baked-store wiring (the macOS equivalent of embed-ignition-iso's live-ISO +// merge on Linux). +func TestApplyMergesImageStoreWhenBaking(t *testing.T) { + cfgDir := t.TempDir() + files := &fakes.FileServer{Root: t.TempDir(), URL: "http://fake:9393"} + inst := &fakes.Installer{} + s := publishpxeassets.New(files, inst) + + cfg := &config.Config{ConfigDir: cfgDir} + c := &config.ClusterConfig{ + Name: "t", NetworkMode: config.NetworkModeNAT, + BakeImages: true, + IPAddresses: []string{"192.168.126.9"}, + MACAddresses: []string{"52:54:00:00:00:09"}, + MasterCount: 1, + } + sc := &interfaces.StageContext{Config: cfg, Cluster: c} + if err := os.MkdirAll(sc.ClusterDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(sc.RHCOSRootfsPath()), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sc.RHCOSRootfsPath(), []byte("ROOTFS"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sc.ClusterDir(), "bootstrap-in-place-for-live-iso.ign"), []byte(`{"ignition":{"version":"3.4.0"}}`), 0o600); err != nil { + t.Fatal(err) + } + + if err := s.Apply(context.Background(), sc); err != nil { + t.Fatalf("Apply: %v", err) + } + if !inst.MergedImageStoreIgnition { + t.Error("expected MergeImageStoreIntoLiveISOIgnition on the served ignition") + } + + // And not when baking is off. + inst2 := &fakes.Installer{} + c.BakeImages = false + if err := publishpxeassets.New(files, inst2).Apply(context.Background(), sc); err != nil { + t.Fatalf("Apply (no bake): %v", err) + } + if inst2.MergedImageStoreIgnition { + t.Error("merge must not run without --bake-images") + } +} From dae613419939e7e214fd50236c88a93262baaaf0 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 25 Aug 2026 01:34:28 -0400 Subject: [PATCH 4/4] docs: macOS specifics in image-baking.md; update ROADMAP Assisted-by: Claude Code/claude-fable-5 Signed-off-by: Raghavendra Talur --- ROADMAP.md | 36 ++++++++++++++++++------------------ docs/dev/image-baking.md | 38 +++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 1438fc9..7823139 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,24 +28,24 @@ mounted with `context=container_file_t` — see the Task 13 notes in the spec), on darwin). Needs a vmnet bridged-mode story before LAN-reachable clusters work on Mac. -## Image baking (`--bake-images`) — branch `rtalur-bake-images` - -Complete and green on Linux (libvirt): opt-in flag, multi-arch store baked via -skopeo + virt-make-fs, attached read-only, wired into live-ISO ignition and a -master MachineConfig. - -- [ ] **macOS integration** (deliberately deferred to the on-hardware phase; - whichever branch merges second closes this): - - merge the baked-store wiring into the HTTP-served ignition in - `publish-pxe-assets` (the macOS equivalent of the live-ISO merge in - `embed-ignition-iso`); - - build the store without libguestfs: `mke2fs -d` (brew `e2fsprogs`) - instead of `virt-make-fs`, raw image instead of qcow2 (vfkit's - virtio-blk takes raw); - - a vfkit-side `ImportDisk` equivalent so `create-master-vms` can attach - the per-cluster store copy. -- [ ] **End-to-end validation on Linux**: a real `--bake-images` install that - confirms the node pulls platform images from the store, not quay.io. +## Image baking (`--bake-images`) — branch `rtalur-bake-images-macos` + +Code-complete on both backends (see `docs/dev/image-baking.md`). Linux: +skopeo + virt-make-fs → qcow2 into the libvirt pool. macOS: skopeo + +`mke2fs -d` → raw image, APFS-cloned per cluster, merged into the HTTP-served +ignition. `make check` green; darwin pipeline validated under `--simulate` and +the mke2fs pack validated against real e2fsprogs. + +Branch note: `rtalur-bake-images` is the same feature rebased onto `main` +(without the macOS backend); `rtalur-bake-images-macos` stacks it on +`rtalur-macos-backend` and adds the macOS integration. Whichever merge order +is chosen, keep only one of the two bake branches. + +- [ ] **End-to-end validation on a real cluster** (either OS): a + `--bake-images` install confirming CRI-O serves release images from the + store, not quay.io, in both bootstrap and post-pivot phases. Needs + ~40+ GB free disk for the multi-arch store — did not fit the dev Mac + alongside the existing cluster. ## Later phases (per project vision) diff --git a/docs/dev/image-baking.md b/docs/dev/image-baking.md index 940a1b9..e42b82b 100644 --- a/docs/dev/image-baking.md +++ b/docs/dev/image-baking.md @@ -30,12 +30,20 @@ a no-op, like the binaries cache). `providers/openshift.OpenShiftImageBaker`: supported arch enumerates the component pullspecs. 2. `skopeo copy --all` copies each into an overlay container store (`store/`). `--all` keeps every manifest-list entry. -3. `virt-make-fs --type=ext4 --label=baked-images --format=qcow2` packs the - store into `store.qcow2`. `virt-make-fs` runs rootless via libguestfs, so no - root is required. - -Needs `skopeo` and `virt-make-fs` (guestfs-tools / libguestfs-tools) on PATH; -the bake stage preflights both. +3. The store is packed into a labeled ext4 disk image (`PackCommand`): + - **Linux**: `virt-make-fs --type=ext4 --label=baked-images --format=qcow2` + → `store.qcow2` (rootless via libguestfs; qcow2 because a per-cluster + copy is uploaded into the libvirt pool). + - **macOS**: `mke2fs -t ext4 -L baked-images -d ` → `store.img` + (raw — vfkit's virtio-blk takes raw). No libguestfs exists on macOS; + `mke2fs -d` populates the fs from a directory without mounting. The + image is sized from the overlay contents (+10% and 1 GiB headroom). + Homebrew's e2fsprogs is keg-only, so its keg sbin paths are probed + (`config.MKE2FSCandidates`). + +Preflight requires `skopeo` plus the per-OS packer: `virt-make-fs` +(guestfs-tools / libguestfs-tools) on Linux, `mke2fs` +(`brew install e2fsprogs`) on macOS. ### Multi-arch @@ -49,10 +57,12 @@ RHCOS live-ISO arch is still selected separately (see ### Attach + wire (node side) -- `stages/createmastervms` uploads a **per-cluster** copy of `store.qcow2` into - the libvirt pool (`ImportDisk`) and attaches it read-only + shareable. Per - cluster — not shared — so `virsh undefine --remove-all-storage` on delete - never strands another cluster. +- `stages/createmastervms` attaches a **per-cluster** copy of the store via + `ImportDisk`: on Linux uploaded into the libvirt pool (read-only + + shareable; per cluster so `virsh undefine --remove-all-storage` on delete + never strands another cluster), on macOS APFS-cloned into the vfkit state + dir (vfkit has no read-only virtio-blk, so the per-cluster copy is the + isolation; `Delete` removes the clone). - The disk is mounted by label (`/dev/disk/by-label/baked-images` → `/var/lib/baked-images`) and registered with CRI-O via `additionalimagestores` in a `storage.conf` drop-in. @@ -61,9 +71,11 @@ RHCOS live-ISO arch is still selected separately (see `openshift/` (`Installer.WriteImageStoreManifest`) so it is rendered into the node's ignition and present from first boot, before CRI-O pulls operators. - - **bootstrap:** the same file + mount unit merged into - `bootstrap-in-place-for-live-iso.ign` - (`Installer.MergeImageStoreIntoLiveISOIgnition`) before the ISO is embedded. + - **bootstrap:** the same file + mount unit merged into the bootstrap + ignition (`Installer.MergeImageStoreIntoLiveISOIgnition`) — on Linux into + `bootstrap-in-place-for-live-iso.ign` before the ISO is embedded + (`embed-ignition-iso`), on macOS into the HTTP-served `config.ign` + (`publish-pxe-assets`). Renderers live in `providers/openshift/baker.go` (`RenderStorageConfDropin`, `RenderMountUnit`, `RenderMachineConfig`, `MergeBakedStoreIntoIgnition`) and are