[CELL-293] cell nix-store push|pull — replace crane CLI in the cache pipeline with native Go subcommands so the local test and CI workflow share one code path; encoding regressions can't recur silently

- feat(cli): add `cell nix-store push` + `cell nix-store pull` — streams a /nix volume to/from an OCI registry as a single tar+gzip layer. Push uses `pkg/v1/stream.NewLayer` to upload uncompressed-tar stdin straight to the registry's chunked-upload API with the digest computed incrementally (no temp file, no disk staging — peak runner disk drops from ~11 GB to ~64 KB). Pull resolves the manifest, streams the last layer's gunzipped bytes, and extracts into either a Docker volume (`--volume`) or a host directory (`--dir`), with `--strip-components` matching `tar --strip-components=N` (defaults to 1 — workflow's prior behavior). The on-wire layer is always single-gzipped: deterministic regardless of base-image schema, eliminates the `crane append --new_layer -` double-gzip + disk-buffer failure modes that caused six CELL-292 iterations
- fix(infra): rewrite the cache-publish step in `.github/workflows/build.dev.yml` to pipe uncompressed tar into `./bin/cell nix-store push` instead of staging an 11 GB tarball to `$RUNNER_TEMP` for `crane append --new_layer FILE`. Workflow now publishes via the same Go binary the local test invokes, so a green `task test:cache` is a faithful regression net for this step
- fix(infra): rewrite both pull paths in `.github/workflows/build.dev.yml` (docker-build warm-start + docker-test stream-pull) to use `./bin/cell nix-store pull --volume VOL` instead of `crane blob | gunzip | docker run alpine tar -x --strip-components=1`. The docker-test job now builds the cell binary too (matching docker-build), so push and pull go through identical bytes-on-disk code
- fix(infra): port the same cache pipeline to `.github/workflows/build.release.yml` — replace the broken `actions/cache@v4` step (the v1 design CELL-292 abandoned because `_data` is root-owned and the cache action silently saved an empty archive) with `cell nix-store pull` hydrate + `cell nix-store push` republish. Release builds now share the `nix-cache2-${arch}-${HASH}` tag with dev builds, so any release that runs after a green dev build hits a populated cache instead of starting from an empty /nix volume
- test(nixstore): table-driven unit tests against go-containerregistry's in-memory OCI registry (`internal/nixstore/{pull,push}_test.go`) — covers byte-for-byte round-trip, last-layer-only extraction, error paths on bad refs, and explicit assertion that the pushed layer is single-gzipped (`raw[0:2] == 1f8b` and `gunzip(layer)[0:4] != 1f8b`). 6/6 PASS
- test(integration): `test/cache_roundtrip_test.go` invokes `./bin/cell nix-store push|pull` via `exec.Command` instead of shell-piping `crane append`/`crane blob` — same binary as the CI workflow, so test ≡ production. End-to-end PASS: 198,643 files round-tripped byte-for-byte through the actual subcommands

Author: DimmKirr

.github/workflows/build.dev.yml

@@ -103,13 +103,16 @@ jobs:
       - name: Build cell binary
         run: task cell:build
 
-      # crane = google/go-containerregistry CLI. We use it to stream the
-      # nix-store volume directly to/from a GHCR layer blob without going
-      # through `docker buildx` or `docker pull`'s overlay2 extract.
-      # Push: tar -czf - | crane append → registry (no 32 GB tar on disk).
-      # Pull: crane blob | gunzip | tar -x → volume (no overlay2 extract).
-      # Net: ~32 GB less peak transient disk per job vs the Docker-native
-      # round-trip.
+      # Cache push/pull goes through `cell nix-store {push,pull}` — a
+      # cell subcommand using `pkg/v1/stream.NewLayer` from
+      # go-containerregistry. Replaces the `crane` CLI usage that
+      # required either file staging (~11 GB temp file) or accepted
+      # double-gzipped layers (CELL-292 iteration history). With the
+      # cell subcommand, the layer is streamed (no temp file) and
+      # always single-gzipped on the wire — see CELL-293.
+      # crane is still installed for the manifest size-probe in the
+      # warm-start guard; can be removed once `cell nix-store inspect`
+      # lands.
       - name: Install crane
         run: |
           set -eo pipefail
@@ -156,23 +159,16 @@ jobs:
             exit 0
           fi
           echo "cache HIT: $IMG"
-          # The appended nix-store layer is the LAST one (atop the busybox
-          # base). Stream the blob → ONE gunzip → tar -x into the volume.
-          # Single-gunzip is correct because the push step writes the
-          # gzipped tar to a FILE first and passes the file to
-          # `crane append --new_layer FILE` (crane detects gzip magic in
-          # the file content and stores it as-is, single-gzipped on the
-          # registry). Earlier `tar -czf | crane append --new_layer -`
-          # stdin pipeline produced a double-gzipped layer on amd64
-          # (CI #217 failure mode); file-based push is deterministic.
-          LAYER_DIGEST=$(crane manifest "$IMG" | jq -r '.layers[-1].digest')
+          # `cell nix-store pull` fetches the LAST layer of $IMG (atop
+          # the busybox base) and streams it directly into the named
+          # Docker volume — internal gunzip + tar -x, no separate
+          # shell pipe. Replaces the prior
+          # `crane blob | gunzip | docker run alpine tar -x` chain.
           docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true
           docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null
-          crane blob "$IMG@$LAYER_DIGEST" \
-            | gunzip \
-            | docker run --rm -i \
-                -v devcell-nix-store-${{ matrix.arch }}:/dest \
-                alpine sh -c 'cd /dest && tar -x --strip-components=1'
+          ./bin/cell nix-store pull \
+            --image "$IMG" \
+            --volume "devcell-nix-store-${{ matrix.arch }}"
           docker run --rm -v devcell-nix-store-${{ matrix.arch }}:/nix:ro alpine \
             sh -c 'echo "hydrated $(du -sh /nix | cut -f1)"'
 
@@ -264,72 +260,37 @@ jobs:
           HB_PID=$!
           trap "kill $HB_PID 2>/dev/null || true" EXIT
 
-          # Stream gzipped tar from the volume → pv (throughput monitor) →
-          # crane → registry as a single OCI layer atop a busybox base.
-          # crane uploads the layer blob in chunks while tar is still
-          # emitting bytes, so the network side is streamed.
-          # CRITICAL: emit gzipped tar (`-czf`, not `-cf`).
-          # `crane append --new_layer -` buffers stdin to a temp file in
-          # the runner's disk to compute the layer digest before it can
-          # start the registry upload. Uncompressed `/nix` is ~32 GB —
-          # that temp file exhausts the runner's disk and crashes the
-          # runner worker itself (System.IO.IOException: No space left
-          # on device on `/home/runner/extracted/_diag/Worker_*.log`,
-          # observed on commit 5a930e3, both arches). With `-czf` the
-          # pipe carries ~10 GB which fits the runner's free disk.
-          # crane does NOT re-compress already-gzipped input — empirical
-          # check: `crane blob ... | gunzip | head -c 4` on the resulting
-          # layer yields tar magic (`nix/`), not a second gzip wrapper.
-          # So pull side uses a single `gunzip | tar -x`.
-          # `-C / nix` (instead of `-C /nix .` + --transform) produces
-          # archive paths with the `nix/` prefix natively, so the layer's
-          # files land at /nix/* inside the cache image (matched by
-          # `--strip-components=1` on the pull side). We use this form
-          # because BusyBox tar (the default in alpine) doesn't implement
-          # GNU tar's --transform option.
-          # --exclude: Unix sockets can't be archived — pre-exclude the
-          # nix-daemon socket so tar doesn't emit a "socket ignored"
-          # warning that could trip stricter tar implementations.
-          # `pv -tabri 30 -f`: every 30s emit elapsed/avg-rate/cur-rate/
-          # bytes-transferred to stderr (-f forces output to a non-TTY).
-          # crane verbosity (`-v`) is DELIBERATELY OFF — when enabled it
-          # logs one stderr line per HTTP chunk upload, which at full
-          # upload speed flooded GitHub Actions' per-step log cap on
-          # commit f592686.
+          # Stream UNCOMPRESSED tar from the volume → `cell nix-store
+          # push` → registry. The cell subcommand uses
+          # `pkg/v1/stream.NewLayer` from go-containerregistry: it
+          # reads stdin lazily, gzips on the fly, and uploads via the
+          # OCI chunked-upload protocol with the digest computed
+          # incrementally. Peak runner disk for this step: ~64 KB
+          # (gzip window buffer + HTTP chunk buffer). The on-wire
+          # layer is SINGLE-gzipped — the test/cache_roundtrip_test
+          # asserts this exactly, so any regression here surfaces in
+          # ~5 min locally instead of ~30 min of CI.
+          # `tar -cf` not `-czf`: cell nix-store push gzips for us.
+          # `-C / nix` (instead of `-C /nix .` + --transform) lands
+          # archive entries under `nix/...` natively (BusyBox tar
+          # doesn't implement --transform).
+          # `--exclude=nix/var/nix/daemon-socket`: Unix sockets can't
+          # be archived.
+          # `pv -tabri 30 -f`: every 30s emit throughput line.
           echo "==== Publish start ($(date -u +%H:%M:%SZ)) ===="
           SECONDS=0
 
-          # Stage tar.gz to a temp file FIRST, then pass the file to
-          # `crane append --new_layer FILE`. Why file instead of stdin:
-          # `crane append --new_layer -` treats stdin as an uncompressed
-          # tarball and gzip-wraps it on upload, even if it's already
-          # gzipped — so `tar -czf | crane append --new_layer -` lands
-          # a double-gzipped layer on the registry. The file-based path
-          # detects gzip magic in the file content and stores it as-is.
-          # This makes the on-wire encoding deterministic (single-gzip)
-          # so the pull side's `crane blob | gunzip | tar -x` works
-          # without guessing how many gunzip passes to apply. Mirrors
-          # `test/cache_roundtrip_test.go`'s push exactly so the local
-          # test is a faithful regression net for this step.
-          # Disk cost: one ~11 GB temp file on the runner; runner has
-          # ~60 GB free after the Free-disk-space step, comfortable.
-          # crane append's `--new_tag` is `string` (singular), not
-          # `strings` — passing it twice silently overwrites, so push
-          # $LATEST then alias $EXACT via `crane tag` (manifest-only,
-          # no re-upload).
-          CACHE_TAR="${RUNNER_TEMP:-/tmp}/nix-cache-${{ matrix.arch }}.tar.gz"
           docker run --rm -v devcell-nix-store-${{ matrix.arch }}:/nix:ro alpine \
-            tar -czf - \
+            tar -cf - \
               --exclude='nix/var/nix/daemon-socket' \
               -C / nix \
             | pv -tabri 30 -f -N tar-bytes \
-            > "$CACHE_TAR"
-          echo "staged tarball: $(stat -c%s "$CACHE_TAR") bytes at $CACHE_TAR"
-          crane append \
-              --base public.ecr.aws/docker/library/busybox:latest \
-              --new_layer "$CACHE_TAR" \
-              --new_tag "$LATEST"
-          rm -f "$CACHE_TAR"
+            | ./bin/cell nix-store push \
+                --base public.ecr.aws/docker/library/busybox:latest \
+                --image "$LATEST"
+          # Alias to the hash-pinned tag via crane (manifest-only,
+          # no re-upload). Could be replaced with a `cell nix-store
+          # tag` subcommand in a follow-up.
           crane tag "$LATEST" "nix-cache2-${{ matrix.arch }}-${HASH}"
 
           echo
@@ -383,6 +344,18 @@ jobs:
           go-version-file: go.mod
           cache-dependency-path: go.sum
 
+      - name: Install go-task
+        uses: arduino/setup-task@v2
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Build cell so we can use `cell nix-store pull` below — same
+      # binary docker-build's publish step uses, so any encoding
+      # divergence between push and pull is structurally impossible.
+      - name: Build cell binary
+        run: task cell:build
+
       - name: Install crane
         run: |
           set -eo pipefail
@@ -396,12 +369,10 @@ jobs:
             | sudo tar -xzC /usr/local/bin crane
           crane version
 
-      # Stream-pull the nix-store cache image's layer blob directly into the
-      # per-arch volume via crane, bypassing Docker's overlay2 extract +
-      # auto-seed copy (~32 GB less peak transient disk vs `docker pull`).
-      # Single seed: we set DEVCELL_NIX_VOLUME on the test step below so
-      # `cell shell` / `cell claude` use this per-arch volume directly
-      # instead of the default-named one — no need to seed two volumes.
+      # Stream-pull the nix-store cache image's last layer into the
+      # per-arch volume via `cell nix-store pull` — same binary the
+      # publish step in docker-build uses, so push/pull encoding
+      # divergence is impossible by construction.
       - name: Stream-pull /nix cache from GHCR into volume
         run: |
           set -eo pipefail
@@ -429,18 +400,11 @@ jobs:
             exit 0
           fi
           echo "cache HIT: $IMG"
-          # ONE gunzip: the publish step writes gzipped tar to a FILE
-          # and uses `crane append --new_layer FILE`, which stores the
-          # file content as-is (single-gzipped on the registry).
-          # Mirrors the warm-start step in docker-build.
-          LAYER_DIGEST=$(crane manifest "$IMG" | jq -r '.layers[-1].digest')
           docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true
           docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null
-          crane blob "$IMG@$LAYER_DIGEST" \
-            | gunzip \
-            | docker run --rm -i \
-                -v devcell-nix-store-${{ matrix.arch }}:/dest \
-                alpine sh -c 'cd /dest && tar -x --strip-components=1'
+          ./bin/cell nix-store pull \
+            --image "$IMG" \
+            --volume "devcell-nix-store-${{ matrix.arch }}"
 
       - name: Show volume status (debug)
         run: |

cmd/nix_store.go

@@ -0,0 +1,118 @@
+package main
+
+// `cell nix-store` — push/pull a /nix volume as an OCI layer against a
+// remote registry. Replaces the `crane` CLI usage in
+// .github/workflows/build.dev.yml so the local cache-roundtrip test
+// and the CI workflow share a single Go code path (CELL-293).
+//
+// Currently implements `pull`. Push lands once pull is validated end
+// to end (see CELL-293's TDD ordering).
+
+import (
+	"io"
+	"os"
+
+	"github.com/DimmKirr/devcell/internal/nixstore"
+	"github.com/spf13/cobra"
+)
+
+var nixStoreCmd = &cobra.Command{
+	Use:   "nix-store",
+	Short: "Push or pull a /nix volume as an OCI layer (cache pipeline)",
+}
+
+var nixStorePullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "Pull the last layer of an OCI image into a Docker volume or directory",
+	Long: `Pull downloads the LAST layer of the given OCI image and extracts the
+gzipped tarball into either a Docker volume (--volume) or a host
+directory (--dir).
+
+Layer selection: we only extract the topmost layer because the
+production cache image is structured as a busybox base + one nix-store
+tar layer. Earlier layers are skipped.
+
+Streaming: the layer's bytes are gunzipped + tar-extracted lazily as
+they arrive, so peak memory stays near the gzip window size (~32 KB)
+regardless of layer size.
+
+Examples:
+  cell nix-store pull --image ghcr.io/org/repo:nix-cache-amd64-latest --volume devcell-nix-store
+  cell nix-store pull --image ghcr.io/org/repo:nix-cache-amd64-latest --dir /tmp/cache-extract`,
+	RunE: runNixStorePull,
+}
+
+var nixStorePushCmd = &cobra.Command{
+	Use:   "push",
+	Short: "Stream an uncompressed tar from stdin into a registry as a single OCI tar+gzip layer",
+	Long: `Push reads an UNCOMPRESSED tar from stdin and uploads it as a single
+OCI tar+gzip layer atop --base, tagged --image, to the destination
+registry.
+
+Streaming: the bytes flow stdin → gzip → registry chunked upload with
+the digest computed incrementally. Peak memory ~32 KB (gzip window),
+no disk staging.
+
+The on-wire layer is SINGLE-gzipped — this function is the
+deterministic alternative to ` + "`crane append --new_layer -`" + ` which
+re-gzipped pre-gzipped stdin.
+
+Example:
+  docker run --rm -v devcell-nix-store:/nix:ro alpine \
+    tar -cf - --exclude='nix/var/nix/daemon-socket' -C / nix \
+  | cell nix-store push \
+      --base public.ecr.aws/docker/library/busybox:latest \
+      --image ghcr.io/org/repo:nix-cache-amd64-latest`,
+	RunE: runNixStorePush,
+}
+
+func init() {
+	nixStorePullCmd.Flags().String("image", "", "OCI image reference to pull (e.g. ghcr.io/org/repo:tag)")
+	nixStorePullCmd.Flags().String("volume", "", "Docker volume name to extract into (mutually exclusive with --dir)")
+	nixStorePullCmd.Flags().String("dir", "", "Host directory to extract into (mutually exclusive with --volume)")
+	nixStorePullCmd.Flags().Int("strip-components", 1, "Strip N leading path elements from archive entries (tar --strip-components semantics)")
+	_ = nixStorePullCmd.MarkFlagRequired("image")
+
+	nixStorePushCmd.Flags().String("base", "", "OCI base image to layer atop (e.g. public.ecr.aws/docker/library/busybox:latest)")
+	nixStorePushCmd.Flags().String("image", "", "destination OCI image reference (e.g. ghcr.io/org/repo:tag)")
+	_ = nixStorePushCmd.MarkFlagRequired("base")
+	_ = nixStorePushCmd.MarkFlagRequired("image")
+
+	nixStoreCmd.AddCommand(nixStorePullCmd)
+	nixStoreCmd.AddCommand(nixStorePushCmd)
+	rootCmd.AddCommand(nixStoreCmd)
+}
+
+func runNixStorePush(cmd *cobra.Command, args []string) error {
+	base, _ := cmd.Flags().GetString("base")
+	image, _ := cmd.Flags().GetString("image")
+	return nixstore.Push(cmd.Context(), base, image, io.NopCloser(os.Stdin))
+}
+
+func runNixStorePull(cmd *cobra.Command, args []string) error {
+	image, _ := cmd.Flags().GetString("image")
+	volume, _ := cmd.Flags().GetString("volume")
+	dir, _ := cmd.Flags().GetString("dir")
+	strip, _ := cmd.Flags().GetInt("strip-components")
+
+	switch {
+	case volume == "" && dir == "":
+		return errFlag("must specify either --volume or --dir")
+	case volume != "" && dir != "":
+		return errFlag("--volume and --dir are mutually exclusive")
+	case volume != "":
+		return nixstore.PullToDockerVolume(cmd.Context(), image, volume, strip)
+	default:
+		return nixstore.Pull(cmd.Context(), image, dir, strip)
+	}
+}
+
+// errFlag wraps a flag-usage error in a way that cobra's `Use:` help
+// is printed alongside.
+func errFlag(msg string) error {
+	return &flagError{msg: msg}
+}
+
+type flagError struct{ msg string }
+
+func (e *flagError) Error() string { return e.msg }

go.mod

@@ -49,10 +49,13 @@ require (
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v29.4.0+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.9.3 // indirect
 	github.com/docker/go-connections v0.6.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -82,6 +85,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/go-archive v0.1.0 // indirect
@@ -118,6 +122,7 @@ require (
 	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/vbatts/tar-split v0.12.2 // indirect
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect