[CELL-293] drop crane from CI/CD entirely and strip the volume-insp…
#225
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dev Build | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - feature/wip | |
| - feature/add-web | |
| - feature/web | |
| workflow_dispatch: | |
| inputs: | |
| skip_nix_cache: | |
| description: 'Skip nix cache (genesis mode — full rebuild, no pre-seeding)' | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| packages: write | |
| id-token: write | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| secrets: | |
| name: Detect Secrets | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| # gitleaks CLI directly — bypasses gitleaks-action's org-license | |
| # requirement (the license gate lives in the action wrapper, not the | |
| # scanner). `detect` exits 1 if leaks are found. | |
| - name: Run gitleaks | |
| run: | | |
| docker run --rm -v "$PWD:/repo" -w /repo \ | |
| ghcr.io/gitleaks/gitleaks:latest \ | |
| detect --source=. --verbose --redact | |
| docker-build: | |
| name: Docker Build (${{ matrix.arch }}) | |
| needs: secrets | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: ubuntu-latest | |
| arch: amd64 | |
| platform: linux/amd64 | |
| - runner: ubuntu-24.04-arm | |
| arch: arm64 | |
| platform: linux/arm64 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune -af | |
| df -h | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| buildkitd-config-inline: | | |
| [worker.oci] | |
| max-parallelism = 4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: go.sum | |
| - name: Install go-task | |
| uses: arduino/setup-task@v2 | |
| with: | |
| version: 3.x | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # task cell:build = swagger:generate → CGO_ENABLED=0 go build with ldflags | |
| # for GitCommit + BuildDate. Single source of truth shared with local dev | |
| # (`task cell:build` works the same on a developer machine). | |
| - name: Build cell binary | |
| run: task cell:build | |
| # Cache push/pull goes through `cell nix-store {push,pull}` — a | |
| # cell subcommand using `pkg/v1/stream.NewLayer` from | |
| # go-containerregistry. Layer is streamed (no temp file) and | |
| # always single-gzipped on the wire — see CELL-293. | |
| # Cache the nix-store as a GHCR image. Lives in the SAME | |
| # `devcell-sh/devcell` GHCR package as the runtime images, with tag | |
| # prefix `nix-cache-` so it doesn't collide with stack tags. | |
| - name: Stream-hydrate /nix volume from prior GHCR cache (if available) | |
| if: inputs.skip_nix_cache != true | |
| run: | | |
| set -eo pipefail | |
| HASH=${{ hashFiles('nixhome/**') }} | |
| EXACT="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-${HASH}" | |
| LATEST="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-latest" | |
| # Pick the first candidate that resolves a manifest. `cell | |
| # nix-store pull` itself fails cleanly on a missing/poisoned | |
| # cache, and the workflow falls back to MISS — no upfront | |
| # size probe needed. | |
| IMG="" | |
| for c in "$EXACT" "$LATEST"; do | |
| if docker manifest inspect "$c" >/dev/null 2>&1; then | |
| IMG="$c" | |
| break | |
| fi | |
| done | |
| if [ -z "$IMG" ]; then | |
| echo "cache MISS — starting from empty volume" | |
| exit 0 | |
| fi | |
| echo "cache HIT: $IMG" | |
| # `cell nix-store pull` streams $IMG's last layer directly | |
| # into the named docker volume (internal gunzip + tar -x). | |
| # On failure, fall back to MISS — the next publish overwrites | |
| # the cache with the current encoding. | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| if ./bin/cell nix-store pull \ | |
| --image "$IMG" \ | |
| --volume "devcell-nix-store-${{ matrix.arch }}"; then | |
| docker run --rm -v devcell-nix-store-${{ matrix.arch }}:/nix:ro alpine \ | |
| sh -c 'echo "hydrated $(du -sh /nix | cut -f1)"' | |
| else | |
| echo "WARN: cache pull failed (likely encoding mismatch with legacy cache image)" | |
| echo " falling back to MISS — will rebuild and publish a fresh cache" | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| fi | |
| # Build both stacks sequentially in the same job so the nix-store | |
| # volume accumulates derivations from both — single tar dump at job | |
| # end carries everything needed by docker-test. `cell build --thin` | |
| # reuses store paths across the two invocations (nix is | |
| # content-addressed), so ultimate after base is incremental. | |
| - name: Build thin image (base stack) | |
| env: | |
| DEVCELL_NIX_VOLUME: devcell-nix-store-${{ matrix.arch }} | |
| DEVCELL_NIXHOME_PATH: ${{ github.workspace }}/nixhome | |
| run: | | |
| BASE_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-base" | |
| ./bin/cell build --thin --stack base --image "$BASE_TAG" --debug | |
| echo "BASE_TAG=$BASE_TAG" >> "$GITHUB_ENV" | |
| - name: Build thin image (ultimate stack) | |
| env: | |
| DEVCELL_NIX_VOLUME: devcell-nix-store-${{ matrix.arch }} | |
| DEVCELL_NIXHOME_PATH: ${{ github.workspace }}/nixhome | |
| run: | | |
| ULT_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate" | |
| ./bin/cell build --thin --stack ultimate --image "$ULT_TAG" --debug | |
| echo "ULT_TAG=$ULT_TAG" >> "$GITHUB_ENV" | |
| - name: Push to GHCR (both stacks) | |
| run: | | |
| docker push "$BASE_TAG" | |
| docker push "$ULT_TAG" | |
| # Stream-publish the populated /nix volume to GHCR as a single- | |
| # layer cache image. `cell nix-store push` uses | |
| # `pkg/v1/stream.NewLayer` from go-containerregistry: stdin is | |
| # gzipped lazily, the digest is computed incrementally, and the | |
| # blob is uploaded via the OCI chunked-upload protocol — peak | |
| # runner disk is the pipe buffer, not the volume size. | |
| # `tar -cf` (not `-czf`): cell nix-store push gzips for us. | |
| # `-C / nix`: archive entries land under `nix/...` natively | |
| # (BusyBox tar doesn't implement --transform). | |
| # `--exclude=nix/var/nix/daemon-socket`: Unix sockets can't tar. | |
| - name: Stream-publish /nix volume to GHCR cache image | |
| timeout-minutes: 80 | |
| run: | | |
| set -eo pipefail | |
| HASH=${{ hashFiles('nixhome/**') }} | |
| EXACT="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-${HASH}" | |
| LATEST="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-latest" | |
| docker run --rm -v devcell-nix-store-${{ matrix.arch }}:/nix:ro alpine \ | |
| tar -cf - \ | |
| --exclude='nix/var/nix/daemon-socket' \ | |
| -C / nix \ | |
| | ./bin/cell nix-store push \ | |
| --base public.ecr.aws/docker/library/busybox:latest \ | |
| --image "$LATEST" | |
| # Alias the *-latest tag to the content-hashed tag without | |
| # re-uploading blobs. | |
| docker buildx imagetools create --tag "$EXACT" "$LATEST" | |
| echo "pushed nix-cache images:" | |
| echo " $EXACT" | |
| echo " $LATEST" | |
| docker-test: | |
| name: Docker Test (${{ matrix.arch }}) | |
| needs: docker-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: ubuntu-latest | |
| arch: amd64 | |
| - runner: ubuntu-24.04-arm | |
| arch: arm64 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: go.sum | |
| - name: Install go-task | |
| uses: arduino/setup-task@v2 | |
| with: | |
| version: 3.x | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Build cell so we can use `cell nix-store pull` below — same | |
| # binary docker-build's publish step uses, so any encoding | |
| # divergence between push and pull is structurally impossible. | |
| - name: Build cell binary | |
| run: task cell:build | |
| # Stream-pull the nix-store cache image's last layer into the | |
| # per-arch volume via `cell nix-store pull` — same binary the | |
| # publish step in docker-build uses, so push/pull encoding | |
| # divergence is impossible by construction. | |
| - name: Stream-pull /nix cache from GHCR into volume | |
| run: | | |
| set -eo pipefail | |
| HASH=${{ hashFiles('nixhome/**') }} | |
| EXACT="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-${HASH}" | |
| LATEST="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-latest" | |
| # Pick the first candidate that resolves a manifest. cell | |
| # nix-store pull fails cleanly on a missing cache, and the | |
| # workflow falls back to MISS — no upfront probe needed. | |
| IMG="" | |
| for c in "$EXACT" "$LATEST"; do | |
| if docker manifest inspect "$c" >/dev/null 2>&1; then | |
| IMG="$c" | |
| break | |
| fi | |
| done | |
| if [ -z "$IMG" ]; then | |
| echo "cache MISS — tests will run on empty /nix volume (lazy build during cell shell)" | |
| exit 0 | |
| fi | |
| echo "cache HIT: $IMG" | |
| # If pull fails, tests fall back to an empty volume (lazy | |
| # build during cell shell). | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| if ! ./bin/cell nix-store pull \ | |
| --image "$IMG" \ | |
| --volume "devcell-nix-store-${{ matrix.arch }}"; then | |
| echo "WARN: cache pull failed — tests will run on empty /nix" | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| fi | |
| - name: Pull test images (base + ultimate) | |
| run: | | |
| docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-base | |
| docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate | |
| - name: Run container tests | |
| # Both `DEVCELL_USER_IMAGE` and `DEVCELL_TEST_IMAGE` point at the | |
| # ultimate image (every module + tool baked in) so module-aware tests | |
| # (kicad, plex, gui) find what they need. `DEVCELL_TEST_BASE_IMAGE` | |
| # points at the smaller base image for entrypoint-only tests | |
| # (TestEntrypoint_DebugTimestamps etc.). Setting `DEVCELL_USER_IMAGE` | |
| # without `DEVCELL_USER_IMAGE_THIN` works because UserImageTagThin() | |
| # falls back to DEVCELL_USER_IMAGE as-is (no `-thin` suffix appended) | |
| # per CELL-286 prep. | |
| # `DEVCELL_NIX_VOLUME` points cell at the per-arch volume we just | |
| # hydrated from the GHCR cache, so `cell shell` / `cell claude` skip | |
| # the lazy-build path entirely instead of falling back to the | |
| # default-named volume (which would be empty). | |
| env: | |
| DEVCELL_NIX_VOLUME: devcell-nix-store-${{ matrix.arch }} | |
| DEVCELL_TEST_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate | |
| DEVCELL_USER_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate | |
| DEVCELL_TEST_BASE_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-base | |
| DEVCELL_TEST_PROJECT_DIR: ${{ runner.temp }}/devcell-test | |
| MCP_SECRET_TEST_PASSWORD: fake-secret-ci-value | |
| MCP_SECRET_GITHUB_TOKEN: fake-token-ci-value | |
| run: go test -v -timeout 1200s ./test/... | |
| docker-manifest: | |
| name: Docker Manifests | |
| needs: [docker-build] | |
| if: always() && needs.docker-build.result == 'success' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| env: | |
| IMAGE_NAME: ${{ github.repository }} | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create GHCR manifests (per stack + ultimate as canonical) | |
| run: | | |
| R=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }} | |
| # Per-stack multi-arch manifests | |
| docker buildx imagetools create \ | |
| -t "${R}:v0.0.0-base" \ | |
| "${R}:v0.0.0-amd64-base" \ | |
| "${R}:v0.0.0-arm64-base" | |
| docker buildx imagetools create \ | |
| -t "${R}:v0.0.0-ultimate" \ | |
| -t "${R}:v0.0.0" \ | |
| -t "${R}:latest" \ | |
| -t "${R}:dev" \ | |
| "${R}:v0.0.0-amd64-ultimate" \ | |
| "${R}:v0.0.0-arm64-ultimate" | |
| cell-build: | |
| name: Cell CLI Dev Build | |
| needs: secrets | |
| runs-on: ubuntu-latest | |
| env: | |
| RELEASE_VERSION: v0.0.0 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: go.sum | |
| - name: Set SHORT_SHA | |
| run: echo "SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-8)" >> $GITHUB_ENV | |
| - name: Delete existing tag | |
| run: | | |
| git config --global user.email "dmitry@atd.sh" | |
| git config --global user.name "Dmitry Kireev" | |
| git tag -d "${{ env.RELEASE_VERSION }}" || echo "No local tag to delete" | |
| git push origin :refs/tags/${{ env.RELEASE_VERSION }} || echo "No remote tag to delete" | |
| - name: Add tag | |
| run: | | |
| git tag -a ${{ env.RELEASE_VERSION }} -m "Development release" | |
| git push origin ${{ github.ref_name }} | |
| - name: Run GoReleaser | |
| uses: goreleaser/goreleaser-action@v6 | |
| with: | |
| distribution: goreleaser | |
| version: "~> v2" | |
| args: release --clean -f .goreleaser.dev.yaml | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} | |
| SHORT_SHA: ${{ env.SHORT_SHA }} | |
| - name: Publish release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh release edit "${{ env.RELEASE_VERSION }}" --draft=false --latest=false --prerelease | |
| e2e-install: | |
| name: E2E Install (${{ matrix.arch }}) | |
| needs: [cell-build, docker-manifest] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: ubuntu-latest | |
| arch: amd64 | |
| - runner: ubuntu-24.04-arm | |
| arch: arm64 | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Download cell binary | |
| run: | | |
| TARBALL="cell-linux-${{ matrix.arch }}.tar.gz" | |
| URL="https://github.com/DimmKirr/devcell/releases/download/v0.0.0/${TARBALL}" | |
| echo "Downloading: ${URL}" | |
| curl -fsSL -o "${TARBALL}" "${URL}" | |
| tar xzf "${TARBALL}" | |
| chmod +x cell | |
| sudo mv cell /usr/local/bin/cell | |
| - name: Verify cell binary | |
| run: | | |
| cell --help | |
| echo "--- cell binary OK ---" | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run cell claude --version (full pipeline) | |
| env: | |
| DEVCELL_NIXHOME_PATH: ${{ github.workspace }}/nixhome | |
| run: | | |
| # Simulate a new user in a fresh project dir | |
| mkdir -p /tmp/e2e-project && cd /tmp/e2e-project | |
| # --plain-text: disable spinners for CI | |
| # Flow: scaffold devcell.toml → pull base image → build user image → start container → print claude version | |
| OUTPUT=$(cell --plain-text claude --version 2>&1) || true | |
| echo "$OUTPUT" | |
| # Assert cell version string is present | |
| if echo "$OUTPUT" | grep -qE "cell\s+v?[0-9]+\.[0-9]+\.[0-9]+"; then | |
| echo "--- PASS: cell version found ---" | |
| else | |
| echo "--- FAIL: cell version string not found in output ---" | |
| exit 1 | |
| fi | |
| # Assert the image was built (user image should exist now) | |
| if echo "$OUTPUT" | grep -qiE "(building|built|image)"; then | |
| echo "--- PASS: image build occurred ---" | |
| else | |
| echo "--- WARN: no build output detected (image may have been cached) ---" | |
| fi | |
| brew-install: | |
| name: Brew Install (${{ matrix.arch }}) | |
| needs: [cell-build] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: ubuntu-latest | |
| arch: amd64 | |
| - runner: ubuntu-24.04-arm | |
| arch: arm64 | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Install Homebrew | |
| run: | | |
| /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" < /dev/null | |
| echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH | |
| - name: Brew install devcell-dev | |
| run: | | |
| brew tap devcell-sh/tap | |
| # Homebrew 4.5+ refuses to load casks from third-party taps without | |
| # explicit trust. `brew trust devcell-sh/tap` trusts every cask in | |
| # the tap so non-interactive installs succeed. | |
| brew trust devcell-sh/tap | |
| brew install --cask devcell-dev || true | |
| # Verify binary was actually linked despite potential broken pipe | |
| if ! command -v cell &>/dev/null; then | |
| echo "Binary not found, retrying..." | |
| brew install --cask devcell-dev | |
| fi | |
| - name: Verify version | |
| run: | | |
| INSTALLED=$(cell --version) | |
| echo "Installed: ${INSTALLED}" | |
| if echo "${INSTALLED}" | grep -qE "^cell version [0-9]+\.[0-9]+\.[0-9]+"; then | |
| echo "--- PASS: cell version OK ---" | |
| else | |
| echo "--- FAIL: unexpected version output ---" | |
| exit 1 | |
| fi |