Repository navigation
Merge pull request #38 from devcell-sh/feature/wip #232
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dev Build | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - feature/wip | |
| - feature/add-web | |
| - feature/web | |
| workflow_dispatch: | |
| inputs: | |
| skip_nix_cache: | |
| description: 'Skip nix cache (genesis mode — full rebuild, no pre-seeding)' | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: write | |
| packages: write | |
| id-token: write | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| secrets: | |
| name: Detect Secrets | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| # gitleaks CLI directly — bypasses gitleaks-action's org-license | |
| # requirement (the license gate lives in the action wrapper, not the | |
| # scanner). `detect` exits 1 if leaks are found. | |
| - name: Run gitleaks | |
| run: | | |
| docker run --rm -v "$PWD:/repo" -w /repo \ | |
| ghcr.io/gitleaks/gitleaks:latest \ | |
| detect --source=. --verbose --redact | |
| docker-build: | |
| name: Docker Build (${{ matrix.arch }}) | |
| needs: secrets | |
| strategy: | |
| fail-fast: false | |
| # Serialize amd64 + arm64 to avoid stressing GHCR + cache.nixos.org | |
| # with two concurrent multi-GB push/pull pipelines from the same | |
| # repo. Doubles wall-clock; halves peak network pressure. | |
| max-parallel: 1 | |
| matrix: | |
| include: | |
| # amd64 temporarily disabled while we debug the post-base | |
| # cache-publish hang on arm64. Re-enable once arm64 is healthy. | |
| # - runner: ubuntu-latest | |
| # arch: amd64 | |
| # platform: linux/amd64 | |
| - runner: blacksmith-4vcpu-ubuntu-2404-arm | |
| arch: arm64 | |
| platform: linux/arm64 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune -af | |
| df -h | |
| - name: Setup Blacksmith Builder | |
| uses: useblacksmith/setup-docker-builder@v1 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: go.sum | |
| - name: Install go-task | |
| uses: arduino/setup-task@v2 | |
| with: | |
| version: 3.x | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # task cell:build = swagger:generate → CGO_ENABLED=0 go build with ldflags | |
| # for GitCommit + BuildDate. Single source of truth shared with local dev | |
| # (`task cell:build` works the same on a developer machine). | |
| - name: Build cell binary | |
| run: task cell:build | |
| # Cache push/pull goes through `cell nix-store {push,pull}` — a | |
| # cell subcommand using `pkg/v1/stream.NewLayer` from | |
| # go-containerregistry. Layer is streamed (no temp file) and | |
| # always single-gzipped on the wire — see CELL-293. | |
| # Cache the nix-store as a GHCR image. Lives in the SAME | |
| # `devcell-sh/devcell` GHCR package as the runtime images, with tag | |
| # prefix `nix-cache-` so it doesn't collide with stack tags. | |
| - name: Stream-hydrate /nix volume from prior GHCR cache (if available) | |
| if: inputs.skip_nix_cache != true | |
| run: | | |
| set -eo pipefail | |
| HASH=${{ hashFiles('nixhome/**') }} | |
| EXACT="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-${HASH}" | |
| LATEST="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-latest" | |
| # Pick the first candidate that resolves a manifest. `cell | |
| # nix-store pull` itself fails cleanly on a missing/poisoned | |
| # cache, and the workflow falls back to MISS — no upfront | |
| # size probe needed. | |
| IMG="" | |
| for c in "$EXACT" "$LATEST"; do | |
| if docker manifest inspect "$c" >/dev/null 2>&1; then | |
| IMG="$c" | |
| break | |
| fi | |
| done | |
| if [ -z "$IMG" ]; then | |
| echo "cache MISS — starting from empty volume" | |
| exit 0 | |
| fi | |
| echo "cache HIT: $IMG" | |
| # `cell nix-store pull` streams $IMG's last layer directly | |
| # into the named docker volume (internal gunzip + tar -x). | |
| # On failure, fall back to MISS — the next publish overwrites | |
| # the cache with the current encoding. | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| if ./bin/cell nix-store pull \ | |
| --image "$IMG" \ | |
| --volume "devcell-nix-store-${{ matrix.arch }}"; then | |
| docker run --rm -v devcell-nix-store-${{ matrix.arch }}:/nix:ro alpine \ | |
| sh -c 'echo "hydrated $(du -sh /nix | cut -f1)"' | |
| else | |
| echo "WARN: cache pull failed (likely encoding mismatch with legacy cache image)" | |
| echo " falling back to MISS — will rebuild and publish a fresh cache" | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| fi | |
| # Build both stacks sequentially in the same job so the nix-store | |
| # volume accumulates derivations from both — single tar dump at job | |
| # end carries everything needed by docker-test. `cell build --thin` | |
| # reuses store paths across the two invocations (nix is | |
| # content-addressed), so ultimate after base is incremental. | |
| - name: Build thin image (base stack) | |
| env: | |
| DEVCELL_NIX_VOLUME: devcell-nix-store-${{ matrix.arch }} | |
| DEVCELL_NIXHOME_PATH: ${{ github.workspace }}/nixhome | |
| run: | | |
| BASE_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-base" | |
| ./bin/cell build --thin --stack base --image "$BASE_TAG" --debug | |
| echo "BASE_TAG=$BASE_TAG" >> "$GITHUB_ENV" | |
| # Interim publish: lock in the base-stack derivations as a usable | |
| # cache. If the ultimate build below fails catastrophically (OOM, | |
| # runner kill, CDN flake), the next run still hydrates the base | |
| # cache and skips re-deriving everything in base. Same semantics | |
| # as the final publish — runs on prior-step failure, skipped when | |
| # cache-bust mode is on or volume is empty. | |
| - name: Publish nix cache (post-base, interim) | |
| if: ${{ !cancelled() && inputs.skip_nix_cache != true }} | |
| env: | |
| ARCH: ${{ matrix.arch }} | |
| HASH: ${{ hashFiles('nixhome/**') }} | |
| STAGE: post-base | |
| run: task nix-cache:publish | |
| - name: Build thin image (ultimate stack) | |
| env: | |
| DEVCELL_NIX_VOLUME: devcell-nix-store-${{ matrix.arch }} | |
| DEVCELL_NIXHOME_PATH: ${{ github.workspace }}/nixhome | |
| run: | | |
| ULT_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate" | |
| ./bin/cell build --thin --stack ultimate --image "$ULT_TAG" --debug | |
| echo "ULT_TAG=$ULT_TAG" >> "$GITHUB_ENV" | |
| - name: Push to GHCR (both stacks) | |
| run: | | |
| docker push "$BASE_TAG" | |
| docker push "$ULT_TAG" | |
| # Final publish: re-pushes after ultimate also lands derivations | |
| # into /nix. Overwrites the interim post-base cache with the | |
| # fuller closure. Idempotent — same composite action as above. | |
| - name: Publish nix cache (post-ultimate, final) | |
| if: ${{ !cancelled() && inputs.skip_nix_cache != true }} | |
| env: | |
| ARCH: ${{ matrix.arch }} | |
| HASH: ${{ hashFiles('nixhome/**') }} | |
| STAGE: post-ultimate | |
| run: task nix-cache:publish | |
| docker-test: | |
| name: Docker Test (${{ matrix.arch }}) | |
| needs: docker-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # amd64 disabled in step with docker-build above. | |
| # - runner: ubuntu-latest | |
| # arch: amd64 | |
| - runner: blacksmith-4vcpu-ubuntu-2404-arm | |
| arch: arm64 | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: go.sum | |
| - name: Install go-task | |
| uses: arduino/setup-task@v2 | |
| with: | |
| version: 3.x | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Build cell so we can use `cell nix-store pull` below — same | |
| # binary docker-build's publish step uses, so any encoding | |
| # divergence between push and pull is structurally impossible. | |
| - name: Build cell binary | |
| run: task cell:build | |
| # Stream-pull the nix-store cache image's last layer into the | |
| # per-arch volume via `cell nix-store pull` — same binary the | |
| # publish step in docker-build uses, so push/pull encoding | |
| # divergence is impossible by construction. | |
| - name: Stream-pull /nix cache from GHCR into volume | |
| run: | | |
| set -eo pipefail | |
| HASH=${{ hashFiles('nixhome/**') }} | |
| EXACT="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-${HASH}" | |
| LATEST="${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:nix-cache-${{ matrix.arch }}-latest" | |
| # Pick the first candidate that resolves a manifest. cell | |
| # nix-store pull fails cleanly on a missing cache, and the | |
| # workflow falls back to MISS — no upfront probe needed. | |
| IMG="" | |
| for c in "$EXACT" "$LATEST"; do | |
| if docker manifest inspect "$c" >/dev/null 2>&1; then | |
| IMG="$c" | |
| break | |
| fi | |
| done | |
| if [ -z "$IMG" ]; then | |
| echo "cache MISS — tests will run on empty /nix volume (lazy build during cell shell)" | |
| exit 0 | |
| fi | |
| echo "cache HIT: $IMG" | |
| # If pull fails, tests fall back to an empty volume (lazy | |
| # build during cell shell). | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| if ! ./bin/cell nix-store pull \ | |
| --image "$IMG" \ | |
| --volume "devcell-nix-store-${{ matrix.arch }}"; then | |
| echo "WARN: cache pull failed — tests will run on empty /nix" | |
| docker volume rm devcell-nix-store-${{ matrix.arch }} 2>/dev/null || true | |
| docker volume create devcell-nix-store-${{ matrix.arch }} >/dev/null | |
| fi | |
| - name: Pull test images (base + ultimate) | |
| run: | | |
| docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-base | |
| docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate | |
| - name: Run container tests | |
| # Both `DEVCELL_USER_IMAGE` and `DEVCELL_TEST_IMAGE` point at the | |
| # ultimate image (every module + tool baked in) so module-aware tests | |
| # (kicad, plex, gui) find what they need. `DEVCELL_TEST_BASE_IMAGE` | |
| # points at the smaller base image for entrypoint-only tests | |
| # (TestEntrypoint_DebugTimestamps etc.). Setting `DEVCELL_USER_IMAGE` | |
| # without `DEVCELL_USER_IMAGE_THIN` works because UserImageTagThin() | |
| # falls back to DEVCELL_USER_IMAGE as-is (no `-thin` suffix appended) | |
| # per CELL-286 prep. | |
| # `DEVCELL_NIX_VOLUME` points cell at the per-arch volume we just | |
| # hydrated from the GHCR cache, so `cell shell` / `cell claude` skip | |
| # the lazy-build path entirely instead of falling back to the | |
| # default-named volume (which would be empty). | |
| env: | |
| DEVCELL_NIX_VOLUME: devcell-nix-store-${{ matrix.arch }} | |
| DEVCELL_TEST_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate | |
| DEVCELL_USER_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-ultimate | |
| DEVCELL_TEST_BASE_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:v0.0.0-${{ matrix.arch }}-base | |
| DEVCELL_TEST_PROJECT_DIR: ${{ runner.temp }}/devcell-test | |
| MCP_SECRET_TEST_PASSWORD: fake-secret-ci-value | |
| MCP_SECRET_GITHUB_TOKEN: fake-token-ci-value | |
| run: go test -v -timeout 1200s ./test/... | |
| docker-manifest: | |
| name: Docker Manifests | |
| needs: [docker-build] | |
| if: always() && needs.docker-build.result == 'success' | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| steps: | |
| - name: Set lowercase image name | |
| run: echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >> $GITHUB_ENV | |
| env: | |
| IMAGE_NAME: ${{ github.repository }} | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create GHCR manifests (per stack + ultimate as canonical) | |
| run: | | |
| R=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }} | |
| # amd64 inputs commented out in step with docker-build matrix | |
| # above — re-add the `${R}:v0.0.0-amd64-*` arguments when amd64 | |
| # is re-enabled. arm64-only manifests are still valid OCI | |
| # indexes; tags resolve cleanly on arm64 hosts. | |
| docker buildx imagetools create \ | |
| -t "${R}:v0.0.0-base" \ | |
| "${R}:v0.0.0-arm64-base" | |
| docker buildx imagetools create \ | |
| -t "${R}:v0.0.0-ultimate" \ | |
| -t "${R}:v0.0.0" \ | |
| -t "${R}:latest" \ | |
| -t "${R}:dev" \ | |
| "${R}:v0.0.0-arm64-ultimate" | |
| cell-build: | |
| name: Cell CLI Dev Build | |
| needs: secrets | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| env: | |
| RELEASE_VERSION: v0.0.0 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: go.sum | |
| - name: Set SHORT_SHA | |
| run: echo "SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-8)" >> $GITHUB_ENV | |
| - name: Delete existing tag | |
| run: | | |
| git config --global user.email "dmitry@atd.sh" | |
| git config --global user.name "Dmitry Kireev" | |
| git tag -d "${{ env.RELEASE_VERSION }}" || echo "No local tag to delete" | |
| git push origin :refs/tags/${{ env.RELEASE_VERSION }} || echo "No remote tag to delete" | |
| - name: Add tag | |
| run: | | |
| git tag -a ${{ env.RELEASE_VERSION }} -m "Development release" | |
| git push origin ${{ github.ref_name }} | |
| - name: Run GoReleaser | |
| uses: goreleaser/goreleaser-action@v6 | |
| with: | |
| distribution: goreleaser | |
| version: "~> v2" | |
| args: release --clean -f .goreleaser.dev.yaml | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} | |
| SHORT_SHA: ${{ env.SHORT_SHA }} | |
| - name: Publish release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh release edit "${{ env.RELEASE_VERSION }}" --draft=false --latest=false --prerelease | |
| e2e-install: | |
| name: E2E Install (${{ matrix.arch }}) | |
| needs: [cell-build, docker-manifest] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # amd64 disabled in step with docker-build matrix above. | |
| # - runner: ubuntu-latest | |
| # arch: amd64 | |
| - runner: blacksmith-4vcpu-ubuntu-2404-arm | |
| arch: arm64 | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Download cell binary | |
| run: | | |
| TARBALL="cell-linux-${{ matrix.arch }}.tar.gz" | |
| URL="https://github.com/DimmKirr/devcell/releases/download/v0.0.0/${TARBALL}" | |
| echo "Downloading: ${URL}" | |
| curl -fsSL -o "${TARBALL}" "${URL}" | |
| tar xzf "${TARBALL}" | |
| chmod +x cell | |
| sudo mv cell /usr/local/bin/cell | |
| - name: Verify cell binary | |
| run: | | |
| cell --help | |
| echo "--- cell binary OK ---" | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run cell claude --version (full pipeline) | |
| env: | |
| DEVCELL_NIXHOME_PATH: ${{ github.workspace }}/nixhome | |
| run: | | |
| # Simulate a new user in a fresh project dir | |
| mkdir -p /tmp/e2e-project && cd /tmp/e2e-project | |
| # --plain-text: disable spinners for CI | |
| # Flow: scaffold devcell.toml → pull base image → build user image → start container → print claude version | |
| OUTPUT=$(cell --plain-text claude --version 2>&1) || true | |
| echo "$OUTPUT" | |
| # Assert cell version string is present | |
| if echo "$OUTPUT" | grep -qE "cell\s+v?[0-9]+\.[0-9]+\.[0-9]+"; then | |
| echo "--- PASS: cell version found ---" | |
| else | |
| echo "--- FAIL: cell version string not found in output ---" | |
| exit 1 | |
| fi | |
| # Assert the image was built (user image should exist now) | |
| if echo "$OUTPUT" | grep -qiE "(building|built|image)"; then | |
| echo "--- PASS: image build occurred ---" | |
| else | |
| echo "--- WARN: no build output detected (image may have been cached) ---" | |
| fi | |
| brew-install: | |
| name: Brew Install (${{ matrix.arch }}) | |
| needs: [cell-build] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # amd64 disabled in step with docker-build matrix above. | |
| # - runner: ubuntu-latest | |
| # arch: amd64 | |
| - runner: blacksmith-4vcpu-ubuntu-2404-arm | |
| arch: arm64 | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Install Homebrew | |
| run: | | |
| /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" < /dev/null | |
| echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH | |
| - name: Brew install devcell-dev | |
| run: | | |
| brew tap devcell-sh/tap | |
| # Homebrew 4.5+ refuses to load casks from third-party taps without | |
| # explicit trust. `brew trust devcell-sh/tap` trusts every cask in | |
| # the tap so non-interactive installs succeed. | |
| brew trust devcell-sh/tap | |
| brew install --cask devcell-dev || true | |
| # Verify binary was actually linked despite potential broken pipe | |
| if ! command -v cell &>/dev/null; then | |
| echo "Binary not found, retrying..." | |
| brew install --cask devcell-dev | |
| fi | |
| - name: Verify version | |
| run: | | |
| INSTALLED=$(cell --version) | |
| echo "Installed: ${INSTALLED}" | |
| if echo "${INSTALLED}" | grep -qE "^cell version [0-9]+\.[0-9]+\.[0-9]+"; then | |
| echo "--- PASS: cell version OK ---" | |
| else | |
| echo "--- FAIL: unexpected version output ---" | |
| exit 1 | |
| fi |