Dev

.github/workflows/build.yml

@@ -1,96 +1,229 @@
-# Build the sovereign `lem` binary for every target and ship the zips.
-# One binary named `lem` per platform (the GPU backend — metal/rocm/cuda — loads
-# its kernel sidecar at runtime); targets are separated by build folder, not by
-# binary name. Zips flow to a rolling `dev` prerelease.
+# Build the sovereign `lem` binary for every hosted-buildable cell and ship the zips.
+#
+# lem links DuckDB (go-store/chathistory) via cgo, so there is NO pure-Go build and
+# NO cheap cross-compile: each cell is a NATIVE cgo build on a matching runner.
+# GitHub owns the portable cpu cells it can build natively; the ROCm/CUDA cells
+# (amd, cuda) stay on GitLab's homelab runner (it has the toolchain); the metal
+# cell needs a macOS 26 / Metal 4 box to compile the metallibs, so it rides a
+# self-hosted Mac lane (dormant until ENABLE_MACOS_METAL is set).
+#
+# Every build ships in two packagings of the same binary (see docs/release-artifacts.md):
+#   Native  {os}-{arch}-lem-{backend}-{version}.zip          -> binary `lem`
+#   Driver  {os}-{arch}-lem-driver-{backend}-{version}.zip   -> binary `lem-{backend}`
+# Zips flow to a rolling `dev` prerelease; version is `dev-<short-sha>`.
 name: Build
 
 on:
   push:
     branches: [dev, main]
+    tags: ['v*']            # a version tag publishes the REAL release (guarded on-main)
   workflow_dispatch:
 
 permissions:
   contents: read
 
+env:
+  GO_VERSION: "1.26"
+
 jobs:
-  build:
-    name: lem · ${{ matrix.goos }}/${{ matrix.goarch }}
-    runs-on: ${{ matrix.os }}
+  # ---- cpu cells: native cgo, one per hosted os/arch (duckdb ships prebuilt libs
+  #      for darwin/{amd64,arm64}, linux/{amd64,arm64}, windows/amd64 — NOT
+  #      windows/arm64, which is therefore not a cell). The workspace (go.work)
+  #      wires cli -> local go/; GOWORK=off cannot build the binary because cli
+  #      pins a published go/inference tag that lags the local tree. ----------------
+  cpu:
+    name: lem cpu · ${{ matrix.os_name }}/${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
     strategy:
       fail-fast: false
       matrix:
         include:
-          - { os: macos-latest,     goos: darwin,  goarch: arm64, metal: true }
-          - { os: ubuntu-latest,    goos: linux,   goarch: amd64 }
-          - { os: ubuntu-24.04-arm, goos: linux,   goarch: arm64 }
-          - { os: windows-latest,   goos: windows, goarch: amd64 }
-    env:
-      CGO_ENABLED: "1"
-      GOWORK: "off"
+          # Dropped cells, proven by live runs 2026-07-22 (docs/release-artifacts.md):
+          #  - macos/x86_64: the macos-13 runner label is retired (queued forever) —
+          #    no hosted Intel Mac remains; revisit only on real demand.
+          #  - windows/x86_64: lem's code is not Windows-portable yet (unix-only
+          #    syscall.Kill + raw-fd reads outside build constraints) — a code
+          #    campaign, not a CI matrix entry; restore the cell when it compiles.
+          - { runner: macos-latest,     os_name: macos,   arch: aarch64, exe: "" }
+          - { runner: ubuntu-latest,    os_name: linux,   arch: x86_64,  exe: "" }
+          - { runner: ubuntu-24.04-arm, os_name: linux,   arch: aarch64, exe: "" }
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive          # metal needs external/mlx
+        # cpu build needs neither external/mlx (metal) nor the ROCm sources.
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
-
-      - name: Install Task
-        run: go install github.com/go-task/task/v3/cmd/task@latest
+          go-version: ${{ env.GO_VERSION }}
+          # go.sum lives in the modules, not the repo root; list both so setup-go's
+          # cache key is honest (build touches cli, cli requires local go/).
+          cache-dependency-path: |
+            go/go.sum
+            cli/go.sum
 
-      - name: Build lem
+      - name: Build lem (native cgo; workspace wires cli -> local go/)
         shell: bash
         env:
-          METAL: ${{ matrix.metal }}
+          CGO_ENABLED: "1"
+          EXE: ${{ matrix.exe }}
         run: |
-          set -e
-          if [ "$METAL" = "true" ]; then
-            task metallib && task build:embed     # self-contained metal lem
-          else
-            task build:native                     # plain lem (GPU sidecar loads at runtime)
-          fi
+          set -euo pipefail
+          if [[ "$GITHUB_REF" == refs/tags/v* ]]; then VERSION="$GITHUB_REF_NAME"; else VERSION="dev-${GITHUB_SHA:0:12}"; fi
+          mkdir -p bin
+          # main.backend is "cpu" for every cell this job builds (the other backends —
+          # metal, amd, cuda — are built by their own dedicated jobs/CI); lem's
+          # self-update asset-selection reads it to pick its own release asset.
+          go build -trimpath -ldflags "-X main.version=${VERSION} -X main.backend=cpu" -o "bin/lem${EXE}" ./cli
+          test -s "bin/lem${EXE}"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: lem-cpu-${{ matrix.os_name }}-${{ matrix.arch }}
+          path: bin/lem${{ matrix.exe }}
+          if-no-files-found: error
+
+  # ---- metal cell: macos/aarch64, self-contained (metallibs embedded). Hosted
+  #      macOS runners are < macOS 26 and cannot compile the Metal 4 metallibs, so
+  #      a plain build would ship a metallib-less, non-runnable binary. This lane is
+  #      DORMANT on normal pushes (the `if` is false when the variable is unset, so
+  #      it never queues or fails): register a self-hosted macOS 26 arm64 runner and
+  #      set repo variable ENABLE_MACOS_METAL=true to activate it. Interim: the metal
+  #      artifact is built manually on the owner's Mac (task metallib && task build:embed).
+  metal:
+    name: lem metal · macos/aarch64 (self-hosted macOS 26)
+    if: ${{ vars.ENABLE_MACOS_METAL == 'true' }}
+    runs-on: [self-hosted, macOS, ARM64]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive          # external/mlx for `task metallib`
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: |
+            go/go.sum
+            cli/go.sum
+
+      - name: Install Task
+        run: go install github.com/go-task/task/v3/cmd/task@latest
 
-      - name: Assemble build folder (build/dist/<target>/lem)
+      - name: Build self-contained metal lem (metallibs baked in)
         shell: bash
-        env:
-          TARGET: ${{ matrix.goos }}-${{ matrix.goarch }}
         run: |
-          set -e
-          mkdir -p "build/dist/$TARGET"
-          cp bin/lem* "build/dist/$TARGET/"
+          set -euo pipefail
+          if [[ "$GITHUB_REF" == refs/tags/v* ]]; then VERSION="$GITHUB_REF_NAME"; else VERSION="dev-${GITHUB_SHA:0:12}"; fi
+          task metallib
+          # GO_VERSION_LDFLAGS is Taskfile.yml's own var, threaded into build:embed's
+          # -ldflags; overriding it here on the CLI (Task's documented per-invocation
+          # override, e.g. the Taskfile's own "task build VERSION=1.2.3" example) folds
+          # in main.backend=metal without touching Taskfile.yml, whose ldflags string
+          # otherwise only knows about main.version.
+          task build:embed VERSION="${VERSION}" GO_VERSION_LDFLAGS="-X main.version=${VERSION} -X main.backend=metal"
+          test -s bin/lem
 
       - uses: actions/upload-artifact@v4
         with:
-          name: lem-${{ matrix.goos }}-${{ matrix.goarch }}
-          path: build/dist/${{ matrix.goos }}-${{ matrix.goarch }}/
+          name: lem-metal-macos-aarch64
+          path: bin/lem
+          if-no-files-found: error
 
+  # ---- release: package each built binary under BOTH names (build once, package
+  #      twice). Two publish shapes: a dev push feeds the rolling `dev`
+  #      prerelease; a v* TAG (guarded: the tagged commit must be on main)
+  #      publishes the real versioned release. Requires cpu green; tolerates
+  #      metal skipped. ----
   release:
-    name: Zip + rolling dev prerelease
-    needs: build
-    if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+    name: Zip (Native + Driver) + release
+    needs: [cpu, metal]
+    if: ${{ !cancelled() && needs.cpu.result == 'success' && github.event_name == 'push' && (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')) }}
     runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0     # the tag-on-main ancestry guard needs history
+
+      # A tag release only publishes if the tagged commit is reachable from
+      # main — "a tag hitting main". A tag pushed off-branch fails LOUDLY
+      # (the run goes red) rather than silently minting a release.
+      - name: Guard — tag must be on main
+        if: startsWith(github.ref, 'refs/tags/v')
+        shell: bash
+        run: |
+          set -euo pipefail
+          git fetch -q origin main
+          if git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then
+            echo "tag $GITHUB_REF_NAME is on main — releasing"
+          else
+            echo "::error::tag $GITHUB_REF_NAME ($GITHUB_SHA) is NOT on main — refusing to release"
+            exit 1
+          fi
+
+      - name: Resolve version + release shape
+        id: shape
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
+            echo "version=$GITHUB_REF_NAME"      >> "$GITHUB_OUTPUT"
+            echo "tag=$GITHUB_REF_NAME"          >> "$GITHUB_OUTPUT"
+            echo "name=$GITHUB_REF_NAME"         >> "$GITHUB_OUTPUT"
+            # A tag containing '-' (v1.2.3-beta.1, v1.2.3-alpha.2, even a bare
+            # v2.0.0-rc.1) is a semver prerelease — publish it as a GitHub
+            # prerelease so go-update's channel filter (alpha/beta substring,
+            # else the prerelease flag) classifies it correctly instead of
+            # every tag looking "stable".
+            if [[ "$GITHUB_REF_NAME" == *-* ]]; then
+              echo "prerelease=true"             >> "$GITHUB_OUTPUT"
+            else
+              echo "prerelease=false"            >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "version=dev-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+            echo "tag=dev"                        >> "$GITHUB_OUTPUT"
+            echo "name=dev (rolling)"             >> "$GITHUB_OUTPUT"
+            echo "prerelease=true"                >> "$GITHUB_OUTPUT"
+          fi
+
       - uses: actions/download-artifact@v4
         with:
-          path: dist        # one subfolder per target (lem-<goos>-<goarch>/lem)
+          path: dist        # one dir per cell: dist/lem-<backend>-<os>-<arch>/lem[.exe]
 
-      - name: Zip each target
+      - name: Package Native + Driver zips
+        shell: bash
+        env:
+          VERSION: ${{ steps.shape.outputs.version }}
         run: |
-          set -e
+          set -euo pipefail
+          mkdir -p out
           cd dist
-          for d in */; do (cd "$d" && zip -r "../${d%/}.zip" .); done
-          ls -la *.zip
+          # ONLY the binary artifacts (lem-cpu-*, lem-metal-*) — lem-sdks also
+          # lands in dist/ and must never enter the packaging loop.
+          for d in lem-cpu-*/ lem-metal-*/; do
+            [ -d "$d" ] || continue                       # unmatched glob (metal skipped)
+            name="${d%/}"                                 # e.g. lem-cpu-macos-aarch64
+            IFS=- read -r _ backend os arch <<< "$name"   # backend/os/arch from the name
+            bin="$(ls "$d")"                              # lem or lem.exe
+            ext=""; case "$bin" in *.exe) ext=".exe";; esac
+            # Native — the binary a user runs locally, named `lem`.
+            cp "$d/$bin" "lem${ext}"; chmod 0755 "lem${ext}"
+            zip -jq "../out/${os}-${arch}-lem-${backend}-${VERSION}.zip" "lem${ext}"
+            rm -f "lem${ext}"
+            # Driver — the binary a compute box serves over the API, named `lem-<backend>`.
+            cp "$d/$bin" "lem-${backend}${ext}"; chmod 0755 "lem-${backend}${ext}"
+            zip -jq "../out/${os}-${arch}-lem-driver-${backend}-${VERSION}.zip" "lem-${backend}${ext}"
+            rm -f "lem-${backend}${ext}"
+          done
+          ls -la ../out
 
-      - name: Publish rolling dev prerelease
+      - name: Publish release
         uses: softprops/action-gh-release@v2
         with:
-          tag_name: dev
-          name: dev (rolling)
-          prerelease: true
-          files: dist/*.zip
+          tag_name: ${{ steps.shape.outputs.tag }}
+          name: ${{ steps.shape.outputs.name }}
+          prerelease: ${{ steps.shape.outputs.prerelease }}
+          files: out/*.zip
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -101,7 +234,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: |
+            go/go.sum
+            cli/go.sum
       - uses: actions/setup-node@v4
         with:
           node-version: "20"
@@ -111,7 +247,7 @@ jobs:
           java-version: "21"
       - name: Install generators
         run: |
-          npm install -g @openapitools/openapi-generator-cli@7.22.0
+          npm install -g @openapitools/openapi-generator-cli@2.40.0   # npm wrapper is 2.x; it manages the 7.x Java generator
           go install github.com/go-task/task/v3/cmd/task@latest
       - name: Generate SDKs (lem spec -> OpenAPI 3.1 -> typed clients)
         run: task sdk

.github/workflows/ci.yml

@@ -1,7 +1,10 @@
-# Portable CI for go-inference — the default-tag (no-GPU) test + coverage lane.
-# go-inference is a Metal/CGO project; the FULL lem build (engine/metal, the
-# metallibs) needs macOS 26 / Metal 4 and lives in build.yml on a self-hosted
-# runner. This lane runs the portable suite (Taskfile `test`/`cover`) on any OS.
+# PR gate + push CI for go-inference — the model-free test surface on all three
+# desktop OSes. The suite is testable without a model loaded (engine/metal skips
+# cleanly without MLX_METALLIB_PATH; engine/hip's contract tests are host-side),
+# so this lane is the go/no-go for PRs into dev: linux + macos are REQUIRED
+# checks; windows is EXPERIMENTAL (continue-on-error) until the cgo-on windows
+# graph is proven — its logs name the real portability offenders either way.
+# The full artifact build (and the metal lane) lives in build.yml.
 name: CI
 
 on:
@@ -13,25 +16,59 @@ on:
 permissions:
   contents: read
 
+env:
+  GO_VERSION: "1.26"
+
 jobs:
   test:
-    name: Portable test + coverage
-    runs-on: ubuntu-latest
+    name: tests · ${{ matrix.os_name }}
+    runs-on: ${{ matrix.runner }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { runner: ubuntu-latest,  os_name: linux,   experimental: false }
+          - { runner: macos-latest,   os_name: macos,   experimental: false }
+          - { runner: windows-latest, os_name: windows, experimental: true }
     env:
-      GOWORK: "off"          # resolve deps from go/go.mod tags, not the workspace
-      CGO_ENABLED: "1"       # go/ links duckdb (go-duckdb/v2 ships prebuilt linux libs)
+      CGO_ENABLED: "1"        # go/ links duckdb (prebuilt libs for all three OSes)
     steps:
       - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.26"
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: |
+            go/go.sum
+            cli/go.sum
 
-      - name: Portable tests + coverage (default tags — engine/metal excluded)
+      # The module suite resolves from go.mod tags (GOWORK=off) — proves the
+      # published dependency surface, not the workspace.
+      - name: Module suite (default tags — engine/metal skips without a metallib)
         working-directory: go
+        env:
+          GOWORK: "off"
+        shell: bash
         run: go test -count=1 -covermode=atomic -coverprofile=coverage.out ./...
 
+      # cli + examples need the workspace (cli requires the LOCAL go/ tree; the
+      # examples module is the acceptance surface — every public feature ships
+      # with its example, so compiling them all catches API breakage model-free).
+      - name: TUI suite (workspace)
+        working-directory: cli
+        shell: bash
+        run: go test -count=1 ./tui/...
+
+      - name: Examples compile + vet (workspace — the acceptance surface)
+        working-directory: examples
+        shell: bash
+        run: |
+          go build ./...
+          go vet ./...
+
       - name: Upload coverage to Codecov
+        if: matrix.os_name == 'linux'
         uses: codecov/codecov-action@v5
         with:
           files: go/coverage.out

Taskfile.yml

@@ -18,8 +18,13 @@ vars:
   VERSION: dev
   GO_VERSION_LDFLAGS: '-X main.version={{.VERSION}}'
   GO_DARWIN_LDFLAGS: '-extldflags=-mmacosx-version-min=26.0'
+  # Core count for cmake --parallel (metallib:mlx only). go-task evaluates global
+  # sh-vars eagerly on every invocation, so this must succeed on EVERY platform:
+  # getconf/nproc cover Linux, sysctl -n hw.ncpu is the macOS path, and the `echo 4`
+  # floor keeps it green where none of the three exist (Windows, task's internal
+  # POSIX shell has no such binaries) — a bare `sysctl -n hw.ncpu` was the CI failure.
   NCPU:
-    sh: getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || sysctl -n hw.ncpu
+    sh: getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4
 env:
   MLX_METALLIB_PATH: '{{.ROOT_DIR}}/build/dist/lib/mlx.metallib'