Publish CI's Linux opam root for coding agents

.github/workflows/copilot-setup-steps.yml

@@ -18,6 +18,7 @@ jobs:
       PLATFORM: linux64
       OCAML_VERSION: 5.3.0
       OPAMYES: 1
+      OPAMROOT: /opt/opam
 
     steps:
       - name: Checkout repository
@@ -31,25 +32,27 @@ jobs:
           set -ex
           sudo apt-get update -qqy
           sudo DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Use-Pty=0 install -y \
-            opam libpcre2-dev zlib1g-dev libgtk2.0-dev libmbedtls-dev ninja-build \
+            opam libpcre2-dev zlib1g-dev libzstd-dev libgtk2.0-dev libmbedtls-dev ninja-build \
             cmake gcc-multilib g++-multilib \
             libturbojpeg-dev libvorbis-dev \
             libuv1-dev libreadline-dev
 
-      - name: Cache opam
-        id: cache-opam
-        uses: actions/cache@v4
-        with:
-          path: ~/.opam/
-          key: ${{ runner.os }}-${{ runner.arch }}-5.3.0-${{ hashFiles('./haxe.opam', './libs/') }}-1
+      # Neko has to come first: building the opam dependencies from source needs it (conf-neko).
+      - name: Setup Neko
+        uses: ./.github/actions/setup-neko
+        env:
+          PLATFORM: linux64
+
+      - name: Make Neko shared libraries available to the dynamic linker
+        run: |
+          echo "/usr/local/lib" | sudo tee /etc/ld.so.conf.d/local.conf
+          sudo ldconfig
 
-      - name: Initialize opam and install OCaml 5.3.0
-        if: steps.cache-opam.outputs.cache-hit != 'true'
+      # Downloads the prebuilt opam root published by CI, or builds it if there is none yet.
+      - name: Set up OCaml
         run: |
           set -ex
-          opam init -c 5.3.0 --no-setup
-          opam pin add haxe . --no-action
-          opam install haxe --deps-only --assume-depexts
+          extra/agent-setup.sh --no-apt --no-build --no-profile
           opam list
           ocamlopt -v
 
@@ -62,16 +65,6 @@ jobs:
           echo "CAML_LD_LIBRARY_PATH=$CAML_LD_LIBRARY_PATH" >> $GITHUB_ENV
           echo "OCAML_TOPLEVEL_PATH=$OCAML_TOPLEVEL_PATH" >> $GITHUB_ENV
 
-      - name: Setup Neko
-        uses: ./.github/actions/setup-neko
-        env:
-          PLATFORM: linux64
-
-      - name: Make Neko shared libraries available to the dynamic linker
-        run: |
-          echo "/usr/local/lib" | sudo tee /etc/ld.so.conf.d/local.conf
-          sudo ldconfig
-
       - name: Build Haxe
         run: |
           set -ex

.github/workflows/main.yml

@@ -70,17 +70,22 @@ jobs:
     env:
       PLATFORM: linux${{ matrix.arch == 'arm64' && '-arm' || '' }}64
       OPAMYES: 1
+      OPAMROOT: /opt/opam
     steps:
       - uses: actions/checkout@v7
         with:
           submodules: recursive
 
+      # The opam root lives at a fixed path so that agents can reuse it (see below).
+      - name: Create opam root
+        run: sudo mkdir -p $OPAMROOT && sudo chown $USER $OPAMROOT
+
       - name: Cache opam
         id: cache-opam
         uses: actions/cache@v6
         with:
-          path: ~/.opam/
-          key: ${{ runner.os }}-${{ runner.arch }}-${{ env.OCAML_VERSION }}-${{ hashFiles('./haxe.opam', './libs/') }}-1
+          path: /opt/opam/
+          key: ${{ runner.os }}-${{ runner.arch }}-${{ env.OCAML_VERSION }}-${{ hashFiles('./haxe.opam', './libs/') }}-2
 
       - uses: ./.github/actions/setup-neko
 
@@ -100,6 +105,21 @@ jobs:
           opam list
           ocamlopt -v
 
+      # Coding agents outside GitHub Actions can't use the cache, so development also
+      # publishes the opam root for extra/agent-setup.sh (see publish-opam-root).
+      - name: Package the opam root for agents
+        if: github.event_name == 'push' && github.ref == 'refs/heads/development'
+        run: tar -C /opt --zstd -cf opam-root.tar.zst opam
+
+      - name: Upload the opam root for agents
+        if: github.event_name == 'push' && github.ref == 'refs/heads/development'
+        uses: actions/upload-artifact@v7
+        with:
+          name: opam-root-${{ matrix.arch == 'arm64' && 'aarch64' || 'x86_64' }}
+          path: opam-root.tar.zst
+          retention-days: 1
+          compression-level: 0
+
       - name: Set ADD_REVISION=1 for non-release
         if: ${{ !startsWith(github.ref, 'refs/tags/') }}
         run: echo "ADD_REVISION=1" >> $GITHUB_ENV
@@ -146,6 +166,55 @@ jobs:
           name: xmldoc
           path: extra/doc
 
+  # Publishes new opam roots from linux-build for coding agents (extra/agent-setup.sh),
+  # and removes the ones they replace, so the release holds one archive per architecture.
+  publish-opam-root:
+    needs: linux-build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/development'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: extra/agent-setup.sh
+
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: opam-root-*
+          path: assets
+
+      # Each artifact is named opam-root-<arch>; extra/agent-setup.sh names the asset it
+      # will look for. Upload it if it's missing, and drop the older ones for that arch.
+      - name: Upload to the release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          set -e
+          if ! gh release view opam-root >/dev/null 2>&1; then
+            gh release create opam-root --prerelease --target "$GITHUB_SHA" \
+              --title "Prebuilt OCaml dependencies" \
+              --notes "Prebuilt opam roots used by extra/agent-setup.sh. Updated automatically from CI's linux-build job."
+          fi
+          existing=$(gh release view opam-root --json assets -q '.assets[].name')
+          for dir in assets/opam-root-*; do
+            arch=${dir#assets/opam-root-}
+            asset=$(OPAM_ROOT_ARCH=$arch extra/agent-setup.sh --print-asset)
+            if grep -qxF "$asset" <<<"$existing"; then
+              echo "$asset is already published"
+              continue
+            fi
+            mv "$dir/opam-root.tar.zst" "$asset"
+            gh release upload opam-root "$asset"
+            for old in $existing; do
+              if [[ "$old" == *"-$arch-"*.tar.zst ]]; then
+                echo "Deleting $old"
+                gh release delete-asset opam-root "$old" -y
+              fi
+            done
+          done
+
   linux-test:
     needs: linux-build
     runs-on: ubuntu-22.04${{ matrix.arch == 'arm64' && '-arm' || '' }}
@@ -625,6 +694,9 @@ jobs:
 
       - name: Download build artifacts
         uses: actions/download-artifact@v8
+        with:
+          # Only the binaries; this skips e.g. the opam-root-* artifacts of linux-build.
+          pattern: '*Binaries'
 
       - name: Install rclone
         run: |

AGENTS.md

@@ -24,6 +24,12 @@ Path-scoped instructions with more detail live in `.github/instructions/`. Read
   - `opam exec -- make ADD_REVISION=1 -f Makefile.win -s -j haxe` - Typical Windows compiler build command used in CI/tasks
   - `dune build` - Build using dune directly
 
+### Setting Up a Fresh Container
+On Ubuntu 24.04 (e.g. a cloud agent container), run `extra/agent-setup.sh` from the repository root before anything else. It installs the system packages, puts a prebuilt OCaml 5 toolchain with all opam dependencies in `/opt/opam` (building it from source only if no prebuilt one matches), adds the opam environment to `~/.bashrc`, and runs `make haxe`. Don't set up OCaml or opam by hand.
+- If `ocamlfind` or `dune` aren't found in a new shell, run `export OPAMROOT=/opt/opam; eval "$(opam env)"`
+- Pass `--no-build` to only set up the toolchain
+- Building haxelib additionally needs the git submodules (`git submodule update --init --recursive`) before `make haxelib`
+
 ### Use the Compiler You Just Built
 This repository may be checked out in several git worktrees, and the global PATH can point to a `haxe` binary from a different checkout. A bare `haxe` command may therefore run a different compiler than the one you just built.
 - Call the compiler and haxelib from the repository root by path, e.g. `./haxe` and `./haxelib` (`.\haxe.exe` in PowerShell), or put the repository root first on PATH for the session

extra/agent-setup.sh

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+# Sets up an OCaml toolchain for building the Haxe compiler on Ubuntu, then builds it.
+# Meant for coding agents in fresh containers (Claude Code, Copilot, ...), but works for
+# anyone on Ubuntu 22.04 or newer.
+#
+# Building OCaml 5 and the opam dependencies from source takes 5-20 minutes, so the
+# script first tries to download the opam root that CI's linux-build job builds on
+# Ubuntu 22.04 and publishes from pushes to `development`, as an asset of the
+# `opam-root` prerelease. It only builds from source if no published archive matches.
+#
+# Usage: extra/agent-setup.sh [options]
+#   --from-source  never download the prebuilt opam root
+#   --no-apt       don't install system packages (the caller has done that)
+#   --no-build     only set up the toolchain, don't run `make haxe`
+#   --no-profile   don't add the opam environment to ~/.bashrc and ~/.profile
+#   --print-asset  print the file name of the matching prebuilt opam root and exit
+#   --print-url    print the download URL of the matching prebuilt opam root and exit
+#
+# Environment:
+#   OPAM_ROOT_RELEASE_URL  where to download the prebuilt opam root from
+#   OPAM_ROOT_ARCH  the architecture to name the archive for (default: uname -m)
+#   OPAMROOT  where the opam root lives (default /opt/opam; the prebuilt archive only
+#             works at that path, because opam roots are not relocatable)
+
+set -euo pipefail
+
+# Keep in sync with OCAML_VERSION in .github/workflows/main.yml.
+OCAML_VERSION=5.3.0
+# The OS the prebuilt archive is built on (linux-build's runner). It also works on newer
+# Ubuntu releases.
+BUILD_OS=ubuntu-22.04
+# Bump to publish a fresh archive, e.g. to pick up newer versions of the dependencies.
+REVISION=1
+DEFAULT_OPAMROOT=/opt/opam
+RELEASE_URL=${OPAM_ROOT_RELEASE_URL:-https://github.com/HaxeFoundation/haxe/releases/download/opam-root}
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export OPAMROOT=${OPAMROOT:-$DEFAULT_OPAMROOT}
+export OPAMYES=1 OPAMROOTISOK=1 OPAMCOLOR=never
+
+FROM_SOURCE=0 APT=1 BUILD=1 PROFILE=1 PRINT=
+for arg in "$@"; do
+	case "$arg" in
+		--from-source) FROM_SOURCE=1 ;;
+		--no-apt) APT=0 ;;
+		--no-build) BUILD=0 ;;
+		--no-profile) PROFILE=0 ;;
+		--print-asset) PRINT=asset ;;
+		--print-url) PRINT=url ;;
+		*) echo "unknown option: $arg" >&2; exit 2 ;;
+	esac
+done
+
+log() { echo "[agent-setup] $*" >&2; }
+
+# The key changes whenever something that affects the contents of the opam root changes.
+cache_key() {
+	{
+		echo "$OCAML_VERSION $BUILD_OS $REVISION"
+		cat "$REPO_ROOT/haxe.opam"
+	} | sha256sum | cut -c1-16
+}
+ASSET="opam-$OCAML_VERSION-$BUILD_OS-${OPAM_ROOT_ARCH:-$(uname -m)}-$(cache_key).tar.zst"
+
+case "$PRINT" in
+	asset) echo "$ASSET"; exit 0 ;;
+	url) echo "$RELEASE_URL/$ASSET"; exit 0 ;;
+esac
+
+SUDO=
+if [ "$(id -u)" != 0 ]; then SUDO=sudo; fi
+
+install_system_packages() {
+	log "installing system packages"
+	export DEBIAN_FRONTEND=noninteractive
+	$SUDO apt-get update -qq
+	$SUDO apt-get install -qqy --no-install-recommends \
+		opam git curl ca-certificates zstd make gcc g++ m4 pkg-config unzip bzip2 rsync patch \
+		libpcre2-dev zlib1g-dev libzstd-dev libmbedtls-dev neko neko-dev >/dev/null
+}
+
+opam_root_ready() {
+	[ -f "$OPAMROOT/config" ] && opam exec -- ocamlfind ocamlopt -version >/dev/null 2>&1 &&
+		opam exec -- dune --version >/dev/null 2>&1
+}
+
+download_prebuilt() {
+	if [ "$OPAMROOT" != "$DEFAULT_OPAMROOT" ]; then
+		log "OPAMROOT is not $DEFAULT_OPAMROOT, skipping the prebuilt download"
+		return 1
+	fi
+	log "downloading $RELEASE_URL/$ASSET"
+	local tmp
+	tmp=$(mktemp -d)
+	if ! curl -fsSL --retry 3 --retry-delay 2 -o "$tmp/$ASSET" "$RELEASE_URL/$ASSET"; then
+		log "no prebuilt opam root for this configuration"
+		rm -rf "$tmp"
+		return 1
+	fi
+	$SUDO mkdir -p "$(dirname "$OPAMROOT")"
+	$SUDO rm -rf "$OPAMROOT"
+	$SUDO tar -C "$(dirname "$OPAMROOT")" --zstd -xf "$tmp/$ASSET"
+	rm -rf "$tmp"
+	if [ -n "$SUDO" ]; then $SUDO chown -R "$(id -u):$(id -g)" "$OPAMROOT"; fi
+	if ! opam_root_ready; then
+		log "the prebuilt opam root doesn't work on this system"
+		$SUDO rm -rf "$OPAMROOT"
+		return 1
+	fi
+}
+
+# Same steps as linux-build in .github/workflows/main.yml.
+build_from_source() {
+	log "building OCaml $OCAML_VERSION and the dependencies from source (this takes a while)"
+	$SUDO mkdir -p "$OPAMROOT"
+	if [ -n "$SUDO" ]; then $SUDO chown "$(id -u):$(id -g)" "$OPAMROOT"; fi
+	# Reuse an existing opam root (e.g. a half-finished earlier run) instead of wiping it.
+	if [ ! -f "$OPAMROOT/config" ]; then
+		# Containers usually can't run opam's bubblewrap sandbox.
+		opam init -c "$OCAML_VERSION" --no-setup --disable-sandboxing
+	fi
+	opam pin add haxe "$REPO_ROOT" --no-action
+	opam install haxe --deps-only --assume-depexts
+	opam clean --all-switches --download-cache --logs --repo-cache
+}
+
+write_profile() {
+	local marker="# haxe agent-setup"
+	local block="$marker
+export OPAMROOT=$OPAMROOT OPAMROOTISOK=1
+eval \"\$(opam env 2>/dev/null)\""
+	for f in "$HOME/.bashrc" "$HOME/.profile"; do
+		if ! grep -qF "$marker" "$f" 2>/dev/null; then
+			printf '\n%s\n' "$block" >> "$f"
+			log "added the opam environment to $f"
+		fi
+	done
+}
+
+if [ "$APT" = 1 ]; then install_system_packages; fi
+if opam_root_ready; then
+	log "opam root at $OPAMROOT is already set up"
+elif [ "$FROM_SOURCE" = 1 ] || ! download_prebuilt; then
+	build_from_source
+fi
+eval "$(opam env)"
+log "OCaml $(ocamlfind ocamlopt -version), dune $(dune --version)"
+
+if [ "$PROFILE" = 1 ]; then write_profile; fi
+
+if [ "$BUILD" = 1 ]; then
+	log "building the compiler"
+	make -C "$REPO_ROOT" -s -j"$(nproc)" haxe
+	"$REPO_ROOT/haxe" -version
+fi
+log "done"