diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 4d4b93e44e8..c0c29886113 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -18,6 +18,7 @@ jobs: PLATFORM: linux64 OCAML_VERSION: 5.3.0 OPAMYES: 1 + OPAMROOT: /opt/opam steps: - name: Checkout repository @@ -31,25 +32,27 @@ jobs: set -ex sudo apt-get update -qqy sudo DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Use-Pty=0 install -y \ - opam libpcre2-dev zlib1g-dev libgtk2.0-dev libmbedtls-dev ninja-build \ + opam libpcre2-dev zlib1g-dev libzstd-dev libgtk2.0-dev libmbedtls-dev ninja-build \ cmake gcc-multilib g++-multilib \ libturbojpeg-dev libvorbis-dev \ libuv1-dev libreadline-dev - - name: Cache opam - id: cache-opam - uses: actions/cache@v4 - with: - path: ~/.opam/ - key: ${{ runner.os }}-${{ runner.arch }}-5.3.0-${{ hashFiles('./haxe.opam', './libs/') }}-1 + # Neko has to come first: building the opam dependencies from source needs it (conf-neko). + - name: Setup Neko + uses: ./.github/actions/setup-neko + env: + PLATFORM: linux64 + + - name: Make Neko shared libraries available to the dynamic linker + run: | + echo "/usr/local/lib" | sudo tee /etc/ld.so.conf.d/local.conf + sudo ldconfig - - name: Initialize opam and install OCaml 5.3.0 - if: steps.cache-opam.outputs.cache-hit != 'true' + # Downloads the prebuilt opam root published by CI, or builds it if there is none yet. + - name: Set up OCaml run: | set -ex - opam init -c 5.3.0 --no-setup - opam pin add haxe . --no-action - opam install haxe --deps-only --assume-depexts + extra/agent-setup.sh --no-apt --no-build --no-profile opam list ocamlopt -v @@ -62,16 +65,6 @@ jobs: echo "CAML_LD_LIBRARY_PATH=$CAML_LD_LIBRARY_PATH" >> $GITHUB_ENV echo "OCAML_TOPLEVEL_PATH=$OCAML_TOPLEVEL_PATH" >> $GITHUB_ENV - - name: Setup Neko - uses: ./.github/actions/setup-neko - env: - PLATFORM: linux64 - - - name: Make Neko shared libraries available to the dynamic linker - run: | - echo "/usr/local/lib" | sudo tee /etc/ld.so.conf.d/local.conf - sudo ldconfig - - name: Build Haxe run: | set -ex diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16bf7dc8c78..c2619473c25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,17 +70,22 @@ jobs: env: PLATFORM: linux${{ matrix.arch == 'arm64' && '-arm' || '' }}64 OPAMYES: 1 + OPAMROOT: /opt/opam steps: - uses: actions/checkout@v7 with: submodules: recursive + # The opam root lives at a fixed path so that agents can reuse it (see below). + - name: Create opam root + run: sudo mkdir -p $OPAMROOT && sudo chown $USER $OPAMROOT + - name: Cache opam id: cache-opam uses: actions/cache@v6 with: - path: ~/.opam/ - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.OCAML_VERSION }}-${{ hashFiles('./haxe.opam', './libs/') }}-1 + path: /opt/opam/ + key: ${{ runner.os }}-${{ runner.arch }}-${{ env.OCAML_VERSION }}-${{ hashFiles('./haxe.opam', './libs/') }}-2 - uses: ./.github/actions/setup-neko @@ -100,6 +105,21 @@ jobs: opam list ocamlopt -v + # Coding agents outside GitHub Actions can't use the cache, so development also + # publishes the opam root for extra/agent-setup.sh (see publish-opam-root). + - name: Package the opam root for agents + if: github.event_name == 'push' && github.ref == 'refs/heads/development' + run: tar -C /opt --zstd -cf opam-root.tar.zst opam + + - name: Upload the opam root for agents + if: github.event_name == 'push' && github.ref == 'refs/heads/development' + uses: actions/upload-artifact@v7 + with: + name: opam-root-${{ matrix.arch == 'arm64' && 'aarch64' || 'x86_64' }} + path: opam-root.tar.zst + retention-days: 1 + compression-level: 0 + - name: Set ADD_REVISION=1 for non-release if: ${{ !startsWith(github.ref, 'refs/tags/') }} run: echo "ADD_REVISION=1" >> $GITHUB_ENV @@ -146,6 +166,55 @@ jobs: name: xmldoc path: extra/doc + # Publishes new opam roots from linux-build for coding agents (extra/agent-setup.sh), + # and removes the ones they replace, so the release holds one archive per architecture. + publish-opam-root: + needs: linux-build + if: github.event_name == 'push' && github.ref == 'refs/heads/development' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: extra/agent-setup.sh + + - uses: actions/download-artifact@v8 + with: + pattern: opam-root-* + path: assets + + # Each artifact is named opam-root-; extra/agent-setup.sh names the asset it + # will look for. Upload it if it's missing, and drop the older ones for that arch. + - name: Upload to the release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + set -e + if ! gh release view opam-root >/dev/null 2>&1; then + gh release create opam-root --prerelease --target "$GITHUB_SHA" \ + --title "Prebuilt OCaml dependencies" \ + --notes "Prebuilt opam roots used by extra/agent-setup.sh. Updated automatically from CI's linux-build job." + fi + existing=$(gh release view opam-root --json assets -q '.assets[].name') + for dir in assets/opam-root-*; do + arch=${dir#assets/opam-root-} + asset=$(OPAM_ROOT_ARCH=$arch extra/agent-setup.sh --print-asset) + if grep -qxF "$asset" <<<"$existing"; then + echo "$asset is already published" + continue + fi + mv "$dir/opam-root.tar.zst" "$asset" + gh release upload opam-root "$asset" + for old in $existing; do + if [[ "$old" == *"-$arch-"*.tar.zst ]]; then + echo "Deleting $old" + gh release delete-asset opam-root "$old" -y + fi + done + done + linux-test: needs: linux-build runs-on: ubuntu-22.04${{ matrix.arch == 'arm64' && '-arm' || '' }} @@ -625,6 +694,9 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v8 + with: + # Only the binaries; this skips e.g. the opam-root-* artifacts of linux-build. + pattern: '*Binaries' - name: Install rclone run: | diff --git a/AGENTS.md b/AGENTS.md index 714a06e2b42..e8ca9598631 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,12 @@ Path-scoped instructions with more detail live in `.github/instructions/`. Read - `opam exec -- make ADD_REVISION=1 -f Makefile.win -s -j haxe` - Typical Windows compiler build command used in CI/tasks - `dune build` - Build using dune directly +### Setting Up a Fresh Container +On Ubuntu 24.04 (e.g. a cloud agent container), run `extra/agent-setup.sh` from the repository root before anything else. It installs the system packages, puts a prebuilt OCaml 5 toolchain with all opam dependencies in `/opt/opam` (building it from source only if no prebuilt one matches), adds the opam environment to `~/.bashrc`, and runs `make haxe`. Don't set up OCaml or opam by hand. +- If `ocamlfind` or `dune` aren't found in a new shell, run `export OPAMROOT=/opt/opam; eval "$(opam env)"` +- Pass `--no-build` to only set up the toolchain +- Building haxelib additionally needs the git submodules (`git submodule update --init --recursive`) before `make haxelib` + ### Use the Compiler You Just Built This repository may be checked out in several git worktrees, and the global PATH can point to a `haxe` binary from a different checkout. A bare `haxe` command may therefore run a different compiler than the one you just built. - Call the compiler and haxelib from the repository root by path, e.g. `./haxe` and `./haxelib` (`.\haxe.exe` in PowerShell), or put the repository root first on PATH for the session diff --git a/extra/agent-setup.sh b/extra/agent-setup.sh new file mode 100755 index 00000000000..5dc8f80cd1c --- /dev/null +++ b/extra/agent-setup.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Sets up an OCaml toolchain for building the Haxe compiler on Ubuntu, then builds it. +# Meant for coding agents in fresh containers (Claude Code, Copilot, ...), but works for +# anyone on Ubuntu 22.04 or newer. +# +# Building OCaml 5 and the opam dependencies from source takes 5-20 minutes, so the +# script first tries to download the opam root that CI's linux-build job builds on +# Ubuntu 22.04 and publishes from pushes to `development`, as an asset of the +# `opam-root` prerelease. It only builds from source if no published archive matches. +# +# Usage: extra/agent-setup.sh [options] +# --from-source never download the prebuilt opam root +# --no-apt don't install system packages (the caller has done that) +# --no-build only set up the toolchain, don't run `make haxe` +# --no-profile don't add the opam environment to ~/.bashrc and ~/.profile +# --print-asset print the file name of the matching prebuilt opam root and exit +# --print-url print the download URL of the matching prebuilt opam root and exit +# +# Environment: +# OPAM_ROOT_RELEASE_URL where to download the prebuilt opam root from +# OPAM_ROOT_ARCH the architecture to name the archive for (default: uname -m) +# OPAMROOT where the opam root lives (default /opt/opam; the prebuilt archive only +# works at that path, because opam roots are not relocatable) + +set -euo pipefail + +# Keep in sync with OCAML_VERSION in .github/workflows/main.yml. +OCAML_VERSION=5.3.0 +# The OS the prebuilt archive is built on (linux-build's runner). It also works on newer +# Ubuntu releases. +BUILD_OS=ubuntu-22.04 +# Bump to publish a fresh archive, e.g. to pick up newer versions of the dependencies. +REVISION=1 +DEFAULT_OPAMROOT=/opt/opam +RELEASE_URL=${OPAM_ROOT_RELEASE_URL:-https://github.com/HaxeFoundation/haxe/releases/download/opam-root} + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export OPAMROOT=${OPAMROOT:-$DEFAULT_OPAMROOT} +export OPAMYES=1 OPAMROOTISOK=1 OPAMCOLOR=never + +FROM_SOURCE=0 APT=1 BUILD=1 PROFILE=1 PRINT= +for arg in "$@"; do + case "$arg" in + --from-source) FROM_SOURCE=1 ;; + --no-apt) APT=0 ;; + --no-build) BUILD=0 ;; + --no-profile) PROFILE=0 ;; + --print-asset) PRINT=asset ;; + --print-url) PRINT=url ;; + *) echo "unknown option: $arg" >&2; exit 2 ;; + esac +done + +log() { echo "[agent-setup] $*" >&2; } + +# The key changes whenever something that affects the contents of the opam root changes. +cache_key() { + { + echo "$OCAML_VERSION $BUILD_OS $REVISION" + cat "$REPO_ROOT/haxe.opam" + } | sha256sum | cut -c1-16 +} +ASSET="opam-$OCAML_VERSION-$BUILD_OS-${OPAM_ROOT_ARCH:-$(uname -m)}-$(cache_key).tar.zst" + +case "$PRINT" in + asset) echo "$ASSET"; exit 0 ;; + url) echo "$RELEASE_URL/$ASSET"; exit 0 ;; +esac + +SUDO= +if [ "$(id -u)" != 0 ]; then SUDO=sudo; fi + +install_system_packages() { + log "installing system packages" + export DEBIAN_FRONTEND=noninteractive + $SUDO apt-get update -qq + $SUDO apt-get install -qqy --no-install-recommends \ + opam git curl ca-certificates zstd make gcc g++ m4 pkg-config unzip bzip2 rsync patch \ + libpcre2-dev zlib1g-dev libzstd-dev libmbedtls-dev neko neko-dev >/dev/null +} + +opam_root_ready() { + [ -f "$OPAMROOT/config" ] && opam exec -- ocamlfind ocamlopt -version >/dev/null 2>&1 && + opam exec -- dune --version >/dev/null 2>&1 +} + +download_prebuilt() { + if [ "$OPAMROOT" != "$DEFAULT_OPAMROOT" ]; then + log "OPAMROOT is not $DEFAULT_OPAMROOT, skipping the prebuilt download" + return 1 + fi + log "downloading $RELEASE_URL/$ASSET" + local tmp + tmp=$(mktemp -d) + if ! curl -fsSL --retry 3 --retry-delay 2 -o "$tmp/$ASSET" "$RELEASE_URL/$ASSET"; then + log "no prebuilt opam root for this configuration" + rm -rf "$tmp" + return 1 + fi + $SUDO mkdir -p "$(dirname "$OPAMROOT")" + $SUDO rm -rf "$OPAMROOT" + $SUDO tar -C "$(dirname "$OPAMROOT")" --zstd -xf "$tmp/$ASSET" + rm -rf "$tmp" + if [ -n "$SUDO" ]; then $SUDO chown -R "$(id -u):$(id -g)" "$OPAMROOT"; fi + if ! opam_root_ready; then + log "the prebuilt opam root doesn't work on this system" + $SUDO rm -rf "$OPAMROOT" + return 1 + fi +} + +# Same steps as linux-build in .github/workflows/main.yml. +build_from_source() { + log "building OCaml $OCAML_VERSION and the dependencies from source (this takes a while)" + $SUDO mkdir -p "$OPAMROOT" + if [ -n "$SUDO" ]; then $SUDO chown "$(id -u):$(id -g)" "$OPAMROOT"; fi + # Reuse an existing opam root (e.g. a half-finished earlier run) instead of wiping it. + if [ ! -f "$OPAMROOT/config" ]; then + # Containers usually can't run opam's bubblewrap sandbox. + opam init -c "$OCAML_VERSION" --no-setup --disable-sandboxing + fi + opam pin add haxe "$REPO_ROOT" --no-action + opam install haxe --deps-only --assume-depexts + opam clean --all-switches --download-cache --logs --repo-cache +} + +write_profile() { + local marker="# haxe agent-setup" + local block="$marker +export OPAMROOT=$OPAMROOT OPAMROOTISOK=1 +eval \"\$(opam env 2>/dev/null)\"" + for f in "$HOME/.bashrc" "$HOME/.profile"; do + if ! grep -qF "$marker" "$f" 2>/dev/null; then + printf '\n%s\n' "$block" >> "$f" + log "added the opam environment to $f" + fi + done +} + +if [ "$APT" = 1 ]; then install_system_packages; fi +if opam_root_ready; then + log "opam root at $OPAMROOT is already set up" +elif [ "$FROM_SOURCE" = 1 ] || ! download_prebuilt; then + build_from_source +fi +eval "$(opam env)" +log "OCaml $(ocamlfind ocamlopt -version), dune $(dune --version)" + +if [ "$PROFILE" = 1 ]; then write_profile; fi + +if [ "$BUILD" = 1 ]; then + log "building the compiler" + make -C "$REPO_ROOT" -s -j"$(nproc)" haxe + "$REPO_ROOT/haxe" -version +fi +log "done"