Set up haxelib and the test libraries in agent-setup.sh

.github/workflows/copilot-setup-steps.yml

@@ -48,13 +48,16 @@ jobs:
           echo "/usr/local/lib" | sudo tee /etc/ld.so.conf.d/local.conf
           sudo ldconfig
 
-      # Downloads the prebuilt opam root published by CI, or builds it if there is none yet.
-      - name: Set up OCaml
+      # Downloads the prebuilt opam root published by CI (or builds it if there is none yet),
+      # builds haxe and haxelib, and installs the haxelib libraries the tests need.
+      - name: Set up OCaml, Haxe and haxelib
         run: |
           set -ex
-          extra/agent-setup.sh --no-apt --no-build --no-profile
+          extra/agent-setup.sh --no-apt --no-profile
           opam list
           ocamlopt -v
+          haxe -version
+          haxelib list
 
       - name: Persist opam environment for subsequent steps
         run: |
@@ -65,29 +68,5 @@ jobs:
           echo "CAML_LD_LIBRARY_PATH=$CAML_LD_LIBRARY_PATH" >> $GITHUB_ENV
           echo "OCAML_TOPLEVEL_PATH=$OCAML_TOPLEVEL_PATH" >> $GITHUB_ENV
 
-      - name: Build Haxe
-        run: |
-          set -ex
-          eval $(opam env)
-          opam config exec -- make -s -j`nproc` STATICLINK=1 haxe
-          opam config exec -- make -s haxelib
-          make -s package_unix
-          ls -l out
-          ldd -v ./haxe
-          ldd -v ./haxelib
-
       - name: Add workspace to PATH
         run: echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
-
-      - name: Setup haxelib
-        run: |
-          set -ex
-          mkdir ~/haxelib
-          haxelib setup ~/haxelib
-          haxelib git utest https://github.com/haxe-utest/utest.git
-          haxelib git hxcoro https://github.com/HaxeFoundation/hxcoro.git
-          haxelib git hxcpp https://github.com/HaxeFoundation/hxcpp.git
-          cd ~/haxelib/hxcpp/git/tools/hxcpp && haxe compile.hxml && cd -
-          haxelib git hxjava https://github.com/HaxeFoundation/hxjava.git
-          haxelib git hxnodejs https://github.com/HaxeFoundation/hxnodejs.git
-          haxelib git haxeserver https://github.com/Simn/haxeserver.git

AGENTS.md

@@ -25,16 +25,17 @@ Path-scoped instructions with more detail live in `.github/instructions/`. Read
   - `dune build` - Build using dune directly
 
 ### Setting Up a Fresh Container
-In a cloud agent container, the environment's setup script usually runs `extra/agent-setup.sh` before the session starts. Then the OCaml toolchain is in `/opt/opam` and, unless it was run with `--no-build`, `./haxe` is already built. Check with `opam exec -- dune --version` and only run the script if that fails. Run `make haxe` again after changing compiler sources.
+In a cloud agent container, the environment's setup script usually runs `extra/agent-setup.sh` before the session starts. Then the OCaml toolchain is in `/opt/opam` and, unless it was run with `--no-build`, `./haxe` and `./haxelib` are already built and the haxelib libraries the tests need (utest, hxnodejs, hxcoro, hxcpp, hxjava, haxeserver) are installed. Check with `opam exec -- dune --version` and only run the script if that fails. Run `make haxe` again after changing compiler sources.
 
-Otherwise, on Ubuntu 22.04 or newer, run `extra/agent-setup.sh` from the repository root before anything else. It installs the system packages, puts a prebuilt OCaml 5 toolchain with all opam dependencies in `/opt/opam` (building it from source only if no prebuilt one matches), adds the opam environment to `~/.bashrc`, and runs `make haxe`. Don't set up OCaml or opam by hand.
+Otherwise, on Ubuntu 22.04 or newer, run `extra/agent-setup.sh` from the repository root before anything else. It installs the system packages, puts a prebuilt OCaml 5 toolchain with all opam dependencies in `/opt/opam` (building it from source only if no prebuilt one matches), adds the opam environment to `~/.bashrc`, runs `make haxe` and `make haxelib`, links both into `/usr/local/bin`, and installs the haxelib libraries listed above (bootstrapping hxcpp). Don't set up OCaml or opam by hand.
 - If `ocamlfind` or `dune` aren't found in a new shell, run `export OPAMROOT=/opt/opam; eval "$(opam env)"`
-- Pass `--no-build` to only set up the toolchain
-- Building haxelib additionally needs the git submodules (`git submodule update --init --recursive`) before `make haxelib`
+- Pass `--no-build` to only set up the toolchain, or `--no-haxelib` to skip haxelib and its libraries
+- Building haxelib by hand additionally needs the git submodules (`git submodule update --init --recursive`) before `make haxelib`
 
 ### Use the Compiler You Just Built
 This repository may be checked out in several git worktrees, and the global PATH can point to a `haxe` binary from a different checkout. A bare `haxe` command may therefore run a different compiler than the one you just built.
 - Call the compiler and haxelib from the repository root by path, e.g. `./haxe` and `./haxelib` (`.\haxe.exe` in PowerShell), or put the repository root first on PATH for the session
+- `-lib` needs `haxelib` on PATH, and haxelib in turn runs `haxe` from PATH; `extra/agent-setup.sh` links both into `/usr/local/bin`, unless an installed haxe or haxelib is already there (it then warns and leaves it alone)
 - When in doubt, check with `haxe -version` or `Get-Command haxe` / `which haxe`
 - The `haxe` commands in this file assume this has been taken care of
 
@@ -250,7 +251,7 @@ The steps below use the `gh` CLI. The checkout may have many remotes (contributo
 - Run `haxelib setup` to set up the repository path
 - Use `haxelib git utest https://github.com/haxe-utest/utest` for utest
 - A git-cloned library can be set as a development library via `haxelib dev libname path`
-- **hxcpp bootstrapping**: The git version of hxcpp requires bootstrapping before C++ compilation works. Run `cd $(haxelib libpath hxcpp)tools/hxcpp && haxe compile.hxml` after installing. Without this, `haxelib run hxcpp` will prompt for interactive input and hang. (The Copilot setup workflow, `.github/workflows/copilot-setup-steps.yml`, does this automatically.)
+- **hxcpp bootstrapping**: The git version of hxcpp requires bootstrapping before C++ compilation works. Run `cd $(haxelib libpath hxcpp)tools/hxcpp && haxe compile.hxml` after installing. Without this, `haxelib run hxcpp` will prompt for interactive input and hang. (`extra/agent-setup.sh` does this automatically.)
 - C++ compilation is slow (minutes for a full build with all static libraries) but should produce constant output. If it hangs with no output, hxcpp likely needs bootstrapping.
 
 ## Other considerations

extra/agent-setup.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Sets up an OCaml toolchain for building the Haxe compiler on Ubuntu, then builds it.
+# Sets up an OCaml toolchain for building the Haxe compiler on Ubuntu, builds it, and
+# sets up haxelib with the libraries the test suites need.
 # Meant for coding agents in fresh containers (Claude Code, Copilot, ...), but works for
 # anyone on Ubuntu 22.04 or newer.
 #
@@ -11,7 +12,8 @@
 # Usage: extra/agent-setup.sh [options]
 #   --from-source  never download the prebuilt opam root
 #   --no-apt       don't install system packages (the caller has done that)
-#   --no-build     only set up the toolchain, don't run `make haxe`
+#   --no-build     only set up the toolchain, don't run `make haxe` (implies --no-haxelib)
+#   --no-haxelib   don't build haxelib or install the haxelib libraries
 #   --no-profile   don't add the opam environment to ~/.bashrc and ~/.profile
 #   --print-asset  print the file name of the matching prebuilt opam root and exit
 #   --print-url    print the download URL of the matching prebuilt opam root and exit
@@ -21,6 +23,11 @@
 #   OPAM_ROOT_ARCH  the architecture to name the archive for (default: uname -m)
 #   OPAMROOT  where the opam root lives (default /opt/opam; the prebuilt archive only
 #             works at that path, because opam roots are not relocatable)
+#   HAXELIB_REPO  the haxelib repository to set up if there is none yet (default ~/haxelib)
+#   HAXELIB_BIN_DIR  where to link the haxe and haxelib executables; both have to be on
+#             PATH, because the compiler runs `haxelib` from PATH for `-lib`, which in turn
+#             runs `haxe` from PATH (default /usr/local/bin). Existing files there are only
+#             replaced if they are links into a Haxe checkout, e.g. made by an earlier run.
 
 set -euo pipefail
 
@@ -38,12 +45,23 @@ REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 export OPAMROOT=${OPAMROOT:-$DEFAULT_OPAMROOT}
 export OPAMYES=1 OPAMROOTISOK=1 OPAMCOLOR=never
 
-FROM_SOURCE=0 APT=1 BUILD=1 PROFILE=1 PRINT=
+# The libraries the test suites use, installed from git like CI does.
+HAXELIB_LIBS=(
+	"utest https://github.com/haxe-utest/utest.git"
+	"hxnodejs https://github.com/HaxeFoundation/hxnodejs.git"
+	"hxcoro https://github.com/HaxeFoundation/hxcoro.git"
+	"hxcpp https://github.com/HaxeFoundation/hxcpp.git"
+	"hxjava https://github.com/HaxeFoundation/hxjava.git"
+	"haxeserver https://github.com/Simn/haxeserver.git"
+)
+
+FROM_SOURCE=0 APT=1 BUILD=1 HAXELIB=1 PROFILE=1 PRINT=
 for arg in "$@"; do
 	case "$arg" in
 		--from-source) FROM_SOURCE=1 ;;
 		--no-apt) APT=0 ;;
-		--no-build) BUILD=0 ;;
+		--no-build) BUILD=0 HAXELIB=0 ;;
+		--no-haxelib) HAXELIB=0 ;;
 		--no-profile) PROFILE=0 ;;
 		--print-asset) PRINT=asset ;;
 		--print-url) PRINT=url ;;
@@ -124,17 +142,80 @@ build_from_source() {
 	opam clean --all-switches --download-cache --logs --repo-cache
 }
 
+# The block goes at the top of the file: Ubuntu's default ~/.bashrc returns early in
+# non-interactive shells, and agents usually run their commands in those.
 write_profile() {
 	local marker="# haxe agent-setup"
 	local block="$marker
 export OPAMROOT=$OPAMROOT OPAMROOTISOK=1
 eval \"\$(opam env 2>/dev/null)\""
+	local f tmp
 	for f in "$HOME/.bashrc" "$HOME/.profile"; do
-		if ! grep -qF "$marker" "$f" 2>/dev/null; then
-			printf '\n%s\n' "$block" >> "$f"
-			log "added the opam environment to $f"
+		if [ "$(head -n 1 "$f" 2>/dev/null)" = "$marker" ]; then continue; fi
+		tmp=$(mktemp)
+		printf '%s\n\n' "$block" > "$tmp"
+		# Drop a block that an earlier version of this script appended at the end.
+		if [ -f "$f" ]; then awk -v m="$marker" '$0 == m { skip = 3 } skip > 0 { skip--; next } { print }' "$f" >> "$tmp"; fi
+		cat "$tmp" > "$f"
+		rm -f "$tmp"
+		log "added the opam environment to the top of $f"
+	done
+}
+
+# Links an executable of this checkout into the bin directory. It never replaces an
+# installed haxe or haxelib there, only links into a Haxe checkout (another worktree, or
+# an earlier run of this script).
+link_into_bin() {
+	local src=$1 dst=$2 old
+	if [ -L "$dst" ]; then
+		old=$(readlink -f "$dst" || true)
+		if [ "$old" = "$(readlink -f "$src")" ]; then return; fi
+		if [ ! -f "$(dirname "$old")/extra/agent-setup.sh" ]; then
+			log "warning: $dst links to $old, not to a Haxe checkout; leaving it alone"
+			return
+		fi
+		log "$dst pointed to $old, now points to $src"
+	elif [ -e "$dst" ]; then
+		log "warning: $dst already exists and is not a link; leaving it alone"
+		return
+	fi
+	$SUDO ln -sfn "$src" "$dst"
+}
+
+setup_haxelib() {
+	log "building haxelib"
+	git -C "$REPO_ROOT" submodule update --init --recursive
+	make -C "$REPO_ROOT" -s haxelib
+	local bin_dir=${HAXELIB_BIN_DIR:-/usr/local/bin}
+	$SUDO mkdir -p "$bin_dir"
+	link_into_bin "$REPO_ROOT/haxe" "$bin_dir/haxe"
+	link_into_bin "$REPO_ROOT/haxelib" "$bin_dir/haxelib"
+	# For this script: hxcpp's bootstrap needs haxe and haxelib.
+	export PATH="$REPO_ROOT:$PATH"
+	if ! haxelib config >/dev/null 2>&1; then
+		local repo=${HAXELIB_REPO:-$HOME/haxelib}
+		mkdir -p "$repo"
+		haxelib setup "$repo" >/dev/null
+	fi
+	log "haxelib repository: $(haxelib config)"
+	local lib name url
+	for lib in "${HAXELIB_LIBS[@]}"; do
+		read -r name url <<< "$lib"
+		if haxelib path "$name" >/dev/null 2>&1; then
+			log "$name is already installed"
+		else
+			log "installing $name"
+			haxelib git "$name" "$url" --quiet --never
 		fi
 	done
+	# The git version of hxcpp has to be bootstrapped, or `haxelib run hxcpp` asks
+	# for input and hangs.
+	local hxcpp_tools
+	hxcpp_tools="$(haxelib libpath hxcpp)tools/hxcpp"
+	if [ ! -f "$hxcpp_tools/../../hxcpp.n" ]; then
+		log "bootstrapping hxcpp"
+		(cd "$hxcpp_tools" && haxe compile.hxml)
+	fi
 }
 
 if [ "$APT" = 1 ]; then install_system_packages; fi
@@ -153,4 +234,6 @@ if [ "$BUILD" = 1 ]; then
 	make -C "$REPO_ROOT" -s -j"$(nproc)" haxe
 	"$REPO_ROOT/haxe" -version
 fi
+
+if [ "$HAXELIB" = 1 ]; then setup_haxelib; fi
 log "done"