chore: lower requires-python to >=3.11 — the platform floor outlier (#443)

.github/workflows/ci.yml

@@ -15,7 +15,11 @@ jobs:
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
-          python-version: "3.12"
+          # The DECLARED FLOOR, not a convenient version. CI must exercise the
+          # weakest supported config, because that is the one nobody runs
+          # locally. Keep equal to requires-python — test_ci_gates.py asserts
+          # it. The other end (3.13) is covered by test-py313 below.
+          python-version: "3.11"
 
       - name: Install uv
         uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4 — pinned, NOT upgraded
@@ -33,7 +37,11 @@ jobs:
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
-          python-version: "3.12"
+          # The DECLARED FLOOR, not a convenient version. CI must exercise the
+          # weakest supported config, because that is the one nobody runs
+          # locally. Keep equal to requires-python — test_ci_gates.py asserts
+          # it. The other end (3.13) is covered by test-py313 below.
+          python-version: "3.11"
 
       - name: Install uv
         uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4 — pinned, NOT upgraded
@@ -51,7 +59,11 @@ jobs:
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
-          python-version: "3.12"
+          # The DECLARED FLOOR, not a convenient version. CI must exercise the
+          # weakest supported config, because that is the one nobody runs
+          # locally. Keep equal to requires-python — test_ci_gates.py asserts
+          # it. The other end (3.13) is covered by test-py313 below.
+          python-version: "3.11"
 
       - name: Install uv
         uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4 — pinned, NOT upgraded
@@ -76,9 +88,64 @@ jobs:
       - name: Install dependencies
         run: uv sync
 
+      - name: The interpreter is the declared floor
+        # An assertion, not an echo. That setup-python's 3.11 is the
+        # interpreter `uv run` then selects is an inference about how two
+        # actions interact via PATH — true today, unobserved until now.
+        # An echo would print the answer into a log nobody reads; this
+        # fails the job. (C-336: claims inferred rather than observed.)
+        run: uv run python -c "import sys; assert sys.version_info[:2] == (3, 11), sys.version"
+
       - name: Test
         run: uv run pytest
 
+  test-py313:
+    # The OTHER end of the supported range, and the reason it is a separate
+    # job rather than a matrix leg: branch protection requires the status
+    # check named exactly `test`, with enforce_admins. A strategy.matrix
+    # renames it to `test (3.11)`, the required `test` never reports, and
+    # every PR waits forever. New job names are safe precisely because they
+    # are NOT required.
+    #
+    # What it buys: since the floor dropped to 3.11 (#443) the lock forks —
+    # 3.11 resolves tifffile 2026.3.3 / imagecodecs 2026.3.6, >=3.12 keeps
+    # 2026.5.15 / 2026.5.10. The required `test` job therefore exercises the
+    # OLD raster line while the dev venv and the server run the new one.
+    # This job is the only thing that runs the production line. It is not
+    # required yet — a required check that has never reported blocks every
+    # merge — so the follow-up in C-347 is to add it to both branches'
+    # required lists once it has reported. Until then it is visible, not
+    # enforced, and that gap is registered rather than glossed.
+    #
+    # Full suite, not raster-only: the fork is a property of the resolver,
+    # so the next upstream to drop 3.11 adds a package a raster-scoped job
+    # would silently miss.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4 — pinned, NOT upgraded
+
+      - name: Install dependencies
+        # --frozen: this job must not be able to repair a stale lock, or it
+        # would mask exactly what `uv lock --check` in `test` exists to catch.
+        run: uv sync --frozen
+
+      - name: The interpreter is not the floor
+        # --frozen on EVERY uv run, not just the sync. `uv run` locks and syncs
+        # by default, so a bare `uv run` here would re-resolve and rewrite a
+        # stale lock — undoing the --frozen above and masking exactly what
+        # `uv lock --check` in `test` exists to catch.
+        run: uv run --frozen python -c "import sys; assert sys.version_info[:2] == (3, 13), sys.version"
+
+      - name: Test
+        run: uv run --frozen pytest
+
   docs:
     # Documentation consistency. The script has existed since the base-docs
     # set was created and ran NOWHERE — not CI, not pytest — so it only ever
@@ -107,7 +174,11 @@ jobs:
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
-          python-version: "3.12"
+          # The DECLARED FLOOR, not a convenient version. CI must exercise the
+          # weakest supported config, because that is the one nobody runs
+          # locally. Keep equal to requires-python — test_ci_gates.py asserts
+          # it. The other end (3.13) is covered by test-py313 below.
+          python-version: "3.11"
 
       - name: Install uv
         uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4 — pinned, NOT upgraded

.github/workflows/publish_package.yml

@@ -29,10 +29,10 @@ jobs:
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
 
-      - name: Install uv (+ Python 3.12)
+      - name: Install uv (+ Python 3.11)
         uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86  # v5
         with:
-          python-version: "3.12"   # requires-python is >=3.12; the wheel is pure-Python (py3-none-any)
+          python-version: "3.11"   # requires-python is >=3.11; the wheel is pure-Python (py3-none-any)
 
       - name: Guard — pyproject version must equal the tag being released
         # #363 / C-346. The guard below checks the version BEATS PyPI. It does
@@ -52,12 +52,12 @@ jobs:
           TAG: ${{ github.ref_name }}
         run: |
           set -euo pipefail
-          # `uv run --no-project --python 3.12`, matching the guard below.
+          # `uv run --no-project --python 3.11`, matching the guard below.
           # Bare `python3` is whatever the runner image ships, and tomllib
           # needs >= 3.11 — drilled locally on python3.10, where the bare
           # form died with ModuleNotFoundError. Pinning the interpreter is
           # the difference between a guard and a guard that works.
-          VERSION=$(uv run --no-project --python 3.12 python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
+          VERSION=$(uv run --no-project --python 3.11 python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
           if [ "$TAG" != "v${VERSION}" ]; then
             echo "::error::Tag/version mismatch — refusing to publish."
             echo "  tag that triggered this run : ${TAG}"
@@ -71,7 +71,7 @@ jobs:
 
       - name: Guard — pyproject version must be newer than what's on PyPI
         run: |
-          uv run --no-project --python 3.12 --with packaging python - <<'PY'
+          uv run --no-project --python 3.11 --with packaging python - <<'PY'
           import json, tomllib, urllib.request
           from packaging.version import parse
           new = tomllib.load(open("pyproject.toml", "rb"))["project"]["version"]

.github/workflows/release-topology.yml

@@ -96,6 +96,17 @@ jobs:
             echo "main is NOT an ancestor of development."
           fi
 
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          # This workflow had NO interpreter pin — it ran `uv sync` and took
+          # whatever uv chose. That was tolerable while requires-python was
+          # >=3.12 and narrow; since #443 widened it to >=3.11 the unpinned
+          # choice also selects which raster fork installs. These gates touch
+          # no rasters, so the impact is nil today — pinned anyway, because
+          # "nil today" is how the deployment guide's Python line survived
+          # three months of being wrong.
+          python-version: "3.11"
+
       - name: Install uv
         uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4 — pinned, NOT upgraded
 

docs/ADRs/030_raster_tooling.md

@@ -4,6 +4,7 @@
 **Date:** 2026-05-18
 **Deciders:** Simon Polichinel von der Maase, Claude Code
 **Applies:** ADR-029 (GHS-POP as First Population Source), ADR-012 (Four-Layer Data Architecture)
+**Amended:** 2026-08-13 — the Python floor is now `>=3.11`. The tooling decision is unchanged; see the Amendment at the end of Consequences. *(A header pointer is an extension of ADR-051's inline-amendment convention, added because this ADR is long and its floor claim appears in five places — a reader who stops after "In scope" would otherwise leave with a fact that is no longer true.)*
 
 ---
 
@@ -159,13 +160,90 @@ Write the Rust raster processing tool immediately for GHS-POP, skip the tifffile
 - **tifffile doesn't parse GeoTIFF CRS metadata as first-class objects.** We validate CRS by reading raw TIFF GeoKey tags rather than through a geospatial API. This is adequate for WGS84 validation but less ergonomic than rasterio.
 - **Python ≥3.12 version bump required.** Current tifffile releases require Python 3.12+. This is assessed as safe for this repository (no 3.10-specific patterns in codebase, all dependencies have 3.12-compatible versions) but requires bumping dependency lower bounds (numpy ≥1.26, pandas ≥2.0, matplotlib ≥3.8).
 
+> ### Amendment, 2026-08-13 — the floor drops to 3.11, and what that costs
+>
+> **The tooling decision is untouched.** tifffile now, Rust long-term, no GDAL, `read_geotiff` as
+> the single reader — all of it stands. Only the version consequence above is amended. Recorded
+> rather than rewritten, per ADR-051.
+>
+> **What changed.** `requires-python` is now `">=3.11"` (#443).
+>
+> **Why the original reason did not survive contact.** The bullet above is the whole argument, and
+> it is a fact about a vendor at a moment — *"current tifffile releases require Python 3.12+"* —
+> not an architectural requirement. It was true when written. What we did not notice is that it
+> made a third party's release schedule into our public API. **3.11 was never considered:** the
+> string does not appear anywhere in this ADR, and there is no Considered Alternative about
+> interpreter versions. The bullet's own supporting evidence assesses the wrong thing — *"no
+> 3.10-specific patterns in codebase"* says nothing about 3.11 or 3.12.
+>
+> **The consumer that forced the question.** The views-models conda environments run 3.11.14 and
+> 3.11.15, and 28 requirements files there reference this package, so `pip install
+> views-datafactory` failed in every one of them. This is named deliberately: without a concrete
+> consumer, the right action would have been to leave the floor alone and merely record that it
+> had been inherited rather than chosen.
+>
+> **Verified before deciding, not after.** Full suite green on 3.11.13 with the same six
+> pre-existing xfails; `ruff` at `py311`, `mypy src/`, and `compileall` all clean; and a search for
+> the constructs that would actually require 3.12 — PEP 695 type parameters, `itertools.batched`,
+> `typing.override`, `Path.walk`, `sys.monitoring` — found none.
+>
+> **The cost, which is real and permanent.** `uv.lock` is now multi-version. Under 3.11 the raster
+> stack resolves `tifffile 2026.3.3` and `imagecodecs 2026.3.6`; under ≥3.12 it keeps `2026.5.15`
+> and `2026.5.10`. Both upstreams dropped 3.11 deliberately and neither will restore it: tifffile
+> adopted PEP 695 syntax at 2026.4.11, so newer releases would raise `SyntaxError` on 3.11 rather
+> than merely warn, and imagecodecs ships `cp312-abi3` wheels only from 2026.5.10. A 3.11 consumer
+> is therefore pinned to the March-2026 raster line for good.
+>
+> **Which decoder decodes production pixels — and the gap this exposed.** §Consequences above says
+> `imagecodecs` covers "exotic compression" and calls it "unlikely for a population grid". That is
+> wrong, and was corrected in code one day after this ADR was accepted without the ADR being
+> updated. Every GHS-POP and GHS-BUILT-S GeoTIFF JRC publishes is LZW-compressed, and tifffile has
+> no pure-Python LZW path: `read_geotiff`'s `page.asarray()` dispatches to `imagecodecs.lzw_decode`
+> unconditionally. Blocking the import and attempting an LZW write raises
+> `KeyError: "<COMPRESSION.LZW: 5> requires the 'imagecodecs' package"` — measured, not inferred.
+>
+> `imagecodecs` is imported by **nothing** under `src/`. So it is load-bearing and invisible: an
+> import-graph audit concludes it is removable, and removing it would break every production raster
+> read while leaving the suite green. **Until #443, no test in this repository had ever written a
+> compressed TIFF on any interpreter** — every `imwrite` was uncompressed, so neither decoder line
+> had been exercised. That hole predates this change by three months; the change only made it
+> visible, because for the first time there are two candidate decoders and a reason to ask which
+> one was tested.
+>
+> **What is still not verified.** Nothing compares the two decoder lines' *output*. We now assert
+> that each can decode LZW; we do not assert they produce identical pixels. Do not read the new
+> test as parity evidence.
+>
+> **CI consequence.** The required `test` check runs the floor, so it exercises the *old* raster
+> line, while the dev venv and the server run the new one. A non-required `test-py313` job covers
+> the production line. A check nobody must satisfy is ignorable, and that residual is registered as
+> **C-347** with the follow-up that closes it: make `test-py313` required on both branches once it
+> has reported. A `strategy.matrix` is not available here — it renames the required `test` context
+> and deadlocks every pull request under `enforce_admins`.
+>
+> **Open Question 4 is answered and replaced.** "Python 3.12 availability on the production server"
+> was the wrong question. The right one is *which interpreter does the server run, and therefore
+> which raster line does it install* — and nothing in this repository asserts it. Registered as
+> **C-348**.
+>
+> **One instance of a class, worth naming.** `hetzner_deployment_guide.md` said "Install Python
+> 3.10+" for the entire three months this ADR mandated `>=3.12` — an instruction that produced an
+> environment where the package could not be installed at all. Neither document was wrong on its
+> own terms; they were never compared. Fixed in #443, and `tests/test_ci_gates.py` now asserts that
+> CI's interpreter pins equal the declared floor, so at least that pair can no longer drift
+> silently.
+
 ---
 
 ## Implementation Notes
 
 ### Immediate (GHS-POP integration)
 
 1. Bump `requires-python` to `">=3.12"` in `pyproject.toml`. Adjust dependency lower bounds as needed.
+   > **Superseded by the Amendment above — do not follow this step.** The floor is `>=3.11`.
+   > Re-raising it here would re-break the views-models 3.11 environments. Left in place rather
+   > than edited, per this repo's amend-don't-rewrite convention, but flagged inline because an
+   > implementer reading only §Implementation Notes would otherwise reverse #443.
 2. Add `tifffile` to dependencies.
 3. Verify `uv run pytest` passes after dependency changes.
 4. GeoTIFF reading pattern: `tifffile.imread(path)` returns numpy array. Validate dimensions (21600×43200 for 30ss global), check GeoKey tags for EPSG:4326, replace nodata (-9999) with 0.
@@ -211,6 +289,9 @@ Write the Rust raster processing tool immediately for GHS-POP, skip the tifffile
 2. **Infrastructure data landscape.** The format survey covered likely sources but infrastructure data has not been broadly surveyed. An unusual infrastructure dataset could change the tooling calculus.
 3. **tifffile compression support.** GHS-POP GeoTIFFs are compressed (likely LZW or DEFLATE). tifffile handles these, but the specific compression used by JRC should be verified on first download.
 4. **Python 3.12 on production server.** The Hetzner server currently runs the pipeline. Python 3.12 availability needs to be confirmed before deployment.
+   > **Answered and replaced by the Amendment above.** The floor is `>=3.11`, so availability is no
+   > longer the question — *which* interpreter the server has is, because it now selects the raster
+   > decoder. Nothing in this repository asserts it; tracked as **C-348**.
 
 ---
 

docs/guides/hetzner_deployment_guide.md

@@ -118,14 +118,36 @@ sudo apt update && sudo apt upgrade -y
 ### 1.3 Install Python and uv
 
 ```bash
-# Install Python 3.10+
-sudo apt install -y python3 python3-pip git
+# Install git; get Python from uv, NOT from apt (see the note below)
+sudo apt install -y git
 
 # Install uv (fast Python package manager)
 curl -LsSf https://astral.sh/uv/install.sh | sh
 source ~/.cargo/env  # or restart shell
+
+# Pin the interpreter EXPLICITLY. Do not rely on `apt install python3`.
+uv python install 3.13
 ```
 
+> **Why the interpreter is named here, and why `apt install python3` is not enough.**
+>
+> This block used to read `# Install Python 3.10+`. For three months the project
+> declared `requires-python = ">=3.12"`, so anyone following that instruction on a
+> distro shipping 3.10 or 3.11 built an environment in which this package could not
+> be installed at all. Nothing detected it; the guide and `pyproject.toml` were each
+> internally consistent and never compared.
+>
+> Since #443 the floor is `>=3.11`, so bare `apt install python3` no longer *fails* —
+> which is worse, because the version it happens to give you now **selects which raster
+> decoder the server runs**. On 3.11, `uv sync` resolves `tifffile 2026.3.3` and
+> `imagecodecs 2026.3.6`; on 3.12+ it resolves the current line. Those are the codecs
+> that decode every GHS-POP and GHS-BUILT-S pixel. A distro default silently choosing
+> between them is not a deployment decision anyone made.
+>
+> Name the version, and keep it equal to what CI's non-floor job exercises. Nothing in
+> this repository asserts which interpreter the server actually has — that gap is
+> registered as **C-348**.
+
 ### 1.4 Clone the repository
 
 ```bash
@@ -1146,7 +1168,7 @@ su - views-deploy -c "sudo ls /"
 # Expected: permission denied (not in sudo group)
 
 # 7. Automated verification (run on server)
-python3 scripts/verify_server_hardening.py
+uv run python scripts/verify_server_hardening.py
 # Expected: all checks pass
 ```
 
@@ -1277,7 +1299,7 @@ rm /root/.ssh/id_ed25519 /root/.ssh/id_ed25519.pub
 #### Verification
 
 ```bash
-python3 /home/views-deploy/views-datafactory/scripts/verify_server_hardening.py
+uv run --directory /home/views-deploy/views-datafactory python scripts/verify_server_hardening.py
 # Expected: 21/21 checks pass
 # Check 20: "Personal SSH key removed from /root — removed"
 # Check 21: "Deploy key exists for service user"
@@ -1458,7 +1480,7 @@ ssh <your-user>@204.168.219.108 "sudo passwd -S <username>"
 Or run the automated check:
 
 ```bash
-python3 /home/views-deploy/views-datafactory/scripts/verify_server_hardening.py
+uv run --directory /home/views-deploy/views-datafactory python scripts/verify_server_hardening.py
 # Looks for named accounts with sudo, verifies each has a password
 # and an authorized_keys file.
 ```