Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.

## [Unreleased]

### Added

- `crs` on a converter: the CRS all sources are reprojected to. Converters with sources in
different CRS need it and fail with a clear error without it.

## [v0.4.3] - 2026-11-07

### Added
Expand Down
73 changes: 73 additions & 0 deletions tests/test_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,79 @@ def test_a_side_file_in_an_archive_can_be_given_as_glob(tmp_folder, side_file):
assert result["code"].tolist() == ["NA", "GH"]


def _sources_in_crs(folder, *crs_list):
import shapely

files = {}
for i, crs in enumerate(crs_list):
gdf = gpd.GeoDataFrame(
{"id": [f"{i}"], "name": ["a"]},
geometry=[shapely.box(10 + i, 50, 10.01 + i, 50.01)],
crs="EPSG:4326",
)
if crs is None:
gdf = gdf.set_crs(None, allow_override=True)
elif crs != "EPSG:4326":
gdf = gdf.to_crs(crs)
path = folder / f"source_{i}.gpkg"
gdf.to_file(path)
files[str(path)] = path.name
return files


@pytest.mark.parametrize(
"crs_list, target, expected",
[
(("EPSG:4326", "EPSG:4326"), None, "EPSG:4326"),
(("EPSG:32632", "EPSG:32632"), None, "EPSG:32632"),
# as before, the single CRS of the other sources applies
(("EPSG:4326", None), None, "EPSG:4326"),
(("EPSG:3857", "EPSG:32632", "EPSG:4326"), "EPSG:4326", "EPSG:4326"),
(("EPSG:32632", "EPSG:32632"), "EPSG:3035", "EPSG:3035"),
],
)
def test_sources_are_reprojected_to_the_converters_crs(tmp_folder, crs_list, target, expected):
Converter = type("Converter", (BaseConverter,), {**CONFIG, "crs": target})
dest = tmp_folder / "converted.parquet"
Converter().convert(dest, input_files=_sources_in_crs(tmp_folder, *crs_list))

result = gpd.read_parquet(dest).sort_values("id")
assert result.crs == expected
centroids = result.to_crs("EPSG:4326").geometry.centroid
assert [round(p.x, 3) for p in centroids] == [10.005 + i for i in range(len(crs_list))]


def test_sources_in_different_crs_require_the_converters_crs(tmp_folder):
Converter = type("Converter", (BaseConverter,), CONFIG)
sources = _sources_in_crs(tmp_folder, "EPSG:3857", "EPSG:32632")
with pytest.raises(
ValueError, match=r"source_1\.gpkg is in another CRS than .*source_0\.gpkg, set `crs`"
):
Converter().convert(tmp_folder / "converted.parquet", input_files=sources)


def test_sources_without_crs_can_not_be_reprojected(tmp_folder):
Converter = type("Converter", (BaseConverter,), {**CONFIG, "crs": "EPSG:4326"})
sources = _sources_in_crs(tmp_folder, "EPSG:3857", None)
with pytest.raises(ValueError, match=r"no CRS for .*source_1\.gpkg"):
Converter().convert(tmp_folder / "converted.parquet", input_files=sources)


def test_equal_crs_with_a_different_wkt_are_not_reprojected():
import shapely
from pyproj import CRS

epsg = CRS.from_epsg(25832)
# e.g. a shapefile's .prj: equal to the EPSG code, but with another WKT and hash
esri = CRS.from_wkt(epsg.to_wkt("WKT1_ESRI"))
assert epsg == esri and hash(epsg) != hash(esri)
gdfs = [
(gpd.GeoDataFrame(geometry=[shapely.Point(500000, 5500000)], crs=crs), str(i))
for i, crs in enumerate([epsg, esri])
]
assert [gdf.crs for gdf in BaseConverter()._reproject_sources(gdfs)] == [epsg, esri]


@pytest.mark.parametrize("bom", [False, True], ids=["without-bom", "with-bom"])
def test_read_geojson_decodes_utf8(tmp_folder, cp1252_locale, bom):
file_path = tmp_folder / "umlaut.json"
Expand Down
10 changes: 10 additions & 0 deletions tests/test_convert_duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,16 @@ def test_duckdb_converter_rejects_side_files(tmp_folder):
)


def test_duckdb_converter_rejects_crs(tmp_folder):
src = _source_file(tmp_folder)
CrsConverter = type("CrsConverter", (Converter,), {"crs": "EPSG:3035"})

with pytest.raises(ValueError, match="don't support crs"):
CrsConverter().convert(
tmp_folder / "converted.parquet", input_files={str(src): "source.parquet"}
)


def test_duckdb_converter_ignores_hive_style_folders(tmp_folder):
source_folder = tmp_folder / "region=north"
source_folder.mkdir()
Expand Down
33 changes: 30 additions & 3 deletions vecorel_cli/conversion/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,14 @@
from ..vecorel.hilbert import hilbert_sort_geodataframe
from ..vecorel.schemas import Schemas
from ..vecorel.typing import Sources
from ..vecorel.util import get_fs, merge_headers, name_from_uri, stream_file, suffix_duplicate_ids
from ..vecorel.util import (
find_differing_crs,
get_fs,
merge_headers,
name_from_uri,
stream_file,
suffix_duplicate_ids,
)
from .deflate64 import deflate64_zip
from .flatdict import FlatDict

Expand Down Expand Up @@ -87,6 +94,8 @@ class BaseConverter(LoggerMixin):
# read_data() leaves them out, read_side_file() reads them, e.g. in migrate()
side_files: Sequence[str] = ()
side_paths: dict[str, str] = {}
# The CRS all sources are reprojected to; required if they are in different CRS
crs: Optional[str] = None
variant: Optional[str] = None

columns: dict[str, str | Sequence[str]] = {}
Expand Down Expand Up @@ -688,9 +697,27 @@ def read_data(self, paths, **kwargs):
data = self.file_migration(data, path, uri, layer)
if not isinstance(data, GeoDataFrame):
raise ValueError("Per-file/layer migration function must return a GeoDataFrame")
gdfs.append(data)
gdfs.append((data, path))

return pd.concat(self._reproject_sources(gdfs))

return pd.concat(gdfs)
def _reproject_sources(self, gdfs) -> list[GeoDataFrame]:
if self.crs is None:
# sources without a CRS take the CRS of the others when concatenated
with_crs = [(gdf, path) for gdf, path in gdfs if gdf.crs is not None]
differing = find_differing_crs([gdf.crs for gdf, _ in with_crs])
if differing is not None:
raise ValueError(
f"{with_crs[differing][1]} is in another CRS than {with_crs[0][1]}, "
"set `crs` on the converter to reproject the sources to one"
)
return [gdf for gdf, _ in gdfs]
missing = [path for gdf, path in gdfs if gdf.crs is None]
if missing:
raise ValueError(
f"Can't reproject to {self.crs}, no CRS for {', '.join(dict.fromkeys(missing))}"
)
return [gdf if gdf.crs == self.crs else gdf.to_crs(self.crs) for gdf, _ in gdfs]

def filter_rows(self, gdf):
if len(self.column_filters) > 0:
Expand Down
3 changes: 3 additions & 0 deletions vecorel_cli/conversion/duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ def convert(
if self.side_files:
# the sources are read in SQL, without the read_data() and migrate() side files need
raise ValueError(f"{type(self).__name__}: DuckDB converters don't support side_files")
if self.crs:
# the sources are combined in SQL without reprojection
raise ValueError(f"{type(self).__name__}: DuckDB converters don't support crs")
self._prewarm_schemas()

directory = os.path.dirname(output_file)
Expand Down
Loading