Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `crs` on a converter: the CRS all sources are reprojected to. Converters with sources in
different CRS need it and fail with a clear error without it.

### Fixed

- Writing GeoParquet no longer fails for date-times with more than millisecond precision: they
are stored in milliseconds, as the specification defines.

## [v0.4.3] - 2026-11-07

### Added
Expand Down
38 changes: 38 additions & 0 deletions tests/test_convert_duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import geopandas as gpd
import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import pytest
import shapely
Expand Down Expand Up @@ -133,6 +135,42 @@ def test_duckdb_converter_rejects_side_files(tmp_folder):
)


def test_duckdb_converter_drops_sub_millisecond_precision(tmp_folder):
src = tmp_folder / "source.parquet"
gpd.GeoDataFrame(
{
"id": ["a", "b"],
"name": ["x", "y"],
"t": pd.to_datetime(
["2021-01-01 10:00:00.123456", "1969-12-31 23:59:59.999500"], utc=True
),
},
geometry=[shapely.box(0, 0, 1, 1), shapely.box(2, 0, 3, 1)],
crs="EPSG:4326",
).to_parquet(src)
TimestampConverter = type(
"TimestampConverter",
(Converter,),
{
"columns": {**CONFIG["columns"], "t": "t"},
"missing_schemas": {
"properties": {"name": {"type": "string"}, "t": {"type": "date-time"}}
},
},
)
dest = tmp_folder / "converted.parquet"

TimestampConverter().convert(dest, input_files={str(src): "source.parquet"})

table = pq.read_table(dest, columns=["id", "t"]).sort_by("id")
assert table.schema.field("t").type == pa.timestamp("ms", tz="UTC")
# floored like pandas, also before 1970
assert [t.isoformat() for t in table.column("t").to_pylist()] == [
"2021-01-01T10:00:00.123000+00:00",
"1969-12-31T23:59:59.999000+00:00",
]


def test_duckdb_converter_rejects_crs(tmp_folder):
src = _source_file(tmp_folder)
CrsConverter = type("CrsConverter", (Converter,), {"crs": "EPSG:3035"})
Expand Down
43 changes: 43 additions & 0 deletions tests/test_encoding_geoparquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import pyarrow as pa
import pyarrow.parquet as pq
import pyarrow.types as pat
import pytest

from vecorel_cli.encoding.geoparquet import GeoParquet
from vecorel_cli.vecorel.collection import Collection
Expand Down Expand Up @@ -179,6 +180,48 @@ def test_write_keeps_all_null_columns_in_the_data(tmp_parquet_file):
assert {"code", "area", "name"} <= set(pq.read_schema(tmp_parquet_file).names)


@pytest.mark.parametrize("with_schema", [True, False], ids=["schema", "no-schema"])
def test_write_drops_sub_millisecond_precision(tmp_parquet_file, with_schema):
from geopandas import GeoDataFrame
from shapely.geometry import box

times = ["2021-01-01 10:00:00.123456", "1969-12-31 23:59:59.999500", None]
gdf = GeoDataFrame(
{
"id": ["1", "2", "3"],
"t": pd.to_datetime(times, utc=True, format="ISO8601"),
"geometry": [box(0, 0, 1, 1), box(1, 0, 2, 1), box(2, 0, 3, 1)],
},
crs="EPSG:4326",
)
gp = GeoParquet(tmp_parquet_file)
gp.set_collection(
{
"schemas": {"c": ["https://vecorel.org/specification/v0.1.0/schema.yaml"]},
**(
{
"schemas:custom": {
"$schema": "https://vecorel.org/sdl/v0.2.0/schema.json",
"properties": {"t": {"type": "date-time"}},
}
}
if with_schema
else {}
),
"collection": "c",
}
)
gp.write(gdf)

assert pq.read_schema(tmp_parquet_file).field("t").type == pa.timestamp("ms", tz="UTC")
written = pq.read_table(tmp_parquet_file, columns=["t"]).column("t").to_pylist()
assert [t.isoformat() if t else None for t in written] == [
"2021-01-01T10:00:00.123000+00:00",
"1969-12-31T23:59:59.999000+00:00",
None,
]


def test_get_pyarrow_type_keeps_the_schema():
from vecorel_cli.parquet.types import get_pyarrow_type

Expand Down
28 changes: 27 additions & 1 deletion vecorel_cli/encoding/geoparquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import Optional, Union

import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.parquet as pq
from geopandas import GeoDataFrame
from geopandas.array import from_wkb
Expand Down Expand Up @@ -238,6 +239,8 @@ def write(
try:
pa_type = get_pyarrow_type_for_geopandas(pd_type) # pyarrow data type
if pa_type is not None:
if pa.types.is_timestamp(pa_type):
data[column] = data[column].dt.floor("ms")
self.warning(
f"{column}: No schema defined, converting {pd_type} to nullable {pa_type}",
)
Expand Down Expand Up @@ -288,6 +291,29 @@ def write(

return True

@staticmethod
def _truncate_timestamps(tbl, schema):
# date-times are stored in milliseconds; the specification accepts losing finer precision
units = {"s": 1, "ms": 10**3, "us": 10**6, "ns": 10**9}
for field in schema:
if field.name not in tbl.column_names or not pa.types.is_timestamp(field.type):
continue
column = tbl[field.name]
if not pa.types.is_timestamp(column.type):
continue
factor = units[column.type.unit] // units[field.type.unit]
if factor <= 1:
continue
# floored like pandas' dt.floor, also before 1970, so the safe cast loses nothing
values = column.cast(pa.int64())
floored = pc.divide(values, factor)
Comment thread
m-mohr marked this conversation as resolved.
Resolved
floored = pc.if_else(
pc.greater(pc.multiply(floored, factor), values), pc.subtract(floored, 1), floored
)
floored = floored.cast(pa.timestamp(field.type.unit, tz=column.type.tz))
tbl = tbl.set_column(tbl.column_names.index(field.name), field.name, floored)
return tbl

def postprocess(
self,
schema_map: SchemaMapping = {},
Expand Down Expand Up @@ -529,7 +555,7 @@ def _rewrite(
# The safe cast fails on lossy conversions (e.g. out-of-range integers),
# so invalid source data is reported instead of silently corrupted
if tbl.schema != new_schema:
tbl = tbl.cast(new_schema)
tbl = self._truncate_timestamps(tbl, new_schema).cast(new_schema)
writer.write_table(tbl)
finally:
writer.close()
Expand Down
5 changes: 4 additions & 1 deletion vecorel_cli/parquet/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,11 @@ def get_geopandas_dtype(type, required=False, schema={}, return_category=False):
return lambda series: series.apply(lambda x: np.array(x))
elif type == "object":
return "object"
elif type == "date" or type == "date-time":
elif type == "date":
return lambda series: pd.to_datetime(series)
elif type == "date-time":
# stored in milliseconds; the specification accepts losing finer precision
return lambda series: pd.to_datetime(series).dt.floor("ms")
Comment thread
m-mohr marked this conversation as resolved.
Resolved
elif type == "geometry":
return lambda series: series # don't convert geometry
elif type == "bounding-box":
Expand Down
Loading