Skip to content

Commit 32c9ee5

Browse files
authored
fix: register object-dtype and out-of-ns-range coordinates (xqlsystems#223) (xqlsystems#224)
## Overview `from_dataset` could not register several common NetCDF/xarray datasets: non-cftime object-dtype variables and coordinates (strings) raised `ArrowNotImplementedError`, and datetime coordinates outside the `datetime64[ns]` range raised `OverflowError`. This fixes both in `_parse_schema` and `_block_metadata`, and revives cftime partition pruning that an object-dtype skip had silently disabled. Fixes xqlsystems#223. ## Before ```python ds = xr.Dataset({"label": (["x"], np.array(["a", "b"], dtype=object))}, coords={"x": [1, 2]}) XarrayContext().from_dataset("t", ds, chunks={"x": 2}) # pyarrow.lib.ArrowNotImplementedError: Unsupported numpy type 17 times = xr.date_range("0001-01-01", periods=3, freq="100YS", use_cftime=True).to_datetimeindex(time_unit="us") ds = xr.Dataset({"v": (["time"], np.arange(3.0))}, coords={"time": times}) XarrayContext().from_dataset("t", ds, chunks={"time": 2}) # OverflowError: Cannot convert Timestamp to nanoseconds without overflow ``` ## After ```python # both register and query cleanly c = XarrayContext(); c.from_dataset("t", ds, chunks={"x": 2}) c.sql("SELECT label FROM t ORDER BY x").to_pandas()["label"].tolist() # ['a', 'b'] # string coordinates, cftime calendars, and pre-1678 / post-2262 dates all register; # out-of-int64 datetimes skip pruning instead of raising, so registration always succeeds. ```
1 parent 6720f89 commit 32c9ee5

4 files changed

Lines changed: 209 additions & 17 deletions

File tree

‎tests/test_cft.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,14 @@ def test_non_gregorian_returns_int64_tag(self, ds_360day):
145145
assert tag == "int64"
146146
assert lo < hi
147147

148+
def test_out_of_int64_range_returns_none(self):
149+
# Year-1 gregorian dates exceed the int64 nanosecond range, so no
150+
# pruning bound can be reported; the caller skips the dimension.
151+
values = xr.date_range(
152+
"0001-01-01", periods=3, freq="100YS", use_cftime=True
153+
).values
154+
assert cft.partition_bounds(values) is None
155+
148156

149157
# -- Integration with _parse_schema ----------------------------------------
150158

‎tests/test_df.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
from xarray_sql.df import (
1010
DEFAULT_BATCH_SIZE,
11+
_ensure_default_indexes,
1112
_parse_schema,
1213
block_slices,
1314
compute_chunks,
@@ -16,6 +17,7 @@
1617
from_map,
1718
from_map_batched,
1819
iter_record_batches,
20+
partition_metadata,
1921
pivot,
2022
)
2123
from xarray_sql.reader import read_xarray, read_xarray_table
@@ -532,3 +534,138 @@ def test_compute_chunks_tuples_sum_to_dim_size():
532534
result = compute_chunks(ds, {"a": 3, "b": 4, "c": 5})
533535
for dim, tup in result.items():
534536
assert sum(tup) == ds.sizes[dim]
537+
538+
539+
# -- Object-dtype and out-of-ns-range coordinate support --------------------
540+
541+
542+
def _field_type(schema, name):
543+
return schema.field(name).type
544+
545+
546+
def test_parse_schema_maps_object_string_data_var_to_string():
547+
# A string variable arrives as numpy object dtype; _parse_schema must not
548+
# hand it to pa.from_numpy_dtype (which raises "Unsupported numpy type 17").
549+
ds = xr.Dataset(
550+
{"label": (["x"], np.array(["a", "b"], dtype=object))},
551+
coords={"x": [1, 2]},
552+
)
553+
schema = _parse_schema(_ensure_default_indexes(ds))
554+
assert _field_type(schema, "label") == pa.string()
555+
556+
557+
def test_parse_schema_maps_object_string_coord_to_string():
558+
# A string dimension coordinate (e.g. station names) is object dtype too.
559+
ds = xr.Dataset(
560+
{"v": (["station"], [1.0, 2.0])},
561+
coords={"station": np.array(["A", "B"], dtype=object)},
562+
)
563+
schema = _parse_schema(_ensure_default_indexes(ds))
564+
assert _field_type(schema, "station") == pa.string()
565+
566+
567+
def test_partition_metadata_skips_out_of_ns_datetime():
568+
# datetime64 coordinates outside the datetime64[ns] range (pre-1678 /
569+
# post-2262) cannot be represented as int64 nanoseconds, so partition
570+
# pruning must be skipped for that dimension rather than raising
571+
# OverflowError. Registration must still succeed.
572+
times = xr.date_range(
573+
"0001-01-01", periods=3, freq="100YS", use_cftime=True
574+
).to_datetimeindex(time_unit="us", unsafe=True)
575+
ds = _ensure_default_indexes(
576+
xr.Dataset({"v": (["time"], np.arange(3.0))}, coords={"time": times})
577+
)
578+
blocks = list(block_slices(ds, chunks={"time": 2}))
579+
580+
meta = partition_metadata(ds, blocks) # must not raise
581+
582+
assert len(meta) == len(blocks)
583+
# "time" is unpruneable here, so it is omitted from every partition.
584+
assert all("time" not in m for m in meta)
585+
586+
587+
def test_parse_schema_all_null_object_var_stays_null():
588+
# An all-null object column has no data to infer a type from; let null be
589+
# null rather than coercing it to a string column.
590+
ds = _ensure_default_indexes(
591+
xr.Dataset(
592+
{"label": (["x"], np.array([None, None], dtype=object))},
593+
coords={"x": [1, 2]},
594+
)
595+
)
596+
schema = _parse_schema(ds)
597+
assert pa.types.is_null(schema.field("label").type)
598+
599+
600+
def test_partition_metadata_prunes_cftime_coord():
601+
# cftime dimension coordinates must produce pruning bounds; previously the
602+
# object-dtype skip shadowed the cftime branch, silently disabling pruning.
603+
times = xr.date_range(
604+
"2000-01-01", periods=4, freq="1D", calendar="noleap", use_cftime=True
605+
)
606+
ds = _ensure_default_indexes(
607+
xr.Dataset({"v": (["time"], np.arange(4.0))}, coords={"time": times})
608+
)
609+
blocks = list(block_slices(ds, chunks={"time": 2}))
610+
611+
meta = partition_metadata(ds, blocks)
612+
613+
assert all("time" in m for m in meta)
614+
for m in meta:
615+
_, _, tag = m["time"]
616+
assert tag == "timestamp_ns"
617+
618+
619+
def test_partition_metadata_skips_ancient_cftime():
620+
# Ancient gregorian cftime dates overflow the int64 nanosecond range, so
621+
# pruning must be skipped for that dim (no raise, dim omitted).
622+
times = xr.date_range(
623+
"0001-01-01", periods=3, freq="100YS", use_cftime=True
624+
)
625+
ds = _ensure_default_indexes(
626+
xr.Dataset({"v": (["time"], np.arange(3.0))}, coords={"time": times})
627+
)
628+
blocks = list(block_slices(ds, chunks={"time": 2}))
629+
630+
meta = partition_metadata(ds, blocks) # must not raise
631+
632+
assert all("time" not in m for m in meta)
633+
634+
635+
def test_string_dataset_round_trips_through_record_batch():
636+
# The schema fix must also flow through the batch builders: a string
637+
# column has to materialize as an Arrow string array, not error out.
638+
ds = _ensure_default_indexes(
639+
xr.Dataset(
640+
{"label": (["x"], np.array(["a", "b", "c", "d"], dtype=object))},
641+
coords={"x": [10, 20, 30, 40]},
642+
)
643+
)
644+
schema = _parse_schema(ds)
645+
646+
batch = dataset_to_record_batch(ds, schema)
647+
assert batch.schema.field("label").type == pa.string()
648+
assert batch.column("label").to_pylist() == ["a", "b", "c", "d"]
649+
650+
# The streaming path must agree with the one-shot path.
651+
streamed = pa.Table.from_batches(
652+
list(iter_record_batches(ds, schema, batch_size=2)), schema=schema
653+
)
654+
assert streamed.column("label").to_pylist() == ["a", "b", "c", "d"]
655+
656+
657+
def test_partition_metadata_in_range_datetime_still_pruned():
658+
# Regression guard: ordinary datetimes must keep producing timestamp_ns
659+
# bounds so filter pushdown still works after the overflow fix.
660+
times = pd.date_range("2000-01-01", periods=4, freq="D")
661+
ds = _ensure_default_indexes(
662+
xr.Dataset({"v": (["time"], np.arange(4.0))}, coords={"time": times})
663+
)
664+
blocks = list(block_slices(ds, chunks={"time": 2}))
665+
666+
meta = partition_metadata(ds, blocks)
667+
668+
assert all("time" in m for m in meta)
669+
for m in meta:
670+
_, _, tag = m["time"]
671+
assert tag == "timestamp_ns"

‎xarray_sql/cftime.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,18 +172,27 @@ def convert_for_field(values, field: pa.Field) -> np.ndarray:
172172

173173
def partition_bounds(
174174
values,
175-
) -> tuple[int, int, str]:
175+
) -> tuple[int, int, str] | None:
176176
"""Return ``(min, max, dtype_tag)`` for a cftime coordinate slice.
177177
178178
Gregorian-like calendars return nanosecond bounds tagged
179179
``"timestamp_ns"`` (compatible with ``ScalarBound::TimestampNanos``
180180
in the Rust pruning layer). Non-Gregorian calendars return int64
181181
offsets tagged ``"int64"``.
182+
183+
Returns ``None`` when the nanosecond bound falls outside the int64 range
184+
(e.g. paleoclimate dates before ~1678), signalling the caller to skip
185+
pruning for that dimension rather than emit a bound the Rust layer would
186+
reject.
182187
"""
183188
cal = values.ravel()[0].calendar
184189
if is_gregorian_like(cal):
185190
us = to_microseconds(values)
186-
return int(us.min()) * 1_000, int(us.max()) * 1_000, "timestamp_ns"
191+
lo, hi = int(us.min()) * 1_000, int(us.max()) * 1_000
192+
int64 = np.iinfo(np.int64)
193+
if lo < int64.min or hi > int64.max:
194+
return None
195+
return lo, hi, "timestamp_ns"
187196
offsets = to_offsets(values, DEFAULT_UNITS, cal)
188197
return int(offsets.min()), int(offsets.max()), "int64"
189198

‎xarray_sql/df.py‎

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,20 @@ def iter_record_batches(
386386
yield pa.RecordBatch.from_arrays(arrays, schema=schema)
387387

388388

389+
def _arrow_type_for_object(values: np.ndarray) -> pa.DataType:
390+
"""Infer an Arrow type for a non-cftime object-dtype array.
391+
392+
``pa.from_numpy_dtype`` cannot map numpy object dtype, so let pyarrow infer
393+
the type from the data instead: strings become ``pa.string()``, bytes
394+
``pa.binary()``, and other representable Python scalars their Arrow
395+
equivalent. An all-null array stays ``pa.null()``, and a column mixing
396+
incompatible types (e.g. str and int) raises, surfacing a clear error
397+
rather than a silent coercion. Object-dtype arrays are never Dask/Zarr
398+
backed, so this triggers no remote I/O.
399+
"""
400+
return pa.array(np.asarray(values).ravel()).type
401+
402+
389403
def _parse_schema(ds: xr.Dataset) -> pa.Schema:
390404
"""Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns.
391405
@@ -414,19 +428,29 @@ def _parse_schema(ds: xr.Dataset) -> pa.Schema:
414428
if cft.is_cftime_index(ds, coord_name):
415429
units, calendar = cft.encoding(ds, coord_name)
416430
columns.append(cft.arrow_field(coord_name, units, calendar))
431+
elif coord_var.dtype == np.dtype("O"):
432+
# Object dtype that isn't cftime (e.g. string station names).
433+
arrow_type = _arrow_type_for_object(coord_var.values)
434+
columns.append(pa.field(coord_name, arrow_type))
417435
else:
418436
pa_type = pa.from_numpy_dtype(coord_var.dtype)
419437
columns.append(pa.field(coord_name, pa_type))
420438

421439
for var_name, var in ds.data_vars.items():
422-
# Data variables are virtually never cftime, but check dtype as a
423-
# cheap guard. Only fall back to _is_cftime (which materializes
424-
# element 0) when dtype is object.
425-
if var.dtype == np.dtype("O") and cft.is_cftime(var.values):
426-
# Rare: a data variable holding cftime objects. Use same encoding
427-
# as the first cftime dimension coordinate, or default.
428-
cal = var.values.ravel()[0].calendar
429-
columns.append(cft.arrow_field(var_name, cft.DEFAULT_UNITS, cal))
440+
# An object-dtype data variable may hold cftime objects (encode it like
441+
# a cftime coordinate) or strings/other Python scalars (infer the Arrow
442+
# type from the data). The dtype check keeps the common numeric path off
443+
# the object branch.
444+
if var.dtype == np.dtype("O"):
445+
if cft.is_cftime(var.values):
446+
# Encode with the same units/calendar as a cftime coordinate.
447+
cal = var.values.ravel()[0].calendar
448+
columns.append(
449+
cft.arrow_field(var_name, cft.DEFAULT_UNITS, cal)
450+
)
451+
else:
452+
arrow_type = _arrow_type_for_object(var.values)
453+
columns.append(pa.field(var_name, arrow_type))
430454
else:
431455
pa_type = pa.from_numpy_dtype(var.dtype)
432456
columns.append(pa.field(var_name, pa_type))
@@ -466,26 +490,40 @@ def _block_metadata(
466490
coord_values = coord_arrays[str(dim)][slc]
467491
if len(coord_values) == 0:
468492
continue
493+
# cftime coordinates are object dtype but carry their own bound
494+
# encoding, so they must be handled before the string/object skip
495+
# below (otherwise pruning is silently disabled for them).
496+
# partition_bounds returns None when the bound overflows int64.
497+
if cft.is_cftime(coord_values):
498+
bounds = cft.partition_bounds(coord_values)
499+
if bounds is not None:
500+
ranges[str(dim)] = bounds
501+
continue
469502
# String/object dtypes are not representable as ScalarBound
470503
# (Int64/Float64/TimestampNanos) and numpy min/max ufuncs do not
471504
# support them. Skip so pruning treats the dimension conservatively.
472505
if coord_values.dtype.kind in ("U", "S", "O"):
473506
continue
474507

475-
if cft.is_cftime(coord_values):
476-
ranges[str(dim)] = cft.partition_bounds(coord_values)
477-
continue
478-
479508
# Use actual min/max rather than first/last so that non-monotonic
480509
# coordinate axes (e.g. descending latitude 90→-90) are handled
481510
# correctly. np.min/max work for both numeric and datetime64 arrays.
482511
min_val = coord_values.min()
483512
max_val = coord_values.max()
484513

485514
if isinstance(min_val, (np.datetime64, pd.Timestamp)):
486-
min_val = int(pd.Timestamp(min_val).value)
487-
max_val = int(pd.Timestamp(max_val).value)
488-
ranges[str(dim)] = (min_val, max_val, "timestamp_ns")
515+
# The Rust pruning layer only accepts int64 nanosecond bounds
516+
# (ScalarBound::TimestampNanos). Dates outside the
517+
# datetime64[ns] range (pre-1678 / post-2262) cannot be
518+
# represented, so skip pruning for this dimension rather than
519+
# raising -- registration still succeeds and the Rust pruner
520+
# treats a missing dimension conservatively (never prunes on it).
521+
try:
522+
min_ns = int(pd.Timestamp(min_val).value)
523+
max_ns = int(pd.Timestamp(max_val).value)
524+
except (OverflowError, pd.errors.OutOfBoundsDatetime):
525+
continue
526+
ranges[str(dim)] = (min_ns, max_ns, "timestamp_ns")
489527
elif hasattr(min_val, "item"):
490528
min_val = min_val.item()
491529
max_val = max_val.item()

0 commit comments

Comments
 (0)