Skip to content

Commit 6719421

Browse files
alxmrsclaude
andcommitted
Address review: every coordinate, every table read, now() when planning again
- A write must keep every coordinate, not only each dim's: a coordinate along two dims went unchecked. - Track every table a prepared query reads, not only the Datasets, by how often its name has been removed or replaced: by deregister_table, or by SQL that creates or drops a table, which DataFusion runs without deregistering. - A call that plans again starts from the optimized plan kept when the query was prepared, so it reads the tables it was prepared with, and now() keeps its value, as on the other path. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h
1 parent 059af80 commit 6719421

2 files changed

Lines changed: 92 additions & 9 deletions

File tree

‎tests/test_prepare.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,43 @@ def test_now_is_the_time_the_query_was_prepared(ctx):
190190
first = query()
191191
time.sleep(0.01)
192192
assert query().equals(first)
193+
194+
195+
def test_a_write_must_keep_every_coordinate(ctx):
196+
query = ctx.prepare(JOIN)
197+
# A coordinate along both dims, which neither dim's coordinate holds.
198+
labelled = _a(np.ones((4, 3))).assign_coords(
199+
area=(("i", "j"), np.ones((4, 3)))
200+
)
201+
with pytest.raises(ValueError, match="coordinates"):
202+
query(a=labelled)
203+
204+
205+
def test_a_table_not_from_a_dataset_counts_too(ctx):
206+
ctx.register_record_batches("c", [ctx.sql("SELECT i, j FROM a").collect()])
207+
query = ctx.prepare("SELECT COUNT(*) AS n FROM a JOIN c USING (i, j)")
208+
assert query().column("n").to_pylist() == [12]
209+
ctx.deregister_table("c")
210+
ctx.register_record_batches("c", [ctx.sql("SELECT i, j FROM b").collect()])
211+
with pytest.raises(ValueError, match="registered again"):
212+
query()
213+
214+
215+
def test_now_is_the_time_a_query_over_a_lazy_table_was_prepared():
216+
pytest.importorskip("dask")
217+
ctx = Context()
218+
ctx.from_dataset("a", _a(np.ones((4, 3))).chunk({"i": 2}))
219+
query = ctx.prepare("SELECT now() AS t, SUM(v) AS s FROM a")
220+
first = query()
221+
time.sleep(0.01)
222+
assert query().equals(first)
223+
224+
225+
def test_a_table_replaced_in_sql_counts_too():
226+
ctx = Context()
227+
ctx.sql("CREATE TABLE c AS VALUES (1), (2)").collect()
228+
query = ctx.prepare("SELECT COUNT(*) AS n FROM c")
229+
assert query().column("n").to_pylist() == [2]
230+
ctx.sql("CREATE OR REPLACE TABLE c AS VALUES (1), (2), (3)").collect()
231+
with pytest.raises(ValueError, match="registered again"):
232+
query()

‎xarray_sql/sql.py‎

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import re
12
import time
23
import uuid
34

@@ -64,6 +65,11 @@ def __init__(self, config: SessionConfig | None = None, *args, **kwargs):
6465
# in SQL (e.g. ``"air"`` for a uniform-dim Dataset, or
6566
# ``"era5.surface"`` for one entry from a multi-dim-group split).
6667
self._registered_datasets: dict[str, xr.Dataset] = {}
68+
# How many times each name's table has been removed or replaced.
69+
# DataFusion refuses to register a name that's taken, so a table
70+
# registered again from Python was deregistered first; SQL that
71+
# creates or drops a table counts too (see sql()).
72+
self._deregistrations: dict[str, int] = {}
6773
# With pyproj installed, every context speaks CRS out of the box:
6874
# reproject(x, y, src_crs, dst_crs), à la PostGIS ST_Transform.
6975
if _proj is not None:
@@ -242,13 +248,21 @@ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame:
242248
Returns:
243249
An [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapping the DataFusion DataFrame.
244250
"""
251+
# DataFusion runs a statement that creates or drops a table as it
252+
# plans it, replacing a table without deregistering it.
253+
replaced = _table_replaced(query)
254+
if replaced is not None:
255+
self._deregistrations[replaced] = (
256+
self._deregistrations.get(replaced, 0) + 1
257+
)
245258
return self._wrap(super().sql(query, *args, **kwargs))
246259

247260
def deregister_table(self, name: str) -> None:
248261
"""Remove a table from the session, and forget the Dataset
249262
registered under its name."""
250263
super().deregister_table(name)
251264
self._registered_datasets.pop(name, None)
265+
self._deregistrations[name] = self._deregistrations.get(name, 0) + 1
252266

253267
def prepare(self, query: str) -> "PreparedQuery":
254268
"""Plan a query once, to run it many times.
@@ -292,11 +306,13 @@ def _write(self, writes: dict[str, xr.Dataset]) -> None:
292306
f"{sorted(registered.data_vars)}, and the new Dataset "
293307
f"has {sorted(ds.data_vars)}"
294308
)
295-
for dim in registered.dims:
296-
if dim not in ds.dims or not ds[dim].equals(registered[dim]):
309+
for coord in set(registered.coords) | set(ds.coords):
310+
if coord not in ds.coords or not ds[coord].equals(
311+
registered.coords.get(coord)
312+
):
297313
raise ValueError(
298314
f"the new Dataset for {name!r} has other coordinates "
299-
f"along {dim!r}; register it again, and prepare the "
315+
f"({coord!r}); register it again, and prepare the "
300316
"query again"
301317
)
302318
for var in registered.data_vars:
@@ -361,16 +377,24 @@ class PreparedQuery:
361377

362378
def __init__(self, ctx: Context, query: str):
363379
self._ctx = ctx
364-
self._query = query
365380
start = time.perf_counter()
366381
df = SessionContext.sql(ctx, query)
367382
plan = df.execution_plan()
368383
planning = time.perf_counter() - start
369384
self._schema = df.schema()
370-
# The Datasets the query reads, as they are registered now.
385+
# Planned again from its optimized form, the query reads the tables
386+
# it was prepared with, and functions such as now() keep the values
387+
# they were folded into.
388+
self._optimized = df.optimized_logical_plan()
389+
# Each table the query reads, by how often its name had been
390+
# deregistered; and of those, the Datasets a call may write into.
391+
self._read = {
392+
name: ctx._deregistrations.get(name, 0)
393+
for name in _tables_read(df.logical_plan())
394+
}
371395
self._registered = {
372396
name: ctx._registered_datasets[name]
373-
for name in _tables_read(df.logical_plan())
397+
for name in self._read
374398
if name in ctx._registered_datasets
375399
}
376400
# A plan keeps state from a run (a hash join keeps the table it
@@ -404,15 +428,16 @@ def __call__(self, **tables: xr.Dataset) -> pa.Table:
404428
f"the query reads no Dataset registered as {name!r} "
405429
"when it was prepared"
406430
)
407-
for name, registered in self._registered.items():
408-
if self._ctx._registered_datasets.get(name) is not registered:
431+
for name, deregistrations in self._read.items():
432+
if self._ctx._deregistrations.get(name, 0) != deregistrations:
409433
raise ValueError(
410434
f"{name!r} has been registered again since the query was "
411435
"prepared; prepare the query again"
412436
)
413437
self._ctx._write(tables)
414438
if self._plan is None:
415-
return SessionContext.sql(self._ctx, self._query).to_arrow_table()
439+
df = self._ctx.create_dataframe_from_logical_plan(self._optimized)
440+
return df.to_arrow_table()
416441
plan = _copy(self._ctx, self._plan)
417442
batches = [
418443
batch.to_pyarrow()
@@ -444,6 +469,24 @@ def _tables_read(plan: LogicalPlan) -> set[str]:
444469
return names
445470

446471

472+
_TABLE_DDL = re.compile(
473+
r"\s*(?:CREATE\s+(?:OR\s+REPLACE\s+)?(?:EXTERNAL\s+)?TABLE|DROP\s+TABLE)"
474+
r"\s+(?:IF\s+(?:NOT\s+)?EXISTS\s+)?(\"[^\"]+\"|[\w.]+)",
475+
re.IGNORECASE,
476+
)
477+
478+
479+
def _table_replaced(query: str) -> str | None:
480+
"""The name of the table ``query`` creates or drops, as DataFusion
481+
resolves it, or ``None`` for any other statement."""
482+
match = _TABLE_DDL.match(query)
483+
if match is None:
484+
return None
485+
name = match.group(1)
486+
# DataFusion keeps a quoted name's case, and lowers an unquoted one.
487+
return name[1:-1] if name.startswith('"') else name.lower()
488+
489+
447490
def _missing(values: np.ndarray) -> np.ndarray | None:
448491
"""Where ``values`` holds NaN or NaT, which xarray-sql reads as NULL, or
449492
``None`` for a dtype that holds neither."""

0 commit comments

Comments
 (0)