|
| 1 | +import re |
1 | 2 | import time |
2 | 3 | import uuid |
3 | 4 |
|
@@ -64,6 +65,11 @@ def __init__(self, config: SessionConfig | None = None, *args, **kwargs): |
64 | 65 | # in SQL (e.g. ``"air"`` for a uniform-dim Dataset, or |
65 | 66 | # ``"era5.surface"`` for one entry from a multi-dim-group split). |
66 | 67 | self._registered_datasets: dict[str, xr.Dataset] = {} |
| 68 | + # How many times each name's table has been removed or replaced. |
| 69 | + # DataFusion refuses to register a name that's taken, so a table |
| 70 | + # registered again from Python was deregistered first; SQL that |
| 71 | + # creates or drops a table counts too (see sql()). |
| 72 | + self._deregistrations: dict[str, int] = {} |
67 | 73 | # With pyproj installed, every context speaks CRS out of the box: |
68 | 74 | # reproject(x, y, src_crs, dst_crs), à la PostGIS ST_Transform. |
69 | 75 | if _proj is not None: |
@@ -242,13 +248,21 @@ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame: |
242 | 248 | Returns: |
243 | 249 | An [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapping the DataFusion DataFrame. |
244 | 250 | """ |
| 251 | + # DataFusion runs a statement that creates or drops a table as it |
| 252 | + # plans it, replacing a table without deregistering it. |
| 253 | + replaced = _table_replaced(query) |
| 254 | + if replaced is not None: |
| 255 | + self._deregistrations[replaced] = ( |
| 256 | + self._deregistrations.get(replaced, 0) + 1 |
| 257 | + ) |
245 | 258 | return self._wrap(super().sql(query, *args, **kwargs)) |
246 | 259 |
|
247 | 260 | def deregister_table(self, name: str) -> None: |
248 | 261 | """Remove a table from the session, and forget the Dataset |
249 | 262 | registered under its name.""" |
250 | 263 | super().deregister_table(name) |
251 | 264 | self._registered_datasets.pop(name, None) |
| 265 | + self._deregistrations[name] = self._deregistrations.get(name, 0) + 1 |
252 | 266 |
|
253 | 267 | def prepare(self, query: str) -> "PreparedQuery": |
254 | 268 | """Plan a query once, to run it many times. |
@@ -292,11 +306,13 @@ def _write(self, writes: dict[str, xr.Dataset]) -> None: |
292 | 306 | f"{sorted(registered.data_vars)}, and the new Dataset " |
293 | 307 | f"has {sorted(ds.data_vars)}" |
294 | 308 | ) |
295 | | - for dim in registered.dims: |
296 | | - if dim not in ds.dims or not ds[dim].equals(registered[dim]): |
| 309 | + for coord in set(registered.coords) | set(ds.coords): |
| 310 | + if coord not in ds.coords or not ds[coord].equals( |
| 311 | + registered.coords.get(coord) |
| 312 | + ): |
297 | 313 | raise ValueError( |
298 | 314 | f"the new Dataset for {name!r} has other coordinates " |
299 | | - f"along {dim!r}; register it again, and prepare the " |
| 315 | + f"({coord!r}); register it again, and prepare the " |
300 | 316 | "query again" |
301 | 317 | ) |
302 | 318 | for var in registered.data_vars: |
@@ -361,16 +377,24 @@ class PreparedQuery: |
361 | 377 |
|
362 | 378 | def __init__(self, ctx: Context, query: str): |
363 | 379 | self._ctx = ctx |
364 | | - self._query = query |
365 | 380 | start = time.perf_counter() |
366 | 381 | df = SessionContext.sql(ctx, query) |
367 | 382 | plan = df.execution_plan() |
368 | 383 | planning = time.perf_counter() - start |
369 | 384 | self._schema = df.schema() |
370 | | - # The Datasets the query reads, as they are registered now. |
| 385 | + # Planned again from its optimized form, the query reads the tables |
| 386 | + # it was prepared with, and functions such as now() keep the values |
| 387 | + # they were folded into. |
| 388 | + self._optimized = df.optimized_logical_plan() |
| 389 | + # Each table the query reads, by how often its name had been |
| 390 | + # deregistered; and of those, the Datasets a call may write into. |
| 391 | + self._read = { |
| 392 | + name: ctx._deregistrations.get(name, 0) |
| 393 | + for name in _tables_read(df.logical_plan()) |
| 394 | + } |
371 | 395 | self._registered = { |
372 | 396 | name: ctx._registered_datasets[name] |
373 | | - for name in _tables_read(df.logical_plan()) |
| 397 | + for name in self._read |
374 | 398 | if name in ctx._registered_datasets |
375 | 399 | } |
376 | 400 | # A plan keeps state from a run (a hash join keeps the table it |
@@ -404,15 +428,16 @@ def __call__(self, **tables: xr.Dataset) -> pa.Table: |
404 | 428 | f"the query reads no Dataset registered as {name!r} " |
405 | 429 | "when it was prepared" |
406 | 430 | ) |
407 | | - for name, registered in self._registered.items(): |
408 | | - if self._ctx._registered_datasets.get(name) is not registered: |
| 431 | + for name, deregistrations in self._read.items(): |
| 432 | + if self._ctx._deregistrations.get(name, 0) != deregistrations: |
409 | 433 | raise ValueError( |
410 | 434 | f"{name!r} has been registered again since the query was " |
411 | 435 | "prepared; prepare the query again" |
412 | 436 | ) |
413 | 437 | self._ctx._write(tables) |
414 | 438 | if self._plan is None: |
415 | | - return SessionContext.sql(self._ctx, self._query).to_arrow_table() |
| 439 | + df = self._ctx.create_dataframe_from_logical_plan(self._optimized) |
| 440 | + return df.to_arrow_table() |
416 | 441 | plan = _copy(self._ctx, self._plan) |
417 | 442 | batches = [ |
418 | 443 | batch.to_pyarrow() |
@@ -444,6 +469,24 @@ def _tables_read(plan: LogicalPlan) -> set[str]: |
444 | 469 | return names |
445 | 470 |
|
446 | 471 |
|
| 472 | +_TABLE_DDL = re.compile( |
| 473 | + r"\s*(?:CREATE\s+(?:OR\s+REPLACE\s+)?(?:EXTERNAL\s+)?TABLE|DROP\s+TABLE)" |
| 474 | + r"\s+(?:IF\s+(?:NOT\s+)?EXISTS\s+)?(\"[^\"]+\"|[\w.]+)", |
| 475 | + re.IGNORECASE, |
| 476 | +) |
| 477 | + |
| 478 | + |
| 479 | +def _table_replaced(query: str) -> str | None: |
| 480 | + """The name of the table ``query`` creates or drops, as DataFusion |
| 481 | + resolves it, or ``None`` for any other statement.""" |
| 482 | + match = _TABLE_DDL.match(query) |
| 483 | + if match is None: |
| 484 | + return None |
| 485 | + name = match.group(1) |
| 486 | + # DataFusion keeps a quoted name's case, and lowers an unquoted one. |
| 487 | + return name[1:-1] if name.startswith('"') else name.lower() |
| 488 | + |
| 489 | + |
447 | 490 | def _missing(values: np.ndarray) -> np.ndarray | None: |
448 | 491 | """Where ``values`` holds NaN or NaT, which xarray-sql reads as NULL, or |
449 | 492 | ``None`` for a dtype that holds neither.""" |
|
0 commit comments