Skip to content

Commit fd5cf9c

Browse files
alxmrsclaude
andcommitted
Register the in-memory table itself, not a view of it
A view over the in-memory table made a query's plan scan the table's generated placeholder name instead of the registered one. ddx finds the table it differentiates by that name, so gradients with respect to a small in-memory Dataset failed with UnknownColumn. Register the in-memory table under a placeholder, take the table and drop the placeholder, so the plan scans the caller's name. The table still shares the Dataset's memory. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h
1 parent 1bde0be commit fd5cf9c

2 files changed

Lines changed: 17 additions & 10 deletions

File tree

‎tests/test_sql.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -592,8 +592,9 @@ def _scans(ctx, table):
592592

593593

594594
def test_small_in_memory_dataset_is_a_native_table_with_the_same_rows():
595-
"""A small in-memory Dataset becomes a native in-memory table, and
596-
queries over it give what a lazily read Dataset gives."""
595+
"""A small in-memory Dataset becomes a native in-memory table, which a
596+
query's plan scans by its registered name, and queries over it give what
597+
a lazily read Dataset gives."""
597598
ds = xr.Dataset(
598599
{"air": (("time", "lat"), np.random.rand(6, 4))},
599600
coords={"time": np.arange(6), "lat": np.arange(4) * 2.5},
@@ -604,6 +605,8 @@ def test_small_in_memory_dataset_is_a_native_table_with_the_same_rows():
604605

605606
assert "XarrayScanExec" not in _scans(ctx, "eager")
606607
assert "XarrayScanExec" in _scans(ctx, "lazy")
608+
plan = ctx.sql("SELECT * FROM eager").logical_plan().display_indent()
609+
assert plan.splitlines()[-1].strip() == "TableScan: eager"
607610
query = "SELECT time, lat, air FROM {} WHERE lat > 2 ORDER BY time, lat"
608611
pd.testing.assert_frame_equal(
609612
ctx.sql(query.format("eager")).to_pandas(),

‎xarray_sql/sql.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import uuid
2+
13
import pyarrow as pa
24
import xarray as xr
35
from datafusion import SessionContext
@@ -182,14 +184,16 @@ def _from_dataset(
182184

183185
def _in_memory_table(self, partitions: list[list[pa.RecordBatch]]):
184186
"""A native DataFusion in-memory table of ``partitions``."""
185-
before = set(self.catalog().schema().names())
186-
view = self.create_dataframe(partitions).into_view()
187-
# create_dataframe registers its in-memory table under a generated
188-
# name. The view holds the table itself, not the name, so the name
189-
# can go.
190-
for name in set(self.catalog().schema().names()) - before:
191-
self.deregister_table(name)
192-
return view
187+
# The in-memory table is reachable from Python only through a
188+
# registered name, so register it under a placeholder, take the
189+
# table, and drop the placeholder. The table, not a view of it, is
190+
# what the caller registers, so a query's plan scans the caller's
191+
# name, which ddx needs to find the table it differentiates.
192+
placeholder = f"_xarray_sql_{uuid.uuid4().hex}"
193+
self.register_record_batches(placeholder, partitions)
194+
table = self.catalog().schema().table(placeholder)
195+
self.deregister_table(placeholder)
196+
return table
193197

194198
def _maybe_register_cftime_udf(self, ds: xr.Dataset) -> None:
195199
"""Auto-register a cftime() UDF for non-Gregorian cftime coordinates."""

0 commit comments

Comments
 (0)