Skip to content

Commit bfc7f3a

Browse files
author
Stefan Jansen
committed
fix(data): preserve panel provenance through pandas persistence
Keep axis identity and native schema metadata serializable without Arrow as a runtime requirement. Exercise an actual pandas Parquet round trip retaining absent observations, nulls and source dtypes. Document the native panel APIs and remove unqualified storage timing claims. Refs #117
1 parent 1d013c7 commit bfc7f3a

4 files changed

Lines changed: 55 additions & 12 deletions

File tree

‎docs/api/index.md‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ key = manager.update("AAPL")
5959

6060
### StorageConfig
6161

62-
Dataclass configuring the storage backend. Controls partitioning strategy,
62+
Validated configuration for the storage backend. Controls partitioning strategy,
6363
compression, locking, and metadata tracking.
6464

6565
```python
@@ -106,8 +106,8 @@ implement this contract.
106106

107107
### HiveStorage
108108

109-
Hive-partitioned storage with configurable time-based partitioning. Delivers
110-
7x query performance improvement for time-range queries via partition pruning.
109+
Hive-partitioned storage with configurable time-based partitioning and
110+
time-range partition pruning.
111111

112112
```python
113113
from ml4t.data.storage import HiveStorage, StorageConfig
@@ -459,3 +459,25 @@ elapsed days into row counts.
459459
members:
460460
- __init__
461461
- select_contracts
462+
463+
## Price-table representations
464+
465+
Convert one price field between native long observations and a pandas clock-by-asset panel.
466+
The functions preserve supplied values and missing cells; they do not fill prices, infer
467+
availability or calculate returns. Other observation fields remain in the original table.
468+
The panel's attrs retain the declared feed, source dtypes and sparse presence information.
469+
Default metadata supports pandas Parquet round trips; caller-supplied provenance must also be
470+
JSON-serializable to use pandas' metadata export.
471+
472+
See the [offline migration recipe](../getting-started/migration.md#move-a-long-price-table-to-a-pandas-analytical-panel)
473+
for required input columns, mapping conflicts and the qualified consumer versions.
474+
475+
::: ml4t.data.utils.conversion.to_price_panel
476+
options:
477+
show_source: false
478+
heading_level: 3
479+
480+
::: ml4t.data.utils.conversion.from_price_panel
481+
options:
482+
show_source: false
483+
heading_level: 3

‎src/ml4t/data/storage/hive.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
"""Hive partitioned storage implementation.
22
3-
Provides efficient time-series data storage using Hive-style partitioning
4-
with measured 7x query performance improvement.
3+
Provides time-series data storage using Hive-style partitioning.
54
"""
65

76
from __future__ import annotations
@@ -54,7 +53,7 @@ class HiveStorage(StorageBackend):
5453
"""Hive partitioned storage with configurable time-based partitioning.
5554
5655
This implementation provides:
57-
- 7x query performance improvement for time-based queries
56+
- Partition pruning for time-based queries
5857
- Configurable partition granularity (year, month, day, hour)
5958
- Atomic writes with temp file pattern
6059
- Metadata tracking in JSON manifests

‎src/ml4t/data/utils/conversion.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from collections.abc import Mapping
66
from copy import deepcopy
7+
from io import BytesIO
78
from typing import Any
89

910
import numpy as np
@@ -165,6 +166,10 @@ def _panel_feed(
165166
return feed.with_overrides(**overrides)
166167

167168

169+
def _axis_keys(index: pd.Index) -> list[str]:
170+
return [f"{type(value).__name__}:{value!r}" for value in index.tolist()]
171+
172+
168173
def to_price_panel(
169174
data: pl.DataFrame | pd.DataFrame,
170175
*,
@@ -223,9 +228,13 @@ def to_price_panel(
223228
"feed_spec": feed.to_dict(),
224229
"metadata": deepcopy(dict(metadata or {})),
225230
"observed": present.to_numpy().reshape(panel.shape).tolist(),
226-
"clock": clock.tolist(),
227-
"assets": assets.tolist(),
228-
"schema": dict(data.select(names).schema) if isinstance(data, pl.DataFrame) else None,
231+
"clock": _axis_keys(clock),
232+
"assets": _axis_keys(assets),
233+
"schema": (
234+
data.select(names).head(0).write_ipc(None).getvalue().hex()
235+
if isinstance(data, pl.DataFrame)
236+
else None
237+
),
229238
}
230239
if isinstance(data, pl.DataFrame) and data.schema[feed.price_col].is_float():
231240
nulls = pd.Series(data[feed.price_col].is_null().to_numpy(), index=indexed.index)
@@ -286,9 +295,8 @@ def from_price_panel(
286295
if observed is None:
287296
mask = np.ones(panel.shape, dtype=bool)
288297
else:
289-
if (
290-
contract.get("clock") != panel.index.tolist()
291-
or contract.get("assets") != panel.columns.tolist()
298+
if contract.get("clock") != _axis_keys(panel.index) or contract.get("assets") != _axis_keys(
299+
panel.columns
292300
):
293301
raise ValueError("Panel observation mask does not match its axes")
294302
mask = _panel_mask(observed, panel.shape)
@@ -307,6 +315,8 @@ def from_price_panel(
307315
if isinstance(timestamp_dtype, pd.DatetimeTZDtype):
308316
result = result.with_columns(pl.col(names[0]).dt.convert_time_zone(str(timestamp_dtype.tz)))
309317
schema = contract.get("schema")
318+
if isinstance(schema, str):
319+
schema = dict(pl.read_ipc(BytesIO(bytes.fromhex(schema))).schema)
310320
if schema is not None:
311321
expressions = []
312322
for name, dtype in schema.items():

‎tests/test_market_input_handoff.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,3 +442,15 @@ def test_pandas_input_retains_a_late_local_session_label_without_a_timezone_decl
442442
feed = FeedSpec(time_contract_version=2, calendar=None, timestamp_semantics="session_label")
443443
restored = from_price_panel(to_price_panel(polars_to_pandas(source), feed_spec=feed))
444444
assert_frame_equal(restored, source)
445+
446+
447+
def test_price_panel_attrs_do_not_break_pandas_parquet_and_sparse_inverse(tmp_path):
448+
pytest.importorskip("pyarrow")
449+
source = captured_prices().filter(
450+
~((pl.col("asset") == 23) & (pl.col("timestamp").dt.day() == 8))
451+
)
452+
panel = to_price_panel(source)
453+
path = tmp_path / "prices.parquet"
454+
panel.to_parquet(path)
455+
restored = from_price_panel(pd.read_parquet(path))
456+
assert_frame_equal(restored, source)

0 commit comments

Comments
 (0)