Skip to content

Commit f9a5fa3

Browse files
Merge branch 'main' into fix/scheduled-futures-rolls
2 parents 44c6732 + e6568c0 commit f9a5fa3

6 files changed

Lines changed: 322 additions & 18 deletions

File tree

‎docs/user-guide/incremental-updates.md‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,12 @@ ml4t-data update --symbol AAPL --no-fill-gaps --storage-path ./data
4949
```
5050

5151
Daily detection counts missing sessions from the selected exchange schedule, not elapsed days.
52-
The updater currently selects NYSE for non-crypto data and 24/7 for crypto. Intraday detection still
53-
uses approximate US overnight/weekend rules. It is not an exchange-calendar reconstruction. Use
54-
[trading-session completion](sessions.md) when exact intraday schedules are required.
52+
The Python updater resolves its explicit `calendar`, then supplied feed or stored metadata, then
53+
NYSE. Asset class and descriptive exchange metadata do not select a schedule. Continuous feeds
54+
must declare `calendar="24/7"`; `calendar=None` disables schedule-derived detection and filling.
55+
Intraday detection through the updater requires a supported declared-grid consumer, which is not
56+
implemented yet. Use `fill_gaps=False` for intraday updates and inspect
57+
[trading-session completion](sessions.md) for the supported one-minute operation.
5558

5659
Detection does not itself fill or modify observations. The updater's separately enabled forward
5760
fill uses a regular elapsed-time grid, not the selected exchange schedule, and can insert closed
@@ -142,6 +145,19 @@ key = manager.update(
142145
`DataManager.update()` returns the storage key. It raises when storage is not configured, stored data
143146
is empty, metadata is invalid, or the provider or storage operation fails.
144147

148+
`load()`, `import_data()` and `update()` accept `calendar` and a shared `feed_spec`. New ingestion
149+
without a feed writes version-2 temporal metadata: an omitted calendar remains omitted, while
150+
explicit `None` remains disabled after storage and model projection. An existing version-1 feed
151+
is not automatically reinterpreted; its null calendar still means inheritance. An explicit calendar
152+
override writes version 2. Calendar names and conflicting frequency/bar-type declarations fail
153+
before a replacement generation is published.
154+
155+
Feed timezone, timestamp semantics and bar-grid declarations survive ingestion. A non-null
156+
`session_start_time` override is not supported by the native schedule adapter and is rejected.
157+
The requested frequency and bar type must agree with declared feed metadata. These declarations
158+
do not themselves validate OHLCV completeness or transform timestamps. Gap detection remains
159+
separate from the updater's optional elapsed-grid imputation described above.
160+
145161
## Inspect Stored State
146162

147163
List keys and inspect one dataset:

‎src/ml4t/data/core/models.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ def serialize_time_declaration(self, handler: Any) -> dict[str, Any]:
149149
data.pop(name, None)
150150
return data
151151

152+
@property
153+
def data_frequency(self) -> str | None:
154+
"""Project a physical frequency without the legacy unknown or event-count label."""
155+
return self.bar_params.get("frequency") if self.bar_type == "time" else None
156+
152157
@property
153158
def frequency(self) -> str:
154159
"""Get frequency string for backward compatibility.

‎src/ml4t/data/data_manager.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,9 @@ def load(
339339
bar_type: str = "time",
340340
bar_threshold: int | None = None,
341341
exchange: str = "UNKNOWN",
342-
calendar: str | None = None,
342+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
343+
*,
344+
feed_spec: Any | None = None,
343345
) -> str:
344346
"""Load data from provider and store it."""
345347
if not self._storage_manager:
@@ -357,6 +359,7 @@ def load(
357359
bar_threshold,
358360
exchange,
359361
calendar,
362+
feed_spec=feed_spec,
360363
)
361364

362365
def import_data(
@@ -369,7 +372,9 @@ def import_data(
369372
bar_type: str = "time",
370373
bar_threshold: int | None = None,
371374
exchange: str = "UNKNOWN",
372-
calendar: str | None = None,
375+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
376+
*,
377+
feed_spec: Any | None = None,
373378
) -> str:
374379
"""Import external data into storage with metadata."""
375380
if not self._storage_manager:
@@ -386,6 +391,7 @@ def import_data(
386391
bar_threshold,
387392
exchange,
388393
calendar,
394+
feed_spec=feed_spec,
389395
)
390396

391397
def update(
@@ -399,6 +405,9 @@ def update(
399405
initial_start: str | None = None,
400406
initial_end: str | None = None,
401407
initial_load_days: int = 365,
408+
*,
409+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
410+
feed_spec: Any | None = None,
402411
) -> str:
403412
"""Update existing data with incremental fetch."""
404413
if not self._storage_manager:
@@ -415,6 +424,8 @@ def update(
415424
initial_start,
416425
initial_end,
417426
initial_load_days,
427+
calendar=calendar,
428+
feed_spec=feed_spec,
418429
)
419430

420431
# ========================================================================

‎src/ml4t/data/managers/storage_manager.py‎

Lines changed: 85 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,27 @@
1919
from tenacity import RetryError
2020

2121
from ml4t.data.core.schemas import align_frames_for_concat, timestamp_bounds
22+
from ml4t.data.sessions.assigner import SessionAssigner
2223
from ml4t.data.storage.backend import normalize_storage_metadata
2324
from ml4t.data.utils.conversion import pandas_to_polars
25+
from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec
2426

2527
if TYPE_CHECKING:
2628
from ml4t.data.managers.fetch_manager import FetchManager
2729

2830
logger = structlog.get_logger()
2931

32+
_TEMPORAL_FIELDS = (
33+
"calendar",
34+
"time_contract_version",
35+
"timezone",
36+
"timestamp_semantics",
37+
"session_start_time",
38+
"bar_grid_anchor",
39+
"bar_label",
40+
"partial_bar_policy",
41+
)
42+
3043

3144
class StorageManager:
3245
"""Manages data storage operations.
@@ -181,6 +194,39 @@ def _ensure_polars_df(self, df: Any) -> pl.DataFrame:
181194
return pandas_to_polars(df)
182195
return df
183196

197+
@staticmethod
198+
def _ingestion_feed(
199+
calendar: str | None | CalendarUnset,
200+
feed_spec: Any | None,
201+
frequency: str,
202+
bar_type: str,
203+
) -> tuple[FeedSpec, dict[str, Any]]:
204+
source = normalize_storage_metadata(feed_spec) if isinstance(feed_spec, dict) else feed_spec
205+
feed = (
206+
FeedSpec.from_any(source) if source is not None else FeedSpec(time_contract_version=2)
207+
)
208+
aliases = {
209+
"hour": "hourly",
210+
"1h": "hourly",
211+
"1hour": "hourly",
212+
"1d": "daily",
213+
"1day": "daily",
214+
"1m": "minute",
215+
"1min": "minute",
216+
"1minute": "minute",
217+
}
218+
requested = aliases.get(frequency.lower(), frequency.lower())
219+
declared = str(feed.data_frequency).lower()
220+
if feed.data_frequency is not None and aliases.get(declared, declared) != requested:
221+
raise ValueError("feed data_frequency must match the requested frequency")
222+
if feed.bar_type not in (None, bar_type):
223+
raise ValueError("feed bar_type must match the requested bar_type")
224+
if calendar is not UNSET_CALENDAR:
225+
feed = feed.with_overrides(calendar=calendar, time_contract_version=2)
226+
SessionAssigner(feed_spec=feed)
227+
wire = feed.to_dict()
228+
return feed, {name: wire[name] for name in _TEMPORAL_FIELDS if name in wire}
229+
184230
def load(
185231
self,
186232
symbol: str,
@@ -192,7 +238,9 @@ def load(
192238
bar_type: str = "time",
193239
bar_threshold: int | None = None,
194240
exchange: str = "UNKNOWN",
195-
calendar: str | None = None,
241+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
242+
*,
243+
feed_spec: Any | None = None,
196244
) -> str:
197245
"""Load data from provider and store it.
198246
@@ -217,6 +265,8 @@ def load(
217265
if not self.storage:
218266
raise ValueError("Storage not configured")
219267

268+
_, temporal_values = self._ingestion_feed(calendar, feed_spec, frequency, bar_type)
269+
220270
logger.info(
221271
"Starting data load",
222272
symbol=symbol,
@@ -270,7 +320,7 @@ def load(
270320
bar_type=bar_type,
271321
bar_params=bar_params,
272322
exchange=exchange,
273-
calendar=calendar,
323+
**temporal_values,
274324
start_date=min_ts,
275325
end_date=max_ts,
276326
last_updated=datetime.now(UTC),
@@ -317,7 +367,9 @@ def import_data(
317367
bar_type: str = "time",
318368
bar_threshold: int | None = None,
319369
exchange: str = "UNKNOWN",
320-
calendar: str | None = None,
370+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
371+
*,
372+
feed_spec: Any | None = None,
321373
) -> str:
322374
"""Import external data into storage with metadata.
323375
@@ -341,6 +393,8 @@ def import_data(
341393
if not self.storage:
342394
raise ValueError("Storage not configured")
343395

396+
_, temporal_values = self._ingestion_feed(calendar, feed_spec, frequency, bar_type)
397+
344398
logger.info(
345399
"Importing external data",
346400
symbol=symbol,
@@ -388,7 +442,7 @@ def import_data(
388442
bar_type=bar_type,
389443
bar_params=bar_params,
390444
exchange=exchange,
391-
calendar=calendar,
445+
**temporal_values,
392446
start_date=min_ts,
393447
end_date=max_ts,
394448
last_updated=datetime.now(UTC),
@@ -423,6 +477,9 @@ def update(
423477
initial_start: str | None = None,
424478
initial_end: str | None = None,
425479
initial_load_days: int = 365,
480+
*,
481+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
482+
feed_spec: Any | None = None,
426483
) -> str:
427484
"""Update existing data with incremental fetch.
428485
@@ -483,6 +540,8 @@ def update(
483540
frequency=frequency,
484541
asset_class=asset_class,
485542
provider=provider,
543+
calendar=calendar,
544+
feed_spec=feed_spec,
486545
)
487546

488547
# Read existing data
@@ -498,6 +557,14 @@ def update(
498557
existing_metadata = (
499558
normalize_storage_metadata(self.storage.get_metadata(key), key) or {}
500559
)
560+
stored_bar_type = existing_metadata.get("bar_type")
561+
effective_bar_type = stored_bar_type if isinstance(stored_bar_type, str) else "time"
562+
effective_feed, temporal_values = self._ingestion_feed(
563+
calendar,
564+
existing_metadata if feed_spec is None else feed_spec,
565+
frequency,
566+
effective_bar_type,
567+
)
501568
existing_provider = existing_metadata.get("provider")
502569
if (
503570
not provider
@@ -578,9 +645,16 @@ def update(
578645

579646
from ml4t.data.utils.gaps import GapDetector
580647

581-
is_crypto = asset_class.lower() == "crypto"
648+
selected_calendar = effective_feed.resolve_calendar()
649+
if selected_calendar is not None and effective_bar_type != "time":
650+
raise NotImplementedError("calendar gap filling requires fixed time bars")
582651
gap_detector = GapDetector()
583-
gaps = gap_detector.detect_gaps(merged_df, frequency=frequency, is_crypto=is_crypto)
652+
gaps = gap_detector.detect_gaps(
653+
merged_df,
654+
frequency=frequency,
655+
calendar=selected_calendar,
656+
feed_spec=effective_feed,
657+
)
584658

585659
if gaps:
586660
gap_summary = gap_detector.summarize_gaps(gaps)
@@ -658,6 +732,10 @@ def update(
658732
"attributes": updated_attributes,
659733
}
660734

735+
for name in _TEMPORAL_FIELDS:
736+
metadata_values.pop(name, None)
737+
metadata_values.update(temporal_values)
738+
661739
try:
662740
updated_metadata = Metadata.model_validate(metadata_values)
663741
except ValidationError as error:
@@ -670,13 +748,7 @@ def update(
670748
"provider",
671749
"symbol",
672750
"asset_class",
673-
"calendar",
674-
"time_contract_version",
675-
"timezone",
676-
"timestamp_semantics",
677-
"bar_grid_anchor",
678-
"bar_label",
679-
"partial_bar_policy",
751+
*_TEMPORAL_FIELDS,
680752
}
681753
if not removable_fields:
682754
raise

‎tests/test_data_manager_storage.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from ml4t.data.storage.flat import FlatStorage
1616
from ml4t.data.storage.hive import HiveStorage
1717
from ml4t.data.storage.keys import storage_key_path
18+
from ml4t.specs import UNSET_CALENDAR
1819

1920

2021
class TestDataManagerLoad:
@@ -258,6 +259,8 @@ def test_update_uses_configured_initial_load_dates(self, manager):
258259
frequency="daily",
259260
asset_class="equities",
260261
provider="mock",
262+
calendar=UNSET_CALENDAR,
263+
feed_spec=None,
261264
)
262265

263266
def test_update_limits_default_initial_load_to_provider_history(self, manager):

0 commit comments

Comments
 (0)