1919from tenacity import RetryError
2020
2121from ml4t .data .core .schemas import align_frames_for_concat , timestamp_bounds
22+ from ml4t .data .sessions .assigner import SessionAssigner
2223from ml4t .data .storage .backend import normalize_storage_metadata
2324from ml4t .data .utils .conversion import pandas_to_polars
25+ from ml4t .specs import UNSET_CALENDAR , CalendarUnset , FeedSpec
2426
2527if TYPE_CHECKING :
2628 from ml4t .data .managers .fetch_manager import FetchManager
2729
2830logger = structlog .get_logger ()
2931
32+ _TEMPORAL_FIELDS = (
33+ "calendar" ,
34+ "time_contract_version" ,
35+ "timezone" ,
36+ "timestamp_semantics" ,
37+ "session_start_time" ,
38+ "bar_grid_anchor" ,
39+ "bar_label" ,
40+ "partial_bar_policy" ,
41+ )
42+
3043
3144class StorageManager :
3245 """Manages data storage operations.
@@ -181,6 +194,39 @@ def _ensure_polars_df(self, df: Any) -> pl.DataFrame:
181194 return pandas_to_polars (df )
182195 return df
183196
197+ @staticmethod
198+ def _ingestion_feed (
199+ calendar : str | None | CalendarUnset ,
200+ feed_spec : Any | None ,
201+ frequency : str ,
202+ bar_type : str ,
203+ ) -> tuple [FeedSpec , dict [str , Any ]]:
204+ source = normalize_storage_metadata (feed_spec ) if isinstance (feed_spec , dict ) else feed_spec
205+ feed = (
206+ FeedSpec .from_any (source ) if source is not None else FeedSpec (time_contract_version = 2 )
207+ )
208+ aliases = {
209+ "hour" : "hourly" ,
210+ "1h" : "hourly" ,
211+ "1hour" : "hourly" ,
212+ "1d" : "daily" ,
213+ "1day" : "daily" ,
214+ "1m" : "minute" ,
215+ "1min" : "minute" ,
216+ "1minute" : "minute" ,
217+ }
218+ requested = aliases .get (frequency .lower (), frequency .lower ())
219+ declared = str (feed .data_frequency ).lower ()
220+ if feed .data_frequency is not None and aliases .get (declared , declared ) != requested :
221+ raise ValueError ("feed data_frequency must match the requested frequency" )
222+ if feed .bar_type not in (None , bar_type ):
223+ raise ValueError ("feed bar_type must match the requested bar_type" )
224+ if calendar is not UNSET_CALENDAR :
225+ feed = feed .with_overrides (calendar = calendar , time_contract_version = 2 )
226+ SessionAssigner (feed_spec = feed )
227+ wire = feed .to_dict ()
228+ return feed , {name : wire [name ] for name in _TEMPORAL_FIELDS if name in wire }
229+
184230 def load (
185231 self ,
186232 symbol : str ,
@@ -192,7 +238,9 @@ def load(
192238 bar_type : str = "time" ,
193239 bar_threshold : int | None = None ,
194240 exchange : str = "UNKNOWN" ,
195- calendar : str | None = None ,
241+ calendar : str | None | CalendarUnset = UNSET_CALENDAR ,
242+ * ,
243+ feed_spec : Any | None = None ,
196244 ) -> str :
197245 """Load data from provider and store it.
198246
@@ -217,6 +265,8 @@ def load(
217265 if not self .storage :
218266 raise ValueError ("Storage not configured" )
219267
268+ _ , temporal_values = self ._ingestion_feed (calendar , feed_spec , frequency , bar_type )
269+
220270 logger .info (
221271 "Starting data load" ,
222272 symbol = symbol ,
@@ -270,7 +320,7 @@ def load(
270320 bar_type = bar_type ,
271321 bar_params = bar_params ,
272322 exchange = exchange ,
273- calendar = calendar ,
323+ ** temporal_values ,
274324 start_date = min_ts ,
275325 end_date = max_ts ,
276326 last_updated = datetime .now (UTC ),
@@ -317,7 +367,9 @@ def import_data(
317367 bar_type : str = "time" ,
318368 bar_threshold : int | None = None ,
319369 exchange : str = "UNKNOWN" ,
320- calendar : str | None = None ,
370+ calendar : str | None | CalendarUnset = UNSET_CALENDAR ,
371+ * ,
372+ feed_spec : Any | None = None ,
321373 ) -> str :
322374 """Import external data into storage with metadata.
323375
@@ -341,6 +393,8 @@ def import_data(
341393 if not self .storage :
342394 raise ValueError ("Storage not configured" )
343395
396+ _ , temporal_values = self ._ingestion_feed (calendar , feed_spec , frequency , bar_type )
397+
344398 logger .info (
345399 "Importing external data" ,
346400 symbol = symbol ,
@@ -388,7 +442,7 @@ def import_data(
388442 bar_type = bar_type ,
389443 bar_params = bar_params ,
390444 exchange = exchange ,
391- calendar = calendar ,
445+ ** temporal_values ,
392446 start_date = min_ts ,
393447 end_date = max_ts ,
394448 last_updated = datetime .now (UTC ),
@@ -423,6 +477,9 @@ def update(
423477 initial_start : str | None = None ,
424478 initial_end : str | None = None ,
425479 initial_load_days : int = 365 ,
480+ * ,
481+ calendar : str | None | CalendarUnset = UNSET_CALENDAR ,
482+ feed_spec : Any | None = None ,
426483 ) -> str :
427484 """Update existing data with incremental fetch.
428485
@@ -483,6 +540,8 @@ def update(
483540 frequency = frequency ,
484541 asset_class = asset_class ,
485542 provider = provider ,
543+ calendar = calendar ,
544+ feed_spec = feed_spec ,
486545 )
487546
488547 # Read existing data
@@ -498,6 +557,14 @@ def update(
498557 existing_metadata = (
499558 normalize_storage_metadata (self .storage .get_metadata (key ), key ) or {}
500559 )
560+ stored_bar_type = existing_metadata .get ("bar_type" )
561+ effective_bar_type = stored_bar_type if isinstance (stored_bar_type , str ) else "time"
562+ effective_feed , temporal_values = self ._ingestion_feed (
563+ calendar ,
564+ existing_metadata if feed_spec is None else feed_spec ,
565+ frequency ,
566+ effective_bar_type ,
567+ )
501568 existing_provider = existing_metadata .get ("provider" )
502569 if (
503570 not provider
@@ -578,9 +645,16 @@ def update(
578645
579646 from ml4t .data .utils .gaps import GapDetector
580647
581- is_crypto = asset_class .lower () == "crypto"
648+ selected_calendar = effective_feed .resolve_calendar ()
649+ if selected_calendar is not None and effective_bar_type != "time" :
650+ raise NotImplementedError ("calendar gap filling requires fixed time bars" )
582651 gap_detector = GapDetector ()
583- gaps = gap_detector .detect_gaps (merged_df , frequency = frequency , is_crypto = is_crypto )
652+ gaps = gap_detector .detect_gaps (
653+ merged_df ,
654+ frequency = frequency ,
655+ calendar = selected_calendar ,
656+ feed_spec = effective_feed ,
657+ )
584658
585659 if gaps :
586660 gap_summary = gap_detector .summarize_gaps (gaps )
@@ -658,6 +732,10 @@ def update(
658732 "attributes" : updated_attributes ,
659733 }
660734
735+ for name in _TEMPORAL_FIELDS :
736+ metadata_values .pop (name , None )
737+ metadata_values .update (temporal_values )
738+
661739 try :
662740 updated_metadata = Metadata .model_validate (metadata_values )
663741 except ValidationError as error :
@@ -670,13 +748,7 @@ def update(
670748 "provider" ,
671749 "symbol" ,
672750 "asset_class" ,
673- "calendar" ,
674- "time_contract_version" ,
675- "timezone" ,
676- "timestamp_semantics" ,
677- "bar_grid_anchor" ,
678- "bar_label" ,
679- "partial_bar_policy" ,
751+ * _TEMPORAL_FIELDS ,
680752 }
681753 if not removable_fields :
682754 raise
0 commit comments