1313
1414from ml4t .data .core .schemas import align_frames_for_concat
1515from ml4t .data .providers .base import BaseProvider
16- from ml4t .data .storage .backend import StorageBackend
16+ from ml4t .data .sessions .assigner import SessionAssigner
17+ from ml4t .data .storage .backend import StorageBackend , normalize_storage_metadata
1718from ml4t .data .storage .metadata_tracker import MetadataTracker , UpdateRecord
19+ from ml4t .specs import UNSET_CALENDAR , CalendarUnset , FeedSpec , TimestampSemantics
1820
1921logger = structlog .get_logger ()
2022
@@ -28,6 +30,14 @@ def _ensure_datetime(value: Any) -> datetime:
2830 raise TypeError (f"Expected a date or datetime, got { type (value ).__name__ } " )
2931
3032
33+ def _session_label (value : Any ) -> datetime :
34+ """Encode a civil session date as UTC midnight without converting an instant."""
35+ if not isinstance (value , date ):
36+ raise TypeError ("'timestamp' must contain non-null Date or Datetime values" )
37+ label = value .date () if isinstance (value , datetime ) else value
38+ return datetime .combine (label , datetime .min .time (), UTC )
39+
40+
3141def _rewrite_preserving_metadata (
3242 storage : StorageBackend ,
3343 key : str ,
@@ -89,34 +99,147 @@ def __post_init__(self) -> None:
8999class GapDetector :
90100 """Detect gaps in time-series data."""
91101
92- def __init__ (self , exclude_weekends : bool = False ) -> None :
102+ def __init__ (
103+ self ,
104+ exclude_weekends : bool | None = None ,
105+ * ,
106+ calendar : str | None | CalendarUnset = UNSET_CALENDAR ,
107+ feed_spec : Any | None = None ,
108+ ) -> None :
93109 """
94110 Initialize gap detector.
95111
96112 Args:
97- exclude_weekends: Whether to exclude weekends from gap detection
113+ exclude_weekends: Legacy hint; explicit False selects 24/7 absent a declaration.
114+ calendar: Explicit schedule selection; None disables schedule classification.
115+ feed_spec: Shared feed declaration, resolved before the NYSE default.
98116 """
99117 self .exclude_weekends = exclude_weekends
118+ self .calendar = calendar
119+ self .feed_spec = feed_spec
120+
121+ def _feed_calendar (
122+ self , calendar : str | None | CalendarUnset , feed_spec : Any | None
123+ ) -> tuple [FeedSpec , str | None ]:
124+ feed = FeedSpec .from_any (self .feed_spec if feed_spec is None else feed_spec )
125+ if calendar is UNSET_CALENDAR :
126+ calendar = self .calendar
127+ if (
128+ calendar is UNSET_CALENDAR
129+ and self .exclude_weekends is False
130+ and (
131+ feed .calendar is UNSET_CALENDAR
132+ or (feed .time_contract_version == 1 and feed .calendar is None )
133+ )
134+ ):
135+ calendar = "24/7"
136+ return feed , feed .resolve_calendar (calendar = calendar )
137+
138+ @staticmethod
139+ def _label_frame (df : pl .DataFrame , feed : FeedSpec ) -> pl .DataFrame :
140+ dtype = df .schema ["timestamp" ]
141+ if dtype != pl .Date and not isinstance (dtype , pl .Datetime ):
142+ raise TypeError ("'timestamp' must contain Date or Datetime values" )
143+ if df ["timestamp" ].null_count ():
144+ raise ValueError ("'timestamp' must not contain null timestamps" )
145+ if isinstance (dtype , pl .Datetime ) and feed .timezone is not None :
146+ timestamp = pl .col ("timestamp" )
147+ if dtype .time_zone is None :
148+ timestamp = timestamp .dt .replace_time_zone (feed .timezone )
149+ else :
150+ timestamp = timestamp .dt .convert_time_zone (feed .timezone )
151+ return df .with_columns (timestamp )
152+ return df
153+
154+ @staticmethod
155+ def _daily_ranges (
156+ df : pl .DataFrame ,
157+ start : datetime ,
158+ end : datetime ,
159+ feed : FeedSpec ,
160+ calendar : str ,
161+ tolerance_days : int = 0 ,
162+ ) -> list [dict [str , Any ]]:
163+ if start .date () > end .date ():
164+ raise ValueError ("start_date must not be after end_date" )
165+ if feed .timestamp_semantics not in (None , TimestampSemantics .SESSION_LABEL ):
166+ raise NotImplementedError ("daily range planning requires session_label timestamps" )
167+ assigner = SessionAssigner (calendar )
168+ observed = {
169+ value .date () if isinstance (value , datetime ) else value
170+ for value in df ["timestamp" ].to_list ()
171+ }
172+ schedule = assigner .calendar .schedule (start_date = start .date (), end_date = end .date ())
173+ gaps = []
174+ missing : list [date ] = []
175+
176+ def append_gap () -> None :
177+ if len (missing ) > tolerance_days :
178+ gaps .append (
179+ {
180+ "start" : datetime .combine (missing [0 ], datetime .min .time (), UTC ),
181+ "end" : datetime .combine (missing [- 1 ], datetime .min .time (), UTC ),
182+ "size_days" : len (missing ),
183+ }
184+ )
185+
186+ for label in schedule .index :
187+ session = label .date ()
188+ if session not in observed :
189+ missing .append (session )
190+ elif missing :
191+ append_gap ()
192+ missing = []
193+ if missing :
194+ append_gap ()
195+ return gaps
100196
101197 def detect_gaps (
102198 self ,
103199 df : pl .DataFrame ,
104200 frequency : str = "daily" ,
105201 tolerance_days : int = 0 ,
202+ * ,
203+ calendar : str | None | CalendarUnset = UNSET_CALENDAR ,
204+ feed_spec : Any | None = None ,
106205 ) -> list [dict [str , Any ]]:
107206 """
108207 Detect gaps in a DataFrame's timestamp column.
109208
110209 Args:
111210 df: DataFrame with timestamp column
112211 frequency: Data frequency (daily, hourly, etc.)
113- tolerance_days: Number of days to tolerate as non-gap
212+ tolerance_days: Missing sessions tolerated for daily data; elapsed days otherwise.
213+ calendar: Per-call override of configured calendar and feed metadata.
214+ feed_spec: Per-call feed declaration.
114215
115216 Returns:
116217 List of gap dictionaries with start, end, and size
117218 """
219+ feed , selected = self ._feed_calendar (calendar , feed_spec )
220+ if selected is None :
221+ return []
118222 if df .is_empty () or "timestamp" not in df .columns :
119223 return []
224+ if frequency .lower () in {"daily" , "1day" }:
225+ df = self ._label_frame (df , feed )
226+ return self ._daily_ranges (
227+ df ,
228+ _session_label (df ["timestamp" ].min ()),
229+ _session_label (df ["timestamp" ].max ()),
230+ feed ,
231+ selected ,
232+ tolerance_days ,
233+ )
234+ if (
235+ calendar is not UNSET_CALENDAR
236+ or self .calendar is not UNSET_CALENDAR
237+ or feed_spec is not None
238+ or self .feed_spec is not None
239+ ):
240+ raise NotImplementedError (
241+ "calendar/feed declarations currently support daily ranges only"
242+ )
120243
121244 # Sort by timestamp
122245 df = df .sort ("timestamp" )
@@ -131,43 +254,16 @@ def detect_gaps(
131254 next_ts = timestamps [i + 1 ]
132255 expected = current + expected_delta
133256
134- # Check if there's a gap
135- if self .exclude_weekends and frequency == "daily" :
136- # Skip weekend days
137- while expected .weekday () >= 5 : # Saturday = 5, Sunday = 6
138- expected += timedelta (days = 1 )
139-
140257 if next_ts > expected + timedelta (days = tolerance_days ):
141- # Found a gap
142258 gap_start = current + expected_delta
143259 gap_end = next_ts - expected_delta
144-
145- # Adjust for weekends if needed
146- if self .exclude_weekends and frequency == "daily" :
147- # Skip weekend at start
148- while gap_start .weekday () >= 5 :
149- gap_start += timedelta (days = 1 )
150- # Skip weekend at end
151- while gap_end .weekday () >= 5 :
152- gap_end -= timedelta (days = 1 )
153-
154- # Only add gap if it's still valid after weekend adjustment
155- if gap_start <= gap_end :
156- gaps .append (
157- {
158- "start" : gap_start ,
159- "end" : gap_end ,
160- "size_days" : (gap_end - gap_start ).days + 1 ,
161- }
162- )
163- else :
164- gaps .append (
165- {
166- "start" : gap_start ,
167- "end" : gap_end ,
168- "size_days" : (gap_end - gap_start ).days + 1 ,
169- }
170- )
260+ gaps .append (
261+ {
262+ "start" : gap_start ,
263+ "end" : gap_end ,
264+ "size_days" : (gap_end - gap_start ).days + 1 ,
265+ }
266+ )
171267
172268 return gaps
173269
@@ -178,6 +274,9 @@ def detect_gaps_in_storage(
178274 start_date : datetime ,
179275 end_date : datetime ,
180276 frequency : str = "daily" ,
277+ * ,
278+ calendar : str | None | CalendarUnset = UNSET_CALENDAR ,
279+ feed_spec : Any | None = None ,
181280 ) -> list [dict [str , Any ]]:
182281 """
183282 Detect gaps through a storage backend.
@@ -192,8 +291,43 @@ def detect_gaps_in_storage(
192291 Returns:
193292 List of gaps found in storage, with UTC-aware bounds
194293 """
294+ if frequency .lower () in {"daily" , "1day" }:
295+ start_date = _session_label (start_date )
296+ end_date = _session_label (end_date )
297+ if feed_spec is None and self .feed_spec is None :
298+ try :
299+ feed_spec = normalize_storage_metadata (storage .get_metadata (key ), key )
300+ except KeyError :
301+ pass
302+ feed , selected = self ._feed_calendar (calendar , feed_spec )
303+ if selected is None :
304+ return []
305+ # Session labels retain their civil date even when stored with a timezone.
306+ start_bound = start_date - timedelta (days = 1 )
307+ end_bound = datetime .combine (end_date .date (), datetime .min .time (), UTC ) + timedelta (
308+ days = 2
309+ )
310+ try :
311+ df = storage .read (key , start_bound , end_bound ).collect ()
312+ except KeyError :
313+ df = pl .DataFrame (schema = {"timestamp" : pl .Datetime ("us" , "UTC" )})
314+ return self ._daily_ranges (
315+ self ._label_frame (df , feed ), start_date , end_date , feed , selected
316+ )
195317 start_date = _ensure_datetime (start_date )
196318 end_date = _ensure_datetime (end_date )
319+ _ , selected = self ._feed_calendar (calendar , feed_spec )
320+ if selected is None :
321+ return []
322+ if (
323+ calendar is not UNSET_CALENDAR
324+ or self .calendar is not UNSET_CALENDAR
325+ or feed_spec is not None
326+ or self .feed_spec is not None
327+ ):
328+ raise NotImplementedError (
329+ "calendar/feed declarations currently support daily ranges only"
330+ )
197331 try :
198332 # Storage end bounds are exclusive. Open the inclusive end by one
199333 # microsecond so a bar timestamped on end_date is part of the range.
@@ -529,14 +663,23 @@ def apply_strategy(
529663 existing_df , new_data = align_frames_for_concat (existing_df , new_data )
530664
531665 # Detect gaps
532- gaps = self .gap_detector .detect_gaps (existing_df )
666+ feed_metadata = normalize_storage_metadata (storage .get_metadata (key ), key )
667+ gaps = self .gap_detector .detect_gaps (existing_df , feed_spec = feed_metadata )
668+ feed , selected_calendar = self .gap_detector ._feed_calendar (
669+ UNSET_CALENDAR , feed_metadata
670+ )
533671 gaps_filled = 0
534672
535673 # Filter new data to only gap periods
536674 gap_data = pl .DataFrame ()
537675 for gap in gaps :
676+ expected = SessionAssigner (selected_calendar ).calendar .schedule (
677+ start_date = gap ["start" ].date (), end_date = gap ["end" ].date ()
678+ )
538679 gap_rows = new_data .filter (
539- (pl .col ("timestamp" ) >= gap ["start" ]) & (pl .col ("timestamp" ) <= gap ["end" ])
680+ self .gap_detector ._label_frame (new_data , feed )["timestamp" ]
681+ .dt .date ()
682+ .is_in ([label .date () for label in expected .index ])
540683 )
541684 if not gap_rows .is_empty ():
542685 gap_data = (
@@ -688,7 +831,8 @@ def identify_candidates(
688831 continue
689832
690833 # Detect gaps
691- gaps = self .gap_detector .detect_gaps (df )
834+ feed_metadata = normalize_storage_metadata (self .storage .get_metadata (key ), key )
835+ gaps = self .gap_detector .detect_gaps (df , feed_spec = feed_metadata )
692836
693837 # Filter significant gaps
694838 significant_gaps = [g for g in gaps if g ["size_days" ] >= min_gap_days ]
0 commit comments