22
33from __future__ import annotations
44
5+ import re
56from bisect import bisect_left , bisect_right
67from dataclasses import dataclass
78from datetime import UTC , date , datetime , timedelta
1011import polars as pl
1112import structlog
1213
13- from ml4t .data .sessions .assigner import SessionAssigner , _daily_session_labels
14- from ml4t .specs import UNSET_CALENDAR , CalendarUnset , FeedSpec
14+ from ml4t .data .sessions .assigner import (
15+ SessionAssigner ,
16+ _daily_session_labels ,
17+ _session_intervals ,
18+ )
19+ from ml4t .specs import (
20+ UNSET_CALENDAR ,
21+ BarLabel ,
22+ CalendarUnset ,
23+ FeedSpec ,
24+ PartialBarPolicy ,
25+ TimestampSemantics ,
26+ )
1527
1628logger = structlog .get_logger ()
1729
@@ -70,7 +82,7 @@ def __init__(self, tolerance: float = DEFAULT_TOLERANCE) -> None:
7082 def detect_gaps (
7183 self ,
7284 df : pl .DataFrame ,
73- frequency : str = "daily" ,
85+ frequency : str | None = None ,
7486 timestamp_col : str = "timestamp" ,
7587 is_crypto : bool = False ,
7688 * ,
@@ -82,16 +94,18 @@ def detect_gaps(
8294
8395 Args:
8496 df: DataFrame with time series data
85- frequency: Data frequency (minute, hourly, daily, etc.)
97+ frequency: Data frequency; inherit the feed declaration, otherwise daily.
8698 timestamp_col: Name of timestamp column
8799 is_crypto: If True, expect 24/7 data; if False, market hours only
88- calendar: Daily schedule override; None disables schedule-based detection.
100+ calendar: Schedule override; None disables schedule-based detection.
89101 feed_spec: Shared feed declaration, resolved before the NYSE default.
90102
91103 Returns:
92104 List of detected gaps
93105 """
94106 feed = FeedSpec .from_any (feed_spec )
107+ if frequency is None :
108+ frequency = str (feed .data_frequency ) if feed .data_frequency is not None else "daily"
95109 # Preserve the old crypto hint only when the feed has no declaration.
96110 if (
97111 frequency .lower () in {"daily" , "1day" }
@@ -106,8 +120,20 @@ def detect_gaps(
106120 selected_calendar = feed .resolve_calendar (calendar = calendar )
107121 if selected_calendar is None :
108122 return []
123+ declared_interval = (
124+ self ._intraday_interval (str (feed .data_frequency ))
125+ if feed .data_frequency is not None
126+ else None
127+ )
128+ if (
129+ declared_interval is not None
130+ and self ._intraday_interval (frequency ) != declared_interval
131+ ):
132+ raise ValueError ("declared grid data_frequency must match the requested frequency" )
109133 if frequency .lower () in {"daily" , "1day" }:
110134 return self ._detect_daily_gaps (df , timestamp_col , frequency , selected_calendar , feed )
135+ if self ._intraday_interval (frequency ) is not None :
136+ return self ._detect_intraday_gaps (df , timestamp_col , frequency , selected_calendar , feed )
111137 if calendar is not UNSET_CALENDAR or feed_spec is not None :
112138 raise NotImplementedError (
113139 "calendar/feed declarations currently support daily gaps only"
@@ -139,21 +165,6 @@ def detect_gaps(
139165 actual_seconds = actual_interval .total_seconds ()
140166 expected_seconds = expected_interval .total_seconds ()
141167
142- # Check if this is a gap (considering market hours for non-crypto)
143- if (
144- not is_crypto
145- and frequency
146- in [
147- "minute" ,
148- "5minute" ,
149- "15minute" ,
150- "30minute" ,
151- "hourly" ,
152- ]
153- and self ._spans_non_trading_hours (current , next_ts , frequency )
154- ):
155- continue
156-
157168 # Detect gap if interval exceeds expected + tolerance
158169 if actual_seconds > expected_seconds + tolerance_seconds :
159170 missing_periods = int (actual_seconds / expected_seconds ) - 1
@@ -177,6 +188,124 @@ def detect_gaps(
177188
178189 return gaps
179190
191+ @staticmethod
192+ def _intraday_interval (frequency : str ) -> timedelta | None :
193+ value = frequency .lower ().strip ().replace ("hourly" , "hour" )
194+ match = re .fullmatch (r"(\d+)?(minutes?|min|m|hours?|h)" , value )
195+ if match is None :
196+ return None
197+ count = int (match [1 ] or 1 )
198+ if count <= 0 :
199+ raise ValueError ("bar frequency must be positive" )
200+ return timedelta (hours = count ) if match [2 ].startswith ("h" ) else timedelta (minutes = count )
201+
202+ def _detect_intraday_gaps (
203+ self ,
204+ df : pl .DataFrame ,
205+ timestamp_col : str ,
206+ frequency : str ,
207+ calendar : str ,
208+ feed : FeedSpec ,
209+ ) -> list [DataGap ]:
210+ interval = self ._intraday_interval (frequency )
211+ if interval is None :
212+ raise ValueError ("unsupported intraday bar frequency" )
213+ required = (
214+ "data_frequency" ,
215+ "timestamp_semantics" ,
216+ "bar_grid_anchor" ,
217+ "bar_label" ,
218+ "partial_bar_policy" ,
219+ )
220+ if feed .time_contract_version != 2 or any (getattr (feed , name ) is None for name in required ):
221+ raise ValueError ("intraday detection requires a complete declared version-2 bar grid" )
222+ if feed .timestamp_semantics == TimestampSemantics .SESSION_LABEL :
223+ raise ValueError ("intraday detection requires a declared event/close bar grid" )
224+ if feed .bar_type not in (None , "time" ):
225+ raise ValueError ("intraday grid detection requires fixed time bars" )
226+ if interval != self ._intraday_interval (str (feed .data_frequency )):
227+ raise ValueError ("declared grid data_frequency must match the requested frequency" )
228+ assigner = SessionAssigner (calendar , feed_spec = feed )
229+ if timestamp_col not in df .columns :
230+ raise ValueError (f"DataFrame must have '{ timestamp_col } ' column" )
231+ dtype = df .schema [timestamp_col ]
232+ if not isinstance (dtype , pl .Datetime ):
233+ raise TypeError (f"'{ timestamp_col } ' must contain Datetime values for an intraday grid" )
234+ if df [timestamp_col ].null_count ():
235+ raise ValueError (f"'{ timestamp_col } ' must not contain null timestamps" )
236+ if (
237+ dtype .time_unit == "ns"
238+ and df .select ((pl .col (timestamp_col ).dt .nanosecond () % 1000 != 0 ).any ()).item ()
239+ ):
240+ raise ValueError ("intraday observations contain off-grid sub-microsecond timestamps" )
241+ if dtype .time_zone is None and feed .timezone is None :
242+ raise ValueError ("naive intraday timestamps require a declared timezone" )
243+ if df .is_empty ():
244+ return []
245+ expr = pl .col (timestamp_col )
246+ if dtype .time_zone is None :
247+ expr = expr .dt .replace_time_zone (feed .timezone )
248+ observed = df .select (expr .dt .convert_time_zone ("UTC" )).to_series ().unique ().sort ().to_list ()
249+ start , end = observed [0 ], observed [- 1 ]
250+ schedule = assigner .calendar .schedule (
251+ start_date = (start - timedelta (days = 2 )).date (),
252+ end_date = (end + timedelta (days = 2 )).date (),
253+ interruptions = True ,
254+ )
255+ grid_intervals : list [tuple [datetime , datetime , int , int ]] = []
256+ total_bars = 0
257+ for _ , row in schedule .iterrows ():
258+ for opening , closing in _session_intervals (row ):
259+ opening , closing = opening .astimezone (UTC ), closing .astimezone (UTC )
260+ count , remainder = divmod (closing - opening , interval )
261+ if remainder and feed .partial_bar_policy == PartialBarPolicy .INCLUDE :
262+ count += 1
263+ if count <= 0 :
264+ continue
265+ if grid_intervals and opening < grid_intervals [- 1 ][1 ]:
266+ raise ValueError ("calendar returned overlapping tradable intervals" )
267+ grid_intervals .append ((opening , closing , count , total_bars ))
268+ total_bars += count
269+ right_labeled = feed .bar_label == BarLabel .RIGHT
270+ boundaries = [row [1 ] if right_labeled else row [0 ] for row in grid_intervals ]
271+
272+ def bar_index (timestamp : datetime ) -> int :
273+ index = (
274+ bisect_left (boundaries , timestamp )
275+ if right_labeled
276+ else bisect_right (boundaries , timestamp ) - 1
277+ )
278+ if index < 0 or index >= len (grid_intervals ):
279+ raise ValueError ("intraday observations contain off-grid timestamps" )
280+ opening , closing , count , offset = grid_intervals [index ]
281+ if right_labeled and timestamp == closing :
282+ if (closing - opening ) % interval == timedelta (0 ) or (
283+ feed .partial_bar_policy == PartialBarPolicy .INCLUDE
284+ ):
285+ return offset + count - 1
286+ raise ValueError ("intraday observations contain off-grid partial-close timestamps" )
287+ position , remainder = divmod (timestamp - opening , interval )
288+ if right_labeled :
289+ position -= 1
290+ if (
291+ remainder
292+ or timestamp < opening
293+ or timestamp >= closing
294+ or not 0 <= position < count
295+ ):
296+ raise ValueError ("intraday observations contain off-grid timestamps" )
297+ return offset + position
298+
299+ positions = [bar_index (timestamp ) for timestamp in observed ]
300+ gaps = []
301+ for current , following , left_index , right_index in zip (
302+ observed , observed [1 :], positions , positions [1 :]
303+ ):
304+ missing = right_index - left_index - 1
305+ if missing > 0 :
306+ gaps .append (DataGap (current , following , missing , following - current , frequency ))
307+ return gaps
308+
180309 def _detect_daily_gaps (
181310 self ,
182311 df : pl .DataFrame ,
@@ -224,22 +353,11 @@ def _get_expected_interval(self, frequency: str, is_crypto: bool = False) -> tim
224353 Expected timedelta between consecutive points
225354 """
226355 freq_lower = frequency .lower ()
356+ interval = self ._intraday_interval (frequency )
357+ if interval is not None :
358+ return interval
227359
228- # Handle minute-based frequencies
229- if "minute" in freq_lower :
230- if freq_lower == "minute" or freq_lower == "1minute" :
231- return timedelta (minutes = 1 )
232- if freq_lower == "5minute" :
233- return timedelta (minutes = 5 )
234- if freq_lower == "15minute" :
235- return timedelta (minutes = 15 )
236- if freq_lower == "30minute" :
237- return timedelta (minutes = 30 )
238-
239- # Handle other frequencies
240- elif freq_lower == "hourly" or freq_lower == "1hour" :
241- return timedelta (hours = 1 )
242- elif freq_lower == "daily" or freq_lower == "1day" :
360+ if freq_lower == "daily" or freq_lower == "1day" :
243361 if is_crypto :
244362 return timedelta (days = 1 )
245363 # For stocks, next trading day could be 1-3 days away
@@ -252,41 +370,6 @@ def _get_expected_interval(self, frequency: str, is_crypto: bool = False) -> tim
252370 # Default to daily
253371 return timedelta (days = 1 )
254372
255- def _spans_non_trading_hours (
256- self ,
257- start : datetime ,
258- end : datetime ,
259- frequency : str , # noqa: ARG002
260- ) -> bool :
261- """
262- Check if gap spans non-trading hours (nights/weekends).
263-
264- Args:
265- start: Start of gap
266- end: End of gap
267- frequency: Data frequency
268-
269- Returns:
270- True if gap is expected (non-trading hours)
271- """
272- # Convert to UTC if needed
273- if start .tzinfo is None :
274- start = start .replace (tzinfo = UTC )
275- if end .tzinfo is None :
276- end = end .replace (tzinfo = UTC )
277-
278- # Check for weekend
279- if start .weekday () == 4 and end .weekday () == 0 : # Friday to Monday
280- return True
281-
282- # Check for overnight gap (4pm to 9:30am ET)
283- # Convert to ET (UTC-5 or UTC-4 for DST)
284- market_close_hour = 21 # 4pm ET in UTC (during standard time)
285- market_open_hour = 14 # 9:30am ET in UTC (during standard time)
286-
287- # Simplified check: if gap starts after market close and ends at/after market open
288- return start .hour >= market_close_hour or end .hour < market_open_hour
289-
290373 def summarize_gaps (self , gaps : list [DataGap ]) -> dict [str , Any ]:
291374 """
292375 Summarize detected gaps.
0 commit comments