Skip to content

Commit 7571793

Browse files
fix: plan and acquire daily backfills from declared sessions (#92)
Closes #91. Use authoritative expected sessions for internal and boundary backfill ranges, retain shared declarations and timezone labels, and acquire only scheduled provider observations.
1 parent 55b86e2 commit 7571793

5 files changed

Lines changed: 417 additions & 43 deletions

File tree

‎docs/user-guide/incremental-updates.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,31 @@ calls retain their existing behavior. Unknown calendars and malformed daily time
8888
modifying the input. `fill_gaps()` remains a separate elapsed-grid operation, not calendar-aware
8989
imputation.
9090

91+
### Plan Daily Backfill Acquisition
92+
93+
`ml4t.data.update_manager.GapDetector` is a separate acquisition-range planner. Its constructor and
94+
`detect_gaps()` / `detect_gaps_in_storage()` accept the canonical `calendar` and shared `feed_spec`
95+
declaration. Explicit selection overrides the declaration, then NYSE applies. Storage reads use the
96+
dataset's persisted declaration unless a feed is supplied. Explicit `None` disables schedule-based
97+
planning, including for absent datasets; 24/7 still expects weekend observations.
98+
99+
Daily plans include missing sessions at either boundary and within the inclusive requested date
100+
range. Closed dates never become missing sessions. Returned `start` and `end` are inclusive session
101+
labels encoded at UTC midnight, not event instants. The legacy `size_days` field and `tolerance_days`
102+
argument count expected sessions in daily mode, not elapsed civil days. A range can span closures
103+
between missing sessions. Backfill selects only scheduled labels from the provider's actual rows;
104+
it does not manufacture observations or insert closed-date bars.
105+
106+
The legacy `exclude_weekends=False` argument selects continuous days only when no canonical calendar
107+
or feed declaration is present. `exclude_weekends=True` now follows NYSE holidays as well as weekends.
108+
Omission follows NYSE, so callers requiring the old elapsed-day counts should pass `calendar="24/7"`.
109+
Intraday/weekly range planning retains its legacy elapsed-time behavior; named calendar/feed
110+
declarations for those frequencies are not supported by this planner yet. `calendar=None` disables
111+
classification. Session-label dates retain their existing timezone's civil date. Explicit event-time
112+
or bar-close declarations are rejected for daily planning rather than silently relabeled. Declare
113+
the feed's `timezone` when a non-UTC label must survive storage's canonical UTC reads; the planner
114+
restores that declared timezone before interpreting dates.
115+
91116
If a provider's history limit cannot reach the stored dataset, the updater records
92117
`provider_history_limited` and skips gap filling across the unavailable interval.
93118

‎src/ml4t/data/update_manager.py‎

Lines changed: 185 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313

1414
from ml4t.data.core.schemas import align_frames_for_concat
1515
from ml4t.data.providers.base import BaseProvider
16-
from ml4t.data.storage.backend import StorageBackend
16+
from ml4t.data.sessions.assigner import SessionAssigner
17+
from ml4t.data.storage.backend import StorageBackend, normalize_storage_metadata
1718
from ml4t.data.storage.metadata_tracker import MetadataTracker, UpdateRecord
19+
from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec, TimestampSemantics
1820

1921
logger = structlog.get_logger()
2022

@@ -28,6 +30,14 @@ def _ensure_datetime(value: Any) -> datetime:
2830
raise TypeError(f"Expected a date or datetime, got {type(value).__name__}")
2931

3032

33+
def _session_label(value: Any) -> datetime:
34+
"""Encode a civil session date as UTC midnight without converting an instant."""
35+
if not isinstance(value, date):
36+
raise TypeError("'timestamp' must contain non-null Date or Datetime values")
37+
label = value.date() if isinstance(value, datetime) else value
38+
return datetime.combine(label, datetime.min.time(), UTC)
39+
40+
3141
def _rewrite_preserving_metadata(
3242
storage: StorageBackend,
3343
key: str,
@@ -89,34 +99,147 @@ def __post_init__(self) -> None:
8999
class GapDetector:
90100
"""Detect gaps in time-series data."""
91101

92-
def __init__(self, exclude_weekends: bool = False) -> None:
102+
def __init__(
103+
self,
104+
exclude_weekends: bool | None = None,
105+
*,
106+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
107+
feed_spec: Any | None = None,
108+
) -> None:
93109
"""
94110
Initialize gap detector.
95111
96112
Args:
97-
exclude_weekends: Whether to exclude weekends from gap detection
113+
exclude_weekends: Legacy hint; explicit False selects 24/7 absent a declaration.
114+
calendar: Explicit schedule selection; None disables schedule classification.
115+
feed_spec: Shared feed declaration, resolved before the NYSE default.
98116
"""
99117
self.exclude_weekends = exclude_weekends
118+
self.calendar = calendar
119+
self.feed_spec = feed_spec
120+
121+
def _feed_calendar(
122+
self, calendar: str | None | CalendarUnset, feed_spec: Any | None
123+
) -> tuple[FeedSpec, str | None]:
124+
feed = FeedSpec.from_any(self.feed_spec if feed_spec is None else feed_spec)
125+
if calendar is UNSET_CALENDAR:
126+
calendar = self.calendar
127+
if (
128+
calendar is UNSET_CALENDAR
129+
and self.exclude_weekends is False
130+
and (
131+
feed.calendar is UNSET_CALENDAR
132+
or (feed.time_contract_version == 1 and feed.calendar is None)
133+
)
134+
):
135+
calendar = "24/7"
136+
return feed, feed.resolve_calendar(calendar=calendar)
137+
138+
@staticmethod
139+
def _label_frame(df: pl.DataFrame, feed: FeedSpec) -> pl.DataFrame:
140+
dtype = df.schema["timestamp"]
141+
if dtype != pl.Date and not isinstance(dtype, pl.Datetime):
142+
raise TypeError("'timestamp' must contain Date or Datetime values")
143+
if df["timestamp"].null_count():
144+
raise ValueError("'timestamp' must not contain null timestamps")
145+
if isinstance(dtype, pl.Datetime) and feed.timezone is not None:
146+
timestamp = pl.col("timestamp")
147+
if dtype.time_zone is None:
148+
timestamp = timestamp.dt.replace_time_zone(feed.timezone)
149+
else:
150+
timestamp = timestamp.dt.convert_time_zone(feed.timezone)
151+
return df.with_columns(timestamp)
152+
return df
153+
154+
@staticmethod
155+
def _daily_ranges(
156+
df: pl.DataFrame,
157+
start: datetime,
158+
end: datetime,
159+
feed: FeedSpec,
160+
calendar: str,
161+
tolerance_days: int = 0,
162+
) -> list[dict[str, Any]]:
163+
if start.date() > end.date():
164+
raise ValueError("start_date must not be after end_date")
165+
if feed.timestamp_semantics not in (None, TimestampSemantics.SESSION_LABEL):
166+
raise NotImplementedError("daily range planning requires session_label timestamps")
167+
assigner = SessionAssigner(calendar)
168+
observed = {
169+
value.date() if isinstance(value, datetime) else value
170+
for value in df["timestamp"].to_list()
171+
}
172+
schedule = assigner.calendar.schedule(start_date=start.date(), end_date=end.date())
173+
gaps = []
174+
missing: list[date] = []
175+
176+
def append_gap() -> None:
177+
if len(missing) > tolerance_days:
178+
gaps.append(
179+
{
180+
"start": datetime.combine(missing[0], datetime.min.time(), UTC),
181+
"end": datetime.combine(missing[-1], datetime.min.time(), UTC),
182+
"size_days": len(missing),
183+
}
184+
)
185+
186+
for label in schedule.index:
187+
session = label.date()
188+
if session not in observed:
189+
missing.append(session)
190+
elif missing:
191+
append_gap()
192+
missing = []
193+
if missing:
194+
append_gap()
195+
return gaps
100196

101197
def detect_gaps(
102198
self,
103199
df: pl.DataFrame,
104200
frequency: str = "daily",
105201
tolerance_days: int = 0,
202+
*,
203+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
204+
feed_spec: Any | None = None,
106205
) -> list[dict[str, Any]]:
107206
"""
108207
Detect gaps in a DataFrame's timestamp column.
109208
110209
Args:
111210
df: DataFrame with timestamp column
112211
frequency: Data frequency (daily, hourly, etc.)
113-
tolerance_days: Number of days to tolerate as non-gap
212+
tolerance_days: Missing sessions tolerated for daily data; elapsed days otherwise.
213+
calendar: Per-call override of configured calendar and feed metadata.
214+
feed_spec: Per-call feed declaration.
114215
115216
Returns:
116217
List of gap dictionaries with start, end, and size
117218
"""
219+
feed, selected = self._feed_calendar(calendar, feed_spec)
220+
if selected is None:
221+
return []
118222
if df.is_empty() or "timestamp" not in df.columns:
119223
return []
224+
if frequency.lower() in {"daily", "1day"}:
225+
df = self._label_frame(df, feed)
226+
return self._daily_ranges(
227+
df,
228+
_session_label(df["timestamp"].min()),
229+
_session_label(df["timestamp"].max()),
230+
feed,
231+
selected,
232+
tolerance_days,
233+
)
234+
if (
235+
calendar is not UNSET_CALENDAR
236+
or self.calendar is not UNSET_CALENDAR
237+
or feed_spec is not None
238+
or self.feed_spec is not None
239+
):
240+
raise NotImplementedError(
241+
"calendar/feed declarations currently support daily ranges only"
242+
)
120243

121244
# Sort by timestamp
122245
df = df.sort("timestamp")
@@ -131,43 +254,16 @@ def detect_gaps(
131254
next_ts = timestamps[i + 1]
132255
expected = current + expected_delta
133256

134-
# Check if there's a gap
135-
if self.exclude_weekends and frequency == "daily":
136-
# Skip weekend days
137-
while expected.weekday() >= 5: # Saturday = 5, Sunday = 6
138-
expected += timedelta(days=1)
139-
140257
if next_ts > expected + timedelta(days=tolerance_days):
141-
# Found a gap
142258
gap_start = current + expected_delta
143259
gap_end = next_ts - expected_delta
144-
145-
# Adjust for weekends if needed
146-
if self.exclude_weekends and frequency == "daily":
147-
# Skip weekend at start
148-
while gap_start.weekday() >= 5:
149-
gap_start += timedelta(days=1)
150-
# Skip weekend at end
151-
while gap_end.weekday() >= 5:
152-
gap_end -= timedelta(days=1)
153-
154-
# Only add gap if it's still valid after weekend adjustment
155-
if gap_start <= gap_end:
156-
gaps.append(
157-
{
158-
"start": gap_start,
159-
"end": gap_end,
160-
"size_days": (gap_end - gap_start).days + 1,
161-
}
162-
)
163-
else:
164-
gaps.append(
165-
{
166-
"start": gap_start,
167-
"end": gap_end,
168-
"size_days": (gap_end - gap_start).days + 1,
169-
}
170-
)
260+
gaps.append(
261+
{
262+
"start": gap_start,
263+
"end": gap_end,
264+
"size_days": (gap_end - gap_start).days + 1,
265+
}
266+
)
171267

172268
return gaps
173269

@@ -178,6 +274,9 @@ def detect_gaps_in_storage(
178274
start_date: datetime,
179275
end_date: datetime,
180276
frequency: str = "daily",
277+
*,
278+
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
279+
feed_spec: Any | None = None,
181280
) -> list[dict[str, Any]]:
182281
"""
183282
Detect gaps through a storage backend.
@@ -192,8 +291,43 @@ def detect_gaps_in_storage(
192291
Returns:
193292
List of gaps found in storage, with UTC-aware bounds
194293
"""
294+
if frequency.lower() in {"daily", "1day"}:
295+
start_date = _session_label(start_date)
296+
end_date = _session_label(end_date)
297+
if feed_spec is None and self.feed_spec is None:
298+
try:
299+
feed_spec = normalize_storage_metadata(storage.get_metadata(key), key)
300+
except KeyError:
301+
pass
302+
feed, selected = self._feed_calendar(calendar, feed_spec)
303+
if selected is None:
304+
return []
305+
# Session labels retain their civil date even when stored with a timezone.
306+
start_bound = start_date - timedelta(days=1)
307+
end_bound = datetime.combine(end_date.date(), datetime.min.time(), UTC) + timedelta(
308+
days=2
309+
)
310+
try:
311+
df = storage.read(key, start_bound, end_bound).collect()
312+
except KeyError:
313+
df = pl.DataFrame(schema={"timestamp": pl.Datetime("us", "UTC")})
314+
return self._daily_ranges(
315+
self._label_frame(df, feed), start_date, end_date, feed, selected
316+
)
195317
start_date = _ensure_datetime(start_date)
196318
end_date = _ensure_datetime(end_date)
319+
_, selected = self._feed_calendar(calendar, feed_spec)
320+
if selected is None:
321+
return []
322+
if (
323+
calendar is not UNSET_CALENDAR
324+
or self.calendar is not UNSET_CALENDAR
325+
or feed_spec is not None
326+
or self.feed_spec is not None
327+
):
328+
raise NotImplementedError(
329+
"calendar/feed declarations currently support daily ranges only"
330+
)
197331
try:
198332
# Storage end bounds are exclusive. Open the inclusive end by one
199333
# microsecond so a bar timestamped on end_date is part of the range.
@@ -529,14 +663,23 @@ def apply_strategy(
529663
existing_df, new_data = align_frames_for_concat(existing_df, new_data)
530664

531665
# Detect gaps
532-
gaps = self.gap_detector.detect_gaps(existing_df)
666+
feed_metadata = normalize_storage_metadata(storage.get_metadata(key), key)
667+
gaps = self.gap_detector.detect_gaps(existing_df, feed_spec=feed_metadata)
668+
feed, selected_calendar = self.gap_detector._feed_calendar(
669+
UNSET_CALENDAR, feed_metadata
670+
)
533671
gaps_filled = 0
534672

535673
# Filter new data to only gap periods
536674
gap_data = pl.DataFrame()
537675
for gap in gaps:
676+
expected = SessionAssigner(selected_calendar).calendar.schedule(
677+
start_date=gap["start"].date(), end_date=gap["end"].date()
678+
)
538679
gap_rows = new_data.filter(
539-
(pl.col("timestamp") >= gap["start"]) & (pl.col("timestamp") <= gap["end"])
680+
self.gap_detector._label_frame(new_data, feed)["timestamp"]
681+
.dt.date()
682+
.is_in([label.date() for label in expected.index])
540683
)
541684
if not gap_rows.is_empty():
542685
gap_data = (
@@ -688,7 +831,8 @@ def identify_candidates(
688831
continue
689832

690833
# Detect gaps
691-
gaps = self.gap_detector.detect_gaps(df)
834+
feed_metadata = normalize_storage_metadata(self.storage.get_metadata(key), key)
835+
gaps = self.gap_detector.detect_gaps(df, feed_spec=feed_metadata)
692836

693837
# Filter significant gaps
694838
significant_gaps = [g for g in gaps if g["size_days"] >= min_gap_days]

0 commit comments

Comments
 (0)