Skip to content

Commit 60f6591

Browse files
author
Stefan Jansen
committed
Merge branch 'fix/stored-calendar-gap-report' into fix/source-backed-gap-recovery
2 parents 438044e + a1f6141 commit 60f6591

6 files changed

Lines changed: 309 additions & 19 deletions

File tree

‎docs/user-guide/incremental-updates.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,26 @@ bars, wholly absent sessions and absent datasets. Breaks, maintenance, holidays,
185185
sessions and partial-bar policy follow the selected schedule. It shares the arithmetic expected grid
186186
with in-memory detection rather than constructing a second calendar or every absent row.
187187

188+
Existing Parquet files do not need to be imported into a Data storage backend. Read the file with
189+
Polars and supply the same declaration and explicit coverage bounds:
190+
191+
```python
192+
import polars as pl
193+
194+
report = GapDetector(feed_spec=feed).detect_gaps(
195+
pl.read_parquet("./prices.parquet"),
196+
start_date=datetime(2026, 10, 16, 0, 0, tzinfo=UTC),
197+
end_date=datetime(2026, 10, 16, 23, 59, tzinfo=UTC),
198+
)
199+
```
200+
201+
Without explicit bounds, frame detection checks between the first and last stored timestamps.
202+
Explicit bounds also reveal missing boundary bars and wholly empty intervals. Use `timestamp_col`
203+
in the feed declaration when the file names its time column differently. A Boolean `is_imputed=True`
204+
marker identifies a known placeholder, which does not satisfy an expected observation. Zero volume
205+
alone does not identify a placeholder. Previously fabricated rows without a marker cannot be
206+
distinguished from real observations by timestamps alone.
207+
188208
An intraday calendar alone is insufficient. Frequency, timestamp side, anchor and partial-bar policy
189209
must be declared, as in the example; contradictory or missing declarations fail clearly. Multi-asset
190210
datasets are compared separately by the declared `entity_col`, or `symbol` when present; their gap

‎src/ml4t/data/update_manager.py‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ def _daily_ranges(
190190
assigner = SessionAssigner(calendar)
191191
observed = {
192192
value.date() if isinstance(value, datetime) else value
193-
for value in df["timestamp"].to_list()
193+
for value in ObservationGapDetector._observed_rows(df)["timestamp"].to_list()
194194
}
195195
schedule = assigner.calendar.schedule(start_date=start.date(), end_date=end.date())
196196
gaps = []
@@ -226,6 +226,8 @@ def detect_gaps(
226226
calendar: str | None | CalendarUnset = UNSET_CALENDAR,
227227
feed_spec: Any | None = None,
228228
tolerance_bars: int = 0,
229+
start_date: datetime | None = None,
230+
end_date: datetime | None = None,
229231
) -> list[dict[str, Any]]:
230232
"""
231233
Detect gaps in a DataFrame's timestamp column.
@@ -237,24 +239,44 @@ def detect_gaps(
237239
calendar: Per-call override of configured calendar and feed metadata.
238240
feed_spec: Per-call feed declaration.
239241
tolerance_bars: Number of consecutive missing intraday bars to tolerate.
242+
start_date: Inclusive requested coverage start; supply together with end_date.
243+
end_date: Inclusive coverage end. Daily bounds are labels; intraday bounds are instants.
240244
241245
Returns:
242246
List of gap dictionaries with start, end, and size
243247
"""
248+
if (start_date is None) != (end_date is None):
249+
raise ValueError("start_date and end_date must be supplied together")
244250
feed, selected = self._feed_calendar(calendar, feed_spec)
245251
frequency = (feed.data_frequency or "daily") if frequency is None else frequency
246252
if selected is None:
247253
return []
248-
if df.is_empty() or "timestamp" not in df.columns:
254+
if df.is_empty() and start_date is None:
249255
return []
256+
if feed.timestamp_col != "timestamp":
257+
if feed.timestamp_col not in df.columns:
258+
raise ValueError(f"declared timestamp column {feed.timestamp_col!r} is missing")
259+
df = df.select(
260+
pl.col(feed.timestamp_col).alias("timestamp"),
261+
*[name for name in df.columns if name not in {"timestamp", feed.timestamp_col}],
262+
)
263+
feed = feed.with_overrides(timestamp_col="timestamp")
264+
if "timestamp" not in df.columns:
265+
if not df.is_empty():
266+
raise ValueError("DataFrame must have 'timestamp' column")
267+
df = df.with_columns(pl.Series("timestamp", [], dtype=pl.Datetime("us", "UTC")))
250268
if frequency.lower() in {"daily", "1day", "1d"}:
251269

252270
def daily_ranges(part: pl.DataFrame) -> list[dict[str, Any]]:
253271
labeled = self._label_frame(part, feed, selected)
254272
return self._daily_ranges(
255273
labeled,
256-
_session_label(labeled["timestamp"].min()),
257-
_session_label(labeled["timestamp"].max()),
274+
_session_label(
275+
start_date if start_date is not None else labeled["timestamp"].min()
276+
),
277+
_session_label(
278+
end_date if end_date is not None else labeled["timestamp"].max()
279+
),
258280
selected,
259281
tolerance_days,
260282
)
@@ -267,7 +289,13 @@ def daily_ranges(part: pl.DataFrame) -> list[dict[str, Any]]:
267289
df,
268290
feed,
269291
lambda part: self._intraday_ranges(
270-
part, frequency, selected, feed, tolerance_bars=tolerance_bars
292+
part,
293+
frequency,
294+
selected,
295+
feed,
296+
tolerance_bars=tolerance_bars,
297+
start=_ensure_datetime(start_date) if start_date is not None else None,
298+
end=_ensure_datetime(end_date) if end_date is not None else None,
271299
),
272300
)
273301
if (
@@ -329,8 +357,8 @@ def _intraday_ranges(
329357
if grid is None:
330358
return []
331359
positions = sorted({grid.position(timestamp) for timestamp in observed})
332-
first = grid.bound(start) if start is not None else positions[0]
333-
stop = grid.bound(end, inclusive=True) if end is not None else positions[-1] + 1
360+
first = grid.bound(grid.start)
361+
stop = grid.bound(grid.end, inclusive=True)
334362
gaps = []
335363
cursor = first
336364
for position in [p for p in positions if first <= p < stop] + [stop]:

‎src/ml4t/data/utils/gaps.py‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,16 @@ def __init__(
6060
interval: timedelta,
6161
right_labeled: bool,
6262
include_partial: bool,
63+
*,
64+
start: datetime,
65+
end: datetime,
6366
) -> None:
6467
self.intervals = intervals
6568
self.interval = interval
6669
self.right_labeled = right_labeled
6770
self.include_partial = include_partial
71+
self.start = start
72+
self.end = end
6873
self.boundaries = [row[1] if right_labeled else row[0] for row in intervals]
6974
self.offsets = [row[3] for row in intervals]
7075
self.size = intervals[-1][3] + intervals[-1][2] if intervals else 0
@@ -176,6 +181,7 @@ def detect_gaps(
176181
selected_calendar = feed.resolve_calendar(calendar=calendar)
177182
if selected_calendar is None:
178183
return []
184+
df = self._observed_rows(df)
179185
declared_interval = (
180186
self._intraday_interval(str(feed.data_frequency))
181187
if feed.data_frequency is not None
@@ -244,6 +250,14 @@ def detect_gaps(
244250

245251
return gaps
246252

253+
@staticmethod
254+
def _observed_rows(df: pl.DataFrame) -> pl.DataFrame:
255+
if "is_imputed" not in df.columns:
256+
return df
257+
if df.schema["is_imputed"] != pl.Boolean:
258+
raise TypeError("is_imputed must be a Boolean placeholder marker")
259+
return df.filter(~pl.col("is_imputed").fill_null(False))
260+
247261
@staticmethod
248262
def _intraday_interval(frequency: str) -> timedelta | None:
249263
value = frequency.lower().strip().replace("hourly", "hour")
@@ -292,23 +306,33 @@ def _intraday_grid(
292306
raise TypeError(f"'{timestamp_col}' must contain Datetime values for an intraday grid")
293307
if df[timestamp_col].null_count():
294308
raise ValueError(f"'{timestamp_col}' must not contain null timestamps")
295-
if (
296-
dtype.time_unit == "ns"
297-
and df.select((pl.col(timestamp_col).dt.nanosecond() % 1000 != 0).any()).item()
298-
):
299-
raise ValueError("intraday observations contain off-grid sub-microsecond timestamps")
300309
if dtype.time_zone is None and feed.timezone is None:
301310
raise ValueError("naive intraday timestamps require a declared timezone")
302311
expr = pl.col(timestamp_col)
303312
if dtype.time_zone is None:
304313
expr = expr.dt.replace_time_zone(feed.timezone)
305-
observed = df.select(expr.dt.convert_time_zone("UTC")).to_series().unique().sort().to_list()
306-
if not observed and (start is None or end is None):
314+
columns = [expr.dt.convert_time_zone("UTC").alias(timestamp_col)]
315+
if "is_imputed" in df.columns:
316+
columns.append(pl.col("is_imputed"))
317+
scoped = df.select(columns)
318+
if start is not None:
319+
scoped = scoped.filter(pl.col(timestamp_col) >= start)
320+
if end is not None:
321+
scoped = scoped.filter(pl.col(timestamp_col) <= end)
322+
timestamps = scoped[timestamp_col].unique().sort().to_list()
323+
if not timestamps and (start is None or end is None):
307324
return None, []
308-
start = start if start is not None else observed[0]
309-
end = end if end is not None else observed[-1]
325+
start = start if start is not None else timestamps[0]
326+
end = end if end is not None else timestamps[-1]
310327
if start > end:
311328
raise ValueError("start_date must not be after end_date")
329+
actual = self._observed_rows(scoped)
330+
if (
331+
dtype.time_unit == "ns"
332+
and actual.select((pl.col(timestamp_col).dt.nanosecond() % 1000 != 0).any()).item()
333+
):
334+
raise ValueError("intraday observations contain off-grid sub-microsecond timestamps")
335+
observed = actual[timestamp_col].unique().sort().to_list()
312336
schedule = _schedule_for_instants(assigner.calendar, start, end, padding_days=2)
313337
grid_intervals: list[tuple[datetime, datetime, int, int]] = []
314338
total_bars = 0
@@ -330,6 +354,8 @@ def _intraday_grid(
330354
interval,
331355
feed.bar_label == BarLabel.RIGHT,
332356
feed.partial_bar_policy == PartialBarPolicy.INCLUDE,
357+
start=start,
358+
end=end,
333359
),
334360
observed,
335361
)

0 commit comments

Comments
 (0)