|
13 | 13 |
|
14 | 14 | from ml4t.data.core.schemas import align_frames_for_concat |
15 | 15 | from ml4t.data.providers.base import BaseProvider |
16 | | -from ml4t.data.sessions.assigner import SessionAssigner |
| 16 | +from ml4t.data.sessions.assigner import SessionAssigner, _daily_session_labels |
17 | 17 | from ml4t.data.storage.backend import StorageBackend, normalize_storage_metadata |
18 | 18 | from ml4t.data.storage.metadata_tracker import MetadataTracker, UpdateRecord |
19 | | -from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec, TimestampSemantics |
| 19 | +from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec |
20 | 20 |
|
21 | 21 | logger = structlog.get_logger() |
22 | 22 |
|
@@ -136,34 +136,20 @@ def _feed_calendar( |
136 | 136 | return feed, feed.resolve_calendar(calendar=calendar) |
137 | 137 |
|
138 | 138 | @staticmethod |
139 | | - def _label_frame(df: pl.DataFrame, feed: FeedSpec) -> pl.DataFrame: |
140 | | - dtype = df.schema["timestamp"] |
141 | | - if dtype != pl.Date and not isinstance(dtype, pl.Datetime): |
142 | | - raise TypeError("'timestamp' must contain Date or Datetime values") |
143 | | - if df["timestamp"].null_count(): |
144 | | - raise ValueError("'timestamp' must not contain null timestamps") |
145 | | - if isinstance(dtype, pl.Datetime) and feed.timezone is not None: |
146 | | - timestamp = pl.col("timestamp") |
147 | | - if dtype.time_zone is None: |
148 | | - timestamp = timestamp.dt.replace_time_zone(feed.timezone) |
149 | | - else: |
150 | | - timestamp = timestamp.dt.convert_time_zone(feed.timezone) |
151 | | - return df.with_columns(timestamp) |
152 | | - return df |
| 139 | + def _label_frame(df: pl.DataFrame, feed: FeedSpec, calendar: str | None) -> pl.DataFrame: |
| 140 | + labels = _daily_session_labels(df, SessionAssigner(calendar, feed_spec=feed)) |
| 141 | + return df.with_columns(labels.cast(pl.Datetime("us")).dt.replace_time_zone("UTC")) |
153 | 142 |
|
154 | 143 | @staticmethod |
155 | 144 | def _daily_ranges( |
156 | 145 | df: pl.DataFrame, |
157 | 146 | start: datetime, |
158 | 147 | end: datetime, |
159 | | - feed: FeedSpec, |
160 | 148 | calendar: str, |
161 | 149 | tolerance_days: int = 0, |
162 | 150 | ) -> list[dict[str, Any]]: |
163 | 151 | if start.date() > end.date(): |
164 | 152 | raise ValueError("start_date must not be after end_date") |
165 | | - if feed.timestamp_semantics not in (None, TimestampSemantics.SESSION_LABEL): |
166 | | - raise NotImplementedError("daily range planning requires session_label timestamps") |
167 | 153 | assigner = SessionAssigner(calendar) |
168 | 154 | observed = { |
169 | 155 | value.date() if isinstance(value, datetime) else value |
@@ -222,12 +208,11 @@ def detect_gaps( |
222 | 208 | if df.is_empty() or "timestamp" not in df.columns: |
223 | 209 | return [] |
224 | 210 | if frequency.lower() in {"daily", "1day"}: |
225 | | - df = self._label_frame(df, feed) |
| 211 | + df = self._label_frame(df, feed, selected) |
226 | 212 | return self._daily_ranges( |
227 | 213 | df, |
228 | 214 | _session_label(df["timestamp"].min()), |
229 | 215 | _session_label(df["timestamp"].max()), |
230 | | - feed, |
231 | 216 | selected, |
232 | 217 | tolerance_days, |
233 | 218 | ) |
@@ -312,7 +297,7 @@ def detect_gaps_in_storage( |
312 | 297 | except KeyError: |
313 | 298 | df = pl.DataFrame(schema={"timestamp": pl.Datetime("us", "UTC")}) |
314 | 299 | return self._daily_ranges( |
315 | | - self._label_frame(df, feed), start_date, end_date, feed, selected |
| 300 | + self._label_frame(df, feed, selected), start_date, end_date, selected |
316 | 301 | ) |
317 | 302 | start_date = _ensure_datetime(start_date) |
318 | 303 | end_date = _ensure_datetime(end_date) |
@@ -677,7 +662,9 @@ def apply_strategy( |
677 | 662 | start_date=gap["start"].date(), end_date=gap["end"].date() |
678 | 663 | ) |
679 | 664 | gap_rows = new_data.filter( |
680 | | - self.gap_detector._label_frame(new_data, feed)["timestamp"] |
| 665 | + self.gap_detector._label_frame(new_data, feed, selected_calendar)[ |
| 666 | + "timestamp" |
| 667 | + ] |
681 | 668 | .dt.date() |
682 | 669 | .is_in([label.date() for label in expected.index]) |
683 | 670 | ) |
|
0 commit comments