Skip to content

Commit 9331160

Browse files
author
Stefan Jansen
committed
Merge remote-tracking branch 'origin/main' into build/published-specs-020
2 parents 7ad6411 + 0737b6a commit 9331160

6 files changed

Lines changed: 308 additions & 20 deletions

File tree

‎docs/user-guide/sessions.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,27 @@ assigner = SessionAssigner("CME Globex Crypto")
114114
Calendar names and schedules can change upstream. Check the installed calendar before assigning a
115115
large dataset.
116116

117+
### CME Crypto and Spot Crypto
118+
119+
`CME` selects CME Globex cryptocurrency futures and options, not all CME products and not
120+
spot-quoted futures. Data preserves the upstream historical schedule through May 29, 2026. After
121+
that day's 16:00-16:30 Chicago maintenance, trading includes weekends and holidays. Regular Globex
122+
maintenance is Monday-Friday 16:00-16:02 Chicago time and Saturday 02:00-04:00. Pre-open periods
123+
are not trading intervals. See the [CME crypto FAQ](https://www.cmegroup.com/articles/faqs/frequently-asked-questions-cryptocurrency-futures.html)
124+
and [launch notice](https://cmegroupclientsite.atlassian.net/wiki/spaces/EPICSANDBOX/pages/1283194884/Cryptocurrency+Futures+and+Options+Migration+to+24-7+Trading).
125+
126+
CME observations retain the following business trade date across weekends and holidays, rather
127+
than acquiring a separate Saturday or Sunday label. Data uses the upstream `CME_TradeDate`
128+
business calendar, qualified against CME's
129+
[2026 reporting calendar](https://www.cmegroup.com/content/dam/cmegroup/notices/market-regulation/2026/01/msn01-09-2026revisedholidayprocessinglargetrader2026.pdf).
130+
Schedule queries include the next business date when needed; a Saturday observation can therefore
131+
map to Monday, or Tuesday after Labor Day. Assignment and completion share these intervals and
132+
exclude maintenance. The adapter does not change the upstream calendar registry or other products.
133+
134+
Ordinary spot crypto uses `calendar="24/7"`, with civil-day labels and no CME maintenance. Explicit
135+
`calendar=None` disables session handling instead. Select the schedule from the feed's actual
136+
market, not from its symbol or asset-class name.
137+
117138
## Complete Minute Sessions
118139

119140
`SessionCompleter` expands minute bars to the exchange schedule. It requires `timestamp`, `open`,
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
"""CME crypto's May 2026 transition, without changing the upstream registry."""
2+
3+
from __future__ import annotations
4+
5+
from typing import Any
6+
7+
import pandas as pd
8+
9+
10+
class CMEGlobexCryptoSchedule:
11+
def __init__(self, legacy: Any, trade_dates: Any) -> None:
12+
self._legacy = legacy
13+
self._trade_dates = trade_dates
14+
self._business_day = trade_dates.holidays()
15+
self.tz = legacy.tz
16+
self.name = legacy.name
17+
18+
def schedule(
19+
self, start_date: Any, end_date: Any, tz: Any = "UTC", *, interruptions: bool = False
20+
) -> pd.DataFrame:
21+
start = pd.Timestamp(start_date).tz_localize(None).normalize()
22+
end = pd.Timestamp(end_date).tz_localize(None).normalize()
23+
if start > end:
24+
raise ValueError("start_date must not be later than end_date")
25+
cutover = pd.Timestamp("2026-05-29")
26+
frames = []
27+
if start <= cutover:
28+
frames.append(
29+
self._legacy.schedule(start, min(end, cutover), tz=tz, interruptions=interruptions)
30+
)
31+
if end > cutover:
32+
labels = self._trade_dates.valid_days(
33+
max(start, cutover + pd.Timedelta(days=1)), end, tz=None
34+
)
35+
rows = []
36+
for label in labels:
37+
previous = label - self._business_day
38+
opening = previous + pd.Timedelta(hours=16, minutes=2)
39+
if previous == cutover:
40+
opening = previous + pd.Timedelta(hours=16, minutes=30)
41+
closing = label + pd.Timedelta(hours=16)
42+
row = {
43+
"market_open": opening.tz_localize(self.tz).tz_convert(tz),
44+
"market_close": closing.tz_localize(self.tz).tz_convert(tz),
45+
}
46+
if interruptions:
47+
closures = []
48+
for day in pd.date_range(previous, label):
49+
if day.dayofweek < 5:
50+
halt = day + pd.Timedelta(hours=16)
51+
reopen = halt + pd.Timedelta(minutes=2)
52+
elif day.dayofweek == 5:
53+
halt = day + pd.Timedelta(hours=2)
54+
reopen = halt + pd.Timedelta(hours=2)
55+
else:
56+
continue
57+
if opening < reopen and halt < closing:
58+
closures.append((halt, reopen))
59+
for index, (halt, reopen) in enumerate(closures, start=1):
60+
row[f"interruption_start_{index}"] = halt.tz_localize(self.tz).tz_convert(
61+
tz
62+
)
63+
row[f"interruption_end_{index}"] = reopen.tz_localize(self.tz).tz_convert(
64+
tz
65+
)
66+
rows.append(row)
67+
frames.append(
68+
pd.DataFrame(
69+
rows, index=labels, columns=None if rows else ["market_open", "market_close"]
70+
)
71+
)
72+
return pd.concat(frames).sort_index()
73+
74+
def schedule_for_instants(self, start: Any, end: Any) -> pd.DataFrame:
75+
start_timestamp, end_timestamp = pd.Timestamp(start), pd.Timestamp(end)
76+
start_local = (
77+
start_timestamp.tz_convert(self.tz) if start_timestamp.tzinfo else start_timestamp
78+
)
79+
end_local = end_timestamp.tz_convert(self.tz) if end_timestamp.tzinfo else end_timestamp
80+
next_trade_date = pd.Timestamp(end_local.date()) + self._business_day
81+
return self.schedule(
82+
pd.Timestamp(start_local.date()) - pd.Timedelta(days=1),
83+
next_trade_date,
84+
interruptions=True,
85+
)

‎src/ml4t/data/sessions/assigner.py‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,20 @@ def _session_intervals(row: Any) -> list[tuple[datetime, datetime]]:
5252
return intervals
5353

5454

55+
def _schedule_for_instants(calendar: Any, start: Any, end: Any, *, padding_days: int = 1) -> Any:
56+
import pandas as pd
57+
58+
from ml4t.data.sessions._cme_crypto import CMEGlobexCryptoSchedule
59+
60+
if isinstance(calendar, CMEGlobexCryptoSchedule):
61+
return calendar.schedule_for_instants(start, end)
62+
return calendar.schedule(
63+
start_date=pd.Timestamp(start).normalize() - pd.Timedelta(days=padding_days),
64+
end_date=pd.Timestamp(end).normalize() + pd.Timedelta(days=padding_days),
65+
interruptions=True,
66+
)
67+
68+
5569
# Exchange to calendar mapping
5670
EXCHANGE_CALENDARS = {
5771
"CME": "CME Globex Crypto",
@@ -111,8 +125,7 @@ class SessionAssigner:
111125
Uses pandas_market_calendars to determine trading sessions and assigns
112126
a session_date column to each timestamp.
113127
114-
For CME futures, sessions start at 5pm CT Sunday and end at 4pm CT Friday.
115-
The session_date is the date when the session ENDS (4pm date).
128+
CME crypto uses business trade dates across overnight and weekend trading.
116129
"""
117130

118131
def __init__(
@@ -168,6 +181,16 @@ def __init__(
168181

169182
try:
170183
self.calendar = mcal.get_calendar(self.calendar_name)
184+
from pandas_market_calendars.calendars.cme_globex_crypto import (
185+
CMEGlobexCryptoExchangeCalendar,
186+
)
187+
188+
if type(self.calendar) is CMEGlobexCryptoExchangeCalendar:
189+
from ml4t.data.sessions._cme_crypto import CMEGlobexCryptoSchedule
190+
191+
self.calendar = CMEGlobexCryptoSchedule(
192+
self.calendar, mcal.get_calendar("CME_TradeDate")
193+
)
171194
except Exception as e:
172195
raise ValueError(f"Unknown calendar '{self.calendar_name}': {e}")
173196

@@ -265,11 +288,7 @@ def assign_sessions(
265288
end_timestamp, pd.Timestamp
266289
):
267290
raise ValueError("start_date and end_date must not be NaT")
268-
start_pd = start_timestamp.normalize() - pd.Timedelta(days=1)
269-
end_pd = end_timestamp.normalize() + pd.Timedelta(days=1)
270-
schedule = self.calendar.schedule(
271-
start_date=start_pd, end_date=end_pd, interruptions=True
272-
)
291+
schedule = _schedule_for_instants(self.calendar, start_timestamp, end_timestamp)
273292
logger.debug(f"Got {len(schedule)} sessions from calendar")
274293

275294
session_map = []

‎src/ml4t/data/sessions/completer.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@
1111
import polars as pl
1212
import structlog
1313

14-
from ml4t.data.sessions.assigner import SessionAssigner, _session_intervals
14+
from ml4t.data.sessions.assigner import (
15+
SessionAssigner,
16+
_schedule_for_instants,
17+
_session_intervals,
18+
)
1519
from ml4t.specs import UNSET_CALENDAR, BarLabel, CalendarUnset, TimestampSemantics
1620

1721
logger = structlog.get_logger()
@@ -166,13 +170,7 @@ def complete_sessions(
166170
):
167171
raise ValueError("start_date and end_date must be minute-aligned")
168172

169-
start_pd = request_start.normalize() - pd.Timedelta(days=1)
170-
end_pd = request_end.normalize() + pd.Timedelta(days=1)
171-
172-
# Get trading schedule
173-
schedule = self.calendar.schedule(
174-
start_date=start_pd, end_date=end_pd, interruptions=True
175-
)
173+
schedule = _schedule_for_instants(self.calendar, request_start, request_end)
176174

177175
if len(schedule) == 0:
178176
logger.warning("No trading sessions found in date range")

‎src/ml4t/data/utils/gaps.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
from ml4t.data.sessions.assigner import (
1515
SessionAssigner,
1616
_daily_session_labels,
17+
_schedule_for_instants,
1718
_session_intervals,
1819
)
1920
from ml4t.specs import (
@@ -247,11 +248,7 @@ def _detect_intraday_gaps(
247248
expr = expr.dt.replace_time_zone(feed.timezone)
248249
observed = df.select(expr.dt.convert_time_zone("UTC")).to_series().unique().sort().to_list()
249250
start, end = observed[0], observed[-1]
250-
schedule = assigner.calendar.schedule(
251-
start_date=(start - timedelta(days=2)).date(),
252-
end_date=(end + timedelta(days=2)).date(),
253-
interruptions=True,
254-
)
251+
schedule = _schedule_for_instants(assigner.calendar, start, end, padding_days=2)
255252
grid_intervals: list[tuple[datetime, datetime, int, int]] = []
256253
total_bars = 0
257254
for _, row in schedule.iterrows():

‎tests/test_cme_crypto_cutover.py‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
"""Real provider rows must survive the CME crypto schedule transition."""
2+
3+
from datetime import date, datetime
4+
5+
import polars as pl
6+
import pytest
7+
from polars.testing import assert_frame_equal
8+
9+
from ml4t.data import DataManager
10+
from ml4t.data.providers import SyntheticProvider
11+
from ml4t.data.sessions import SessionAssigner, SessionCompleter
12+
from ml4t.data.utils.gaps import GapDetector
13+
from ml4t.specs import FeedSpec
14+
15+
16+
def provider_rows(day: str, clocks: list[str]) -> pl.DataFrame:
17+
with SyntheticProvider(calendar_mode="continuous") as provider:
18+
frame = provider.fetch_ohlcv("BTCUSD", day, day, "minute")
19+
selected = frame.filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(clocks))
20+
assert selected.height == len(clocks)
21+
return selected
22+
23+
24+
@pytest.mark.parametrize(
25+
"day,clock,expected",
26+
[
27+
("2026-05-23", "12:00", None),
28+
("2026-05-30", "12:00", date(2026, 6, 1)),
29+
("2026-10-17", "12:00", date(2026, 10, 19)),
30+
("2026-10-16", "21:03", date(2026, 10, 19)),
31+
("2026-10-16", "21:00", None),
32+
("2026-10-16", "21:01", None),
33+
("2026-10-17", "07:59", None),
34+
("2026-10-17", "09:00", date(2026, 10, 19)),
35+
("2026-12-05", "08:00", None),
36+
("2026-12-05", "10:00", date(2026, 12, 7)),
37+
("2026-05-29", "21:15", None),
38+
("2026-05-29", "21:30", date(2026, 6, 1)),
39+
("2026-09-05", "12:00", date(2026, 9, 8)),
40+
("2026-07-02", "21:03", date(2026, 7, 6)),
41+
("2026-06-18", "21:03", date(2026, 6, 22)),
42+
("2026-11-25", "22:03", date(2026, 11, 27)),
43+
("2026-12-24", "22:03", date(2026, 12, 28)),
44+
("2026-10-12", "12:00", date(2026, 10, 12)),
45+
("2026-11-11", "12:00", date(2026, 11, 11)),
46+
("2026-05-29", "20:59", date(2026, 5, 29)),
47+
("2026-05-28", "21:30", None),
48+
("2026-05-28", "22:00", date(2026, 5, 29)),
49+
("2026-11-01", "07:30", date(2026, 11, 2)),
50+
],
51+
)
52+
def test_cme_crypto_uses_cutover_maintenance_and_business_dates(day, clock, expected):
53+
frame = provider_rows(day, [clock])
54+
assigned = SessionAssigner.from_exchange("CME").assign_sessions(frame, bar_frequency="intraday")
55+
assert assigned["session_date"].to_list() == [expected]
56+
assert_frame_equal(assigned.select(frame.columns), frame)
57+
58+
59+
def test_cme_completion_preserves_weekend_observations_and_fills_only_open_gap():
60+
frame = provider_rows("2026-10-17", ["12:00", "12:02"])
61+
completed = SessionCompleter("CME Globex Crypto").complete_sessions(frame)
62+
assert completed.height == 3
63+
assert completed["is_imputed"].to_list() == [False, True, False]
64+
assert completed["session_date"].to_list() == [date(2026, 10, 19)] * 3
65+
observed = completed.filter(~pl.col("is_imputed")).select(frame.columns).cast(frame.schema)
66+
assert_frame_equal(observed, frame)
67+
68+
69+
def test_spot_continuous_calendar_keeps_civil_dates_not_cme_trade_dates():
70+
frame = provider_rows("2026-10-17", ["12:00", "12:02"])
71+
assigned = SessionAssigner("24/7").assign_sessions(frame, bar_frequency="intraday")
72+
assert assigned["session_date"].to_list() == [date(2026, 10, 17)] * 2
73+
completed = SessionCompleter("24/7").complete_sessions(frame)
74+
assert completed.height == 3
75+
assert completed["session_date"].to_list() == [date(2026, 10, 17)] * 3
76+
77+
78+
def test_calendar_none_never_assigns_or_manufactures_cme_or_spot_rows():
79+
frame = provider_rows("2026-10-17", ["12:00", "12:02"])
80+
assert_frame_equal(SessionAssigner(None).assign_sessions(frame), frame)
81+
assert_frame_equal(SessionCompleter(None).complete_sessions(frame), frame)
82+
83+
84+
@pytest.mark.parametrize(
85+
"day,clocks,expected",
86+
[
87+
("2026-10-17", ["06:59", "09:00"], [date(2026, 10, 19)] * 2),
88+
("2026-09-07", ["20:59", "21:02"], [date(2026, 9, 8)] * 2),
89+
("2026-05-29", ["20:59", "21:30"], [date(2026, 5, 29), date(2026, 6, 1)]),
90+
],
91+
)
92+
def test_completion_does_not_fill_weekly_holiday_or_launch_maintenance(day, clocks, expected):
93+
frame = provider_rows(day, clocks)
94+
completed = SessionCompleter("CME Globex Crypto").complete_sessions(frame)
95+
assert completed.height == 2
96+
assert completed["is_imputed"].to_list() == [False, False]
97+
assert completed["session_date"].to_list() == expected
98+
assert_frame_equal(completed.select(frame.columns).cast(frame.schema), frame)
99+
100+
101+
@pytest.mark.parametrize("alias", ["CME Globex Crypto", "CME Globex Cryptocurrencies"])
102+
def test_cme_aliases_and_facade_honor_shared_override_and_disable(alias):
103+
frame = provider_rows("2026-10-17", ["12:00"])
104+
feed = FeedSpec(calendar=alias, timestamp_semantics="event_time")
105+
with DataManager() as manager:
106+
assert manager.assign_sessions(frame, feed_spec=feed)["session_date"].to_list() == [
107+
date(2026, 10, 19)
108+
]
109+
assert manager.assign_sessions(frame, calendar="24/7", feed_spec=feed)[
110+
"session_date"
111+
].to_list() == [date(2026, 10, 17)]
112+
assert_frame_equal(manager.assign_sessions(frame, calendar=None, feed_spec=feed), frame)
113+
114+
115+
def test_selected_crypto_schedule_does_not_modify_upstream_or_other_product_calendars():
116+
import pandas_market_calendars as mcal
117+
from pandas_market_calendars.calendars.cme_globex_crypto import CMEGlobexCryptoExchangeCalendar
118+
119+
SessionAssigner("CME Globex Crypto")
120+
legacy = mcal.get_calendar("CME Globex Crypto")
121+
assert type(legacy) is CMEGlobexCryptoExchangeCalendar
122+
assert legacy.schedule("2026-10-17", "2026-10-17").empty
123+
other = SessionAssigner("CME_Equity")
124+
frame = provider_rows("2026-10-17", ["12:00"])
125+
assert other.assign_sessions(frame, bar_frequency="intraday")["session_date"].to_list() == [
126+
None
127+
]
128+
129+
130+
def test_cme_daily_labels_are_business_dates_not_each_open_weekend_civil_date():
131+
with SyntheticProvider(calendar_mode="continuous") as provider:
132+
daily = provider.fetch_ohlcv("BTCUSD", "2026-10-17", "2026-10-19", "daily")
133+
feed = FeedSpec(calendar="CME Globex Crypto", timestamp_semantics="session_label")
134+
assigned = SessionAssigner(feed_spec=feed).assign_sessions(daily)
135+
assert assigned["session_date"].to_list() == [None, None, date(2026, 10, 19)]
136+
info = SessionCompleter("CME Globex Crypto").get_session_info(datetime(2026, 10, 19))
137+
assert info["market_open"].isoformat() == "2026-10-16T21:02:00+00:00"
138+
assert info["market_close"].isoformat() == "2026-10-19T21:00:00+00:00"
139+
140+
141+
@pytest.mark.parametrize(
142+
"day,clocks,missing",
143+
[
144+
("2026-09-05", ["12:00", "12:02"], 1),
145+
("2026-10-17", ["06:59", "09:00"], 0),
146+
("2026-09-07", ["20:59", "21:02"], 0),
147+
("2026-05-29", ["20:59", "21:30"], 0),
148+
],
149+
)
150+
def test_intraday_grid_uses_business_date_coverage_and_excludes_cme_maintenance(
151+
day, clocks, missing
152+
):
153+
frame = provider_rows(day, clocks)
154+
feed = FeedSpec(
155+
time_contract_version=2,
156+
calendar="CME Globex Crypto",
157+
data_frequency="minute",
158+
timestamp_semantics="event_time",
159+
bar_grid_anchor="session_open",
160+
bar_label="left",
161+
partial_bar_policy="exclude",
162+
)
163+
gaps = GapDetector().detect_gaps(frame, feed_spec=feed)
164+
assert sum(gap.missing_periods for gap in gaps) == missing
165+
if day == "2026-10-17":
166+
spot = GapDetector().detect_gaps(frame, calendar="24/7", feed_spec=feed)
167+
assert sum(gap.missing_periods for gap in spot) == 120
168+
assert GapDetector().detect_gaps(frame, calendar=None, feed_spec=feed) == []

0 commit comments

Comments
 (0)