Skip to content

Commit 8fab64b

Browse files
committed
Merge qualified Roll covariance maintenance into tick classification repair
2 parents f5710e7 + 828d594 commit 8fab64b

18 files changed

Lines changed: 717 additions & 457 deletions

File tree

‎.github/workflows/ecosystem.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
qualification:
1818
permissions:
1919
contents: read
20-
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@da2cbb9a9d0b4d2f166ad630470ff7d7a1aa2643 # 2026-09-19 policy snapshot
20+
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@066dc737a879faea51a5b8a900315441712ea89a # 2026-09-25 policy snapshot
2121
with:
2222
import-package: ml4t.engineer
2323
prerelease-exception: python-315-polars

‎.github/workflows/python315-canary.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,5 +47,5 @@ jobs:
4747
PYTHONPATH: src
4848
run: >-
4949
uv run --isolated --no-project --python 3.15
50-
--with-requirements python315-core.txt --with pytest
50+
--with-requirements python315-core.txt --with pytest --with pytest-cov
5151
pytest tests/test_api.py tests/test_bars.py tests/bars -q

‎.github/workflows/release.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ jobs:
7575
needs: validate
7676
permissions:
7777
contents: read
78-
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@da2cbb9a9d0b4d2f166ad630470ff7d7a1aa2643 # 2026-09-19 policy snapshot
78+
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@066dc737a879faea51a5b8a900315441712ea89a # 2026-09-25 policy snapshot
7979
with:
8080
import-package: ml4t.engineer
8181
prerelease-exception: python-315-polars

‎.pre-commit-config.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
repos:
22
- repo: https://github.com/astral-sh/ruff-pre-commit
3-
rev: v0.16.1
3+
rev: v0.17.0
44
hooks:
55
- id: ruff-format
6-
- id: ruff
6+
types_or: [python, pyi, jupyter]
7+
- id: ruff-check
78
args: [--fix]
89

910
- repo: https://github.com/pre-commit/pre-commit-hooks

‎docs/user-guide/labeling.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,12 @@ result = calendar_aware_labels(
426426
)
427427
```
428428

429-
The calendar prevents forward returns from crossing session breaks (e.g., CME overnight gap from 4:00 PM to 5:00 PM CT).
429+
The calendar prevents forward returns from crossing genuine session breaks, such as the
430+
`CME_Equity` daily maintenance period from 4:00 PM to 5:00 PM CT. Its historical 3:15 PM to
431+
3:30 PM pause applies only before trade date June 28, 2021. Modern labels can cross that
432+
former pause because trading is continuous until 4:00 PM.
433+
See the [CME exchange notice](https://www.cmegroup.com/notices/electronic-trading/2021/06/20210621.html)
434+
for the affected equity-index contracts.
430435

431436
## Sample Weighting
432437

‎pyproject.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ store = [
106106

107107
# Trading calendar support
108108
calendars = [
109-
"pandas-market-calendars>=4.0.0",
109+
"pandas-market-calendars>=5.5.0",
110110
]
111111

112112
# Statistical diagnostics
@@ -141,7 +141,7 @@ dev = [
141141
"ipdb>=0.13.0",
142142
"pre-commit>=3.3.0",
143143
"duckdb>=0.9.0",
144-
"pandas-market-calendars>=4.0.0",
144+
"pandas-market-calendars>=5.5.0",
145145
"ta-lib>=0.4.0,<0.8",
146146
"yfinance>=0.2.0",
147147
"lightgbm>=4.0.0",
@@ -162,7 +162,7 @@ docs = [
162162
all = [
163163
"ta-lib>=0.4.0,<0.8",
164164
"duckdb>=0.9.0",
165-
"pandas-market-calendars>=4.0.0",
165+
"pandas-market-calendars>=5.5.0",
166166
"plotly>=5.15.0",
167167
"matplotlib>=3.7.0",
168168
"seaborn>=0.12.0",
@@ -209,7 +209,7 @@ dev = [
209209
"pre-commit>=3.3.0",
210210
"twine>=6.0.0",
211211
"pip-audit>=2.10.1",
212-
"pandas-market-calendars>=4.0.0",
212+
"pandas-market-calendars>=5.5.0",
213213
"duckdb>=0.9.0",
214214
"matplotlib>=3.7.0",
215215
"pyarrow>=14.0.0",

‎src/ml4t/engineer/bars/base.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ def _validate_data(self, data: pl.DataFrame) -> None:
9494
raise DataValidationError("Side values must not be null")
9595
if not data["side"].dtype.is_numeric():
9696
raise DataValidationError("Side column must be numeric")
97-
if not data["side"].is_in([-1, 1]).all():
97+
side = data["side"]
98+
if not ((side == -1) | (side == 1)).all():
9899
raise DataValidationError("Side values must be either -1 or 1")
99100

100101
@staticmethod

‎src/ml4t/engineer/bars/run.py‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
"""Run bar sampler implementations.
22
3-
AFML-compliant run bars per López de Prado Chapter 2.3.
3+
Run bars based on López de Prado's cumulative run definition.
44
5-
Run bar threshold formula:
5+
Tick-run threshold formula:
66
E[θ_T] = E[T] × max{P[b=1], 1-P[b=1]}
77
88
Where:
99
E[T] = EWMA of bar lengths (ticks per bar)
1010
P[b=1] = buy probability
1111
12+
Volume and dollar variants use cumulative volume/value as the adaptive size
13+
in the same threshold calculation, rather than mixing value units with tick counts.
14+
1215
CRITICAL: θ_T = max{Σ(all buys in bar), Σ(all sells in bar)}
1316
- NOT consecutive same-sided trades
1417
- NO reset on direction change within bar
@@ -51,8 +54,10 @@ def _calculate_run_bars_nb(
5154
npt.NDArray[np.float64],
5255
npt.NDArray[np.float64],
5356
npt.NDArray[np.float64],
57+
float,
58+
float,
5459
]:
55-
"""Calculate AFML-compliant run bar indices.
60+
"""Calculate cumulative weighted run bar indices.
5661
5762
AFML Chapter 2.3 formula:
5863
θ_T = max{Σ(all buys in bar), Σ(all sells in bar)}
@@ -68,7 +73,7 @@ def _calculate_run_bars_nb(
6873
sides : ndarray
6974
Array of trade signs (+1 for buy, -1 for sell)
7075
initial_expected_t : float
71-
Initial expected ticks per bar (E[T])
76+
Initial expected total value per bar, in the same units as values
7277
initial_p_buy : float
7378
Initial buy probability P[b=1]
7479
alpha : float
@@ -78,9 +83,9 @@ def _calculate_run_bars_nb(
7883
7984
Returns
8085
-------
81-
tuple of arrays
86+
tuple
8287
(bar_indices, thetas, expected_thetas, expected_ts, p_buys,
83-
cumulative_buys, cumulative_sells)
88+
cumulative_buys, cumulative_sells, current_expected_t, current_p_buy)
8489
"""
8590
n = len(values)
8691

@@ -137,10 +142,13 @@ def _calculate_run_bars_nb(
137142

138143
n_bars += 1
139144

140-
# Update EWMAs after warmup period
145+
# Update EWMAs after warmup period.
146+
# expected_t is denominated in the same units as `values`: ticks when
147+
# every value is 1, and volume or dollars when those series are passed.
148+
# Updating it with the tick count collapses a value-scaled threshold.
141149
if n_bars > min_bars_warmup:
142-
# Update E[T] - expected ticks per bar
143-
expected_t = alpha * bar_tick_count + (1 - alpha) * expected_t
150+
bar_value = cumulative_buys + cumulative_sells
151+
expected_t = alpha * bar_value + (1 - alpha) * expected_t
144152

145153
# Update P[b=1] - buy probability
146154
bar_p_buy = bar_buy_count / bar_tick_count if bar_tick_count > 0 else 0.5
@@ -160,6 +168,8 @@ def _calculate_run_bars_nb(
160168
np.array(p_buys, dtype=np.float64),
161169
np.array(cumulative_buys_out, dtype=np.float64),
162170
np.array(cumulative_sells_out, dtype=np.float64),
171+
float(expected_t),
172+
float(p_buy),
163173
)
164174

165175

@@ -258,6 +268,8 @@ def sample(
258268
p_buys,
259269
cumulative_buys,
260270
cumulative_sells,
271+
current_expected_t,
272+
current_p_buy,
261273
) = _calculate_run_bars_nb(
262274
values,
263275
sides,
@@ -315,11 +327,7 @@ def sample(
315327
cum_sells = float(np.sum(bar_sides < 0))
316328
current_theta = max(cum_buys, cum_sells)
317329

318-
last_expected_t = (
319-
expected_ts[-1] if len(expected_ts) > 0 else float(self.expected_ticks_per_bar)
320-
)
321-
last_p_buy = p_buys[-1] if len(p_buys) > 0 else self.initial_p_buy
322-
expected_theta = last_expected_t * max(last_p_buy, 1 - last_p_buy)
330+
expected_theta = current_expected_t * max(current_p_buy, 1 - current_p_buy)
323331

324332
bar = self._create_ohlcv_bar(
325333
bar_ticks,
@@ -330,8 +338,8 @@ def sample(
330338
"expected_run": float(expected_theta),
331339
"theta": float(current_theta),
332340
"expected_theta": float(expected_theta),
333-
"expected_t": float(last_expected_t),
334-
"p_buy": float(last_p_buy),
341+
"expected_t": current_expected_t,
342+
"p_buy": current_p_buy,
335343
"cumulative_buys": float(cum_buys),
336344
"cumulative_sells": float(cum_sells),
337345
},
@@ -369,7 +377,7 @@ def _empty_run_bars_df(self) -> pl.DataFrame:
369377

370378

371379
class VolumeRunBarSampler(BarSampler):
372-
"""Sample bars based on cumulative volume runs (AFML-compliant).
380+
"""Sample bars based on cumulative volume runs.
373381
374382
AFML Chapter 2.3 formula with volume weighting:
375383
θ_T = max{Σ(buy volumes in bar), Σ(sell volumes in bar)}
@@ -448,6 +456,8 @@ def sample(
448456
p_buys,
449457
cumulative_buys,
450458
cumulative_sells,
459+
current_expected_t,
460+
current_p_buy,
451461
) = _calculate_run_bars_nb(
452462
volumes, # Use volumes as values
453463
sides,
@@ -503,11 +513,7 @@ def sample(
503513
# Calculate current theta (cumulative volumes)
504514
current_theta = max(buy_vol, sell_vol)
505515

506-
last_expected_t = (
507-
expected_ts[-1] if len(expected_ts) > 0 else initial_expected_t_scaled
508-
)
509-
last_p_buy = p_buys[-1] if len(p_buys) > 0 else self.initial_p_buy
510-
expected_theta = last_expected_t * max(last_p_buy, 1 - last_p_buy)
516+
expected_theta = current_expected_t * max(current_p_buy, 1 - current_p_buy)
511517

512518
bar = self._create_ohlcv_bar(
513519
bar_ticks,
@@ -518,8 +524,8 @@ def sample(
518524
"expected_run": float(expected_theta),
519525
"theta": float(current_theta),
520526
"expected_theta": float(expected_theta),
521-
"expected_t": float(last_expected_t),
522-
"p_buy": float(last_p_buy),
527+
"expected_t": current_expected_t,
528+
"p_buy": current_p_buy,
523529
"cumulative_buys": float(buy_vol),
524530
"cumulative_sells": float(sell_vol),
525531
},
@@ -557,7 +563,7 @@ def _empty_run_bars_df(self) -> pl.DataFrame:
557563

558564

559565
class DollarRunBarSampler(BarSampler):
560-
"""Sample bars based on cumulative dollar value runs (AFML-compliant).
566+
"""Sample bars based on cumulative dollar value runs.
561567
562568
AFML Chapter 2.3 formula with dollar weighting:
563569
θ_T = max{Σ(buy dollars in bar), Σ(sell dollars in bar)}
@@ -635,6 +641,8 @@ def sample(
635641
p_buys,
636642
cumulative_buys,
637643
cumulative_sells,
644+
current_expected_t,
645+
current_p_buy,
638646
) = _calculate_run_bars_nb(
639647
dollar_volumes, # Use dollar volumes as values
640648
sides,
@@ -702,11 +710,7 @@ def sample(
702710
sell_dollars = float(np.sum(bar_dollars[bar_sides < 0]))
703711
current_theta = max(buy_dollars, sell_dollars)
704712

705-
last_expected_t = (
706-
expected_ts[-1] if len(expected_ts) > 0 else initial_expected_t_scaled
707-
)
708-
last_p_buy = p_buys[-1] if len(p_buys) > 0 else self.initial_p_buy
709-
expected_theta = last_expected_t * max(last_p_buy, 1 - last_p_buy)
713+
expected_theta = current_expected_t * max(current_p_buy, 1 - current_p_buy)
710714

711715
bar = self._create_ohlcv_bar(
712716
bar_ticks,
@@ -719,8 +723,8 @@ def sample(
719723
"expected_run": float(expected_theta),
720724
"theta": float(current_theta),
721725
"expected_theta": float(expected_theta),
722-
"expected_t": float(last_expected_t),
723-
"p_buy": float(last_p_buy),
726+
"expected_t": current_expected_t,
727+
"p_buy": current_p_buy,
724728
"cumulative_buys": float(buy_dollars),
725729
"cumulative_sells": float(sell_dollars),
726730
},

‎src/ml4t/engineer/core/lookbacks.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ def _ulcer_index(params: Mapping[str, Any]) -> int:
153153
"price_impact_ratio": _parameter("period", -1),
154154
"quote_stuffing_indicator": _scaled_parameter("period", 4, -1),
155155
"realized_spread": _parameter("period", -1),
156-
"roll_spread_estimator": _scaled_parameter("period", 2),
156+
"roll_spread_estimator": _parameter("period", 1),
157157
"trade_intensity": _parameter("period", -1),
158158
"volume_at_price_ratio": _parameter("period", -1),
159159
"volume_synchronicity": _scaled_parameter("period", 2, -1),

‎src/ml4t/engineer/features/microstructure/roll_spread_estimator.py‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def roll_spread_estimator(close: pl.Expr | str, period: int = 20) -> pl.Expr:
3232
Returns
3333
-------
3434
pl.Expr
35-
Estimated bid-ask spread
35+
Estimated bid-ask spread, null for an unobserved covariance window
3636
3737
Raises
3838
------
@@ -46,23 +46,18 @@ def roll_spread_estimator(close: pl.Expr | str, period: int = 20) -> pl.Expr:
4646

4747
close = pl.col(close) if isinstance(close, str) else close
4848

49-
# Calculate price changes
49+
# Serial covariance of price changes. Demeaning each point by its own
50+
# trailing mean and then averaging those products is not the covariance of
51+
# the window; cov = E[xy] - E[x]E[y] over the same window is.
5052
price_changes = close.diff()
51-
52-
# Calculate rolling covariance of price changes with lagged changes
53-
# Note: Polars doesn't have rolling covariance, so we approximate
5453
lag_changes = price_changes.shift(1)
55-
56-
# Calculate components for covariance
5754
mean_change = price_changes.rolling_mean(period)
5855
mean_lag = lag_changes.rolling_mean(period)
59-
60-
# Approximate covariance
61-
cov_approx = ((price_changes - mean_change) * (lag_changes - mean_lag)).rolling_mean(period)
56+
cov_approx = (price_changes * lag_changes).rolling_mean(period) - mean_change * mean_lag
6257

6358
# Roll spread = 2 * sqrt(max(-cov, 0))
64-
spread = 2 * pl.when(cov_approx.is_null()).then(None).when(cov_approx < 0).then(
65-
(-cov_approx).sqrt()
66-
).otherwise(0)
59+
spread = 2 * pl.when(cov_approx.is_null() | cov_approx.is_nan()).then(None).when(
60+
cov_approx < 0
61+
).then((-cov_approx).sqrt()).otherwise(0)
6762

6863
return spread

0 commit comments

Comments
 (0)