Skip to content

Commit ce0e7e8

Browse files
fix(etfs): rebuild combined file from partitions, not stale aggregate
1 parent bde542a commit ce0e7e8

2 files changed

Lines changed: 79 additions & 1 deletion

File tree

‎src/ml4t/data/etfs/downloader.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,7 @@ def _save_combined(self, df: pl.DataFrame) -> None:
350350

351351
def _regenerate_combined(self) -> None:
352352
"""Regenerate combined file from individual ticker files."""
353-
all_data = self.load_all()
353+
all_data = self.load_symbols(self.config.get_all_symbols())
354354
if not all_data.is_empty():
355355
self._save_combined(all_data)
356356

‎tests/test_regenerate_combined.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
"""Regression tests for combined-file regeneration."""
2+
3+
from datetime import datetime
4+
5+
import polars as pl
6+
7+
from ml4t.data.etfs.downloader import ETFConfig, ETFDataManager
8+
9+
10+
def _bars(symbol: str, timestamps: list[datetime]) -> pl.DataFrame:
11+
"""Minimal OHLCV frame for one symbol."""
12+
n = len(timestamps)
13+
return pl.DataFrame(
14+
{
15+
"timestamp": timestamps,
16+
"symbol": [symbol] * n,
17+
"open": [1.0] * n,
18+
"high": [1.0] * n,
19+
"low": [1.0] * n,
20+
"close": [1.0] * n,
21+
"volume": [100.0] * n,
22+
},
23+
schema_overrides={"timestamp": pl.Datetime},
24+
)
25+
26+
27+
class TestRegenerateCombined:
28+
"""Tests for ETFDataManager._regenerate_combined."""
29+
30+
def test_rebuilds_from_partitions_when_aggregate_is_stale(self, tmp_path):
31+
"""A stale combined file must not survive regeneration.
32+
33+
``update()`` writes fresh bars to the per-ticker partitions and then
34+
calls ``_regenerate_combined()``. If regeneration reads the existing
35+
aggregate instead of the partitions, the stale file is rewritten
36+
unchanged and every downstream reader silently sees old data.
37+
"""
38+
config = ETFConfig(
39+
storage_path=tmp_path,
40+
tickers={"equity": {"symbols": ["SPY"]}},
41+
)
42+
manager = ETFDataManager(config)
43+
44+
# Partition holds bars through the newer date.
45+
fresh = _bars("SPY", [datetime(2025, 12, 31), datetime(2026, 8, 20)])
46+
manager._save_ticker(fresh, "SPY")
47+
48+
# Aggregate is stale: it stops at the older date.
49+
stale = _bars("SPY", [datetime(2025, 12, 31)])
50+
manager._save_combined(stale)
51+
52+
manager._regenerate_combined()
53+
54+
combined = pl.read_parquet(tmp_path / "etf_universe.parquet")
55+
assert combined["timestamp"].max() == datetime(2026, 8, 20), (
56+
"combined file was not rebuilt from the ticker partitions"
57+
)
58+
assert combined.height == 2
59+
60+
def test_picks_up_a_symbol_absent_from_the_aggregate(self, tmp_path):
61+
"""A partition missing from the aggregate must appear after regeneration."""
62+
config = ETFConfig(
63+
storage_path=tmp_path,
64+
tickers={"equity": {"symbols": ["SPY", "QQQ"]}},
65+
)
66+
manager = ETFDataManager(config)
67+
68+
ts = [datetime(2026, 8, 20)]
69+
manager._save_ticker(_bars("SPY", ts), "SPY")
70+
manager._save_ticker(_bars("QQQ", ts), "QQQ")
71+
72+
# Aggregate predates QQQ being added to the universe.
73+
manager._save_combined(_bars("SPY", ts))
74+
75+
manager._regenerate_combined()
76+
77+
combined = pl.read_parquet(tmp_path / "etf_universe.parquet")
78+
assert set(combined["symbol"].unique()) == {"SPY", "QQQ"}

0 commit comments

Comments
 (0)