|
| 1 | +"""Regression tests for combined-file regeneration.""" |
| 2 | + |
| 3 | +from datetime import datetime |
| 4 | + |
| 5 | +import polars as pl |
| 6 | + |
| 7 | +from ml4t.data.etfs.downloader import ETFConfig, ETFDataManager |
| 8 | + |
| 9 | + |
| 10 | +def _bars(symbol: str, timestamps: list[datetime]) -> pl.DataFrame: |
| 11 | + """Minimal OHLCV frame for one symbol.""" |
| 12 | + n = len(timestamps) |
| 13 | + return pl.DataFrame( |
| 14 | + { |
| 15 | + "timestamp": timestamps, |
| 16 | + "symbol": [symbol] * n, |
| 17 | + "open": [1.0] * n, |
| 18 | + "high": [1.0] * n, |
| 19 | + "low": [1.0] * n, |
| 20 | + "close": [1.0] * n, |
| 21 | + "volume": [100.0] * n, |
| 22 | + }, |
| 23 | + schema_overrides={"timestamp": pl.Datetime}, |
| 24 | + ) |
| 25 | + |
| 26 | + |
| 27 | +class TestRegenerateCombined: |
| 28 | + """Tests for ETFDataManager._regenerate_combined.""" |
| 29 | + |
| 30 | + def test_rebuilds_from_partitions_when_aggregate_is_stale(self, tmp_path): |
| 31 | + """A stale combined file must not survive regeneration. |
| 32 | +
|
| 33 | + ``update()`` writes fresh bars to the per-ticker partitions and then |
| 34 | + calls ``_regenerate_combined()``. If regeneration reads the existing |
| 35 | + aggregate instead of the partitions, the stale file is rewritten |
| 36 | + unchanged and every downstream reader silently sees old data. |
| 37 | + """ |
| 38 | + config = ETFConfig( |
| 39 | + storage_path=tmp_path, |
| 40 | + tickers={"equity": {"symbols": ["SPY"]}}, |
| 41 | + ) |
| 42 | + manager = ETFDataManager(config) |
| 43 | + |
| 44 | + # Partition holds bars through the newer date. |
| 45 | + fresh = _bars("SPY", [datetime(2025, 12, 31), datetime(2026, 8, 20)]) |
| 46 | + manager._save_ticker(fresh, "SPY") |
| 47 | + |
| 48 | + # Aggregate is stale: it stops at the older date. |
| 49 | + stale = _bars("SPY", [datetime(2025, 12, 31)]) |
| 50 | + manager._save_combined(stale) |
| 51 | + |
| 52 | + manager._regenerate_combined() |
| 53 | + |
| 54 | + combined = pl.read_parquet(tmp_path / "etf_universe.parquet") |
| 55 | + assert combined["timestamp"].max() == datetime(2026, 8, 20), ( |
| 56 | + "combined file was not rebuilt from the ticker partitions" |
| 57 | + ) |
| 58 | + assert combined.height == 2 |
| 59 | + |
| 60 | + def test_picks_up_a_symbol_absent_from_the_aggregate(self, tmp_path): |
| 61 | + """A partition missing from the aggregate must appear after regeneration.""" |
| 62 | + config = ETFConfig( |
| 63 | + storage_path=tmp_path, |
| 64 | + tickers={"equity": {"symbols": ["SPY", "QQQ"]}}, |
| 65 | + ) |
| 66 | + manager = ETFDataManager(config) |
| 67 | + |
| 68 | + ts = [datetime(2026, 8, 20)] |
| 69 | + manager._save_ticker(_bars("SPY", ts), "SPY") |
| 70 | + manager._save_ticker(_bars("QQQ", ts), "QQQ") |
| 71 | + |
| 72 | + # Aggregate predates QQQ being added to the universe. |
| 73 | + manager._save_combined(_bars("SPY", ts)) |
| 74 | + |
| 75 | + manager._regenerate_combined() |
| 76 | + |
| 77 | + combined = pl.read_parquet(tmp_path / "etf_universe.parquet") |
| 78 | + assert set(combined["symbol"].unique()) == {"SPY", "QQQ"} |
0 commit comments