Skip to content

Commit c7b9dfa

Browse files
fix: reuse recorded provider during incremental acquisition
Closes #95. Verified real-provider regression, both storage backends and all applicable exact-head hosted checks.
1 parent 7571793 commit c7b9dfa

2 files changed

Lines changed: 50 additions & 0 deletions

File tree

‎src/ml4t/data/managers/storage_manager.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,8 @@ def update(
508508
f"Stored metadata provider for '{key}' must be a string, "
509509
f"got {type(existing_provider).__name__}"
510510
)
511+
if not provider and existing_provider not in (None, "", "auto"):
512+
provider = existing_provider
511513

512514
# Get the date range from existing data
513515
_, last_timestamp = timestamp_bounds(existing_df)
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""A recorded provider must control acquisition, not only rewritten metadata."""
2+
3+
import pytest
4+
from polars.testing import assert_frame_equal
5+
6+
from ml4t.data import DataManager
7+
from ml4t.data.providers.synthetic import SyntheticProvider
8+
from ml4t.data.storage.config import StorageConfig
9+
from ml4t.data.storage.flat import FlatStorage
10+
from ml4t.data.storage.hive import HiveStorage
11+
12+
13+
@pytest.fixture(params=[FlatStorage, HiveStorage])
14+
def stored_provider(request, tmp_path):
15+
storage = request.param(StorageConfig(base_path=tmp_path))
16+
with SyntheticProvider() as provider:
17+
frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-07", "daily")
18+
storage.write(frame, "equities/daily/SPY", {"provider": "synthetic"})
19+
return storage, frame
20+
21+
22+
def test_real_incremental_fetch_reuses_the_recorded_adapter_without_a_route(stored_provider):
23+
storage, frame = stored_provider
24+
with DataManager(storage=storage) as manager:
25+
key = manager.update("SPY", fill_gaps=False)
26+
assert key == "equities/daily/SPY"
27+
assert manager.get_metadata_for_key(key)["provider"] == "synthetic"
28+
assert storage.read(key).collect().height > frame.height
29+
30+
31+
def test_explicit_provider_still_overrides_the_recorded_adapter(stored_provider):
32+
storage, frame = stored_provider
33+
storage.write(frame, "equities/daily/SPY", {"provider": "no-such-recorded-provider"})
34+
with DataManager(storage=storage) as manager:
35+
key = manager.update("SPY", fill_gaps=False, provider="synthetic")
36+
assert manager.get_metadata_for_key(key)["provider"] == "synthetic"
37+
38+
39+
def test_invalid_explicit_adapter_does_not_fall_back_to_the_recorded_one(stored_provider):
40+
storage, frame = stored_provider
41+
before = storage.get_metadata("equities/daily/SPY")
42+
with (
43+
DataManager(storage=storage) as manager,
44+
pytest.raises(ValueError, match="not.*available|not found|Unknown|unknown"),
45+
):
46+
manager.update("SPY", fill_gaps=False, provider="not-a-provider")
47+
assert storage.get_metadata("equities/daily/SPY") == before
48+
assert_frame_equal(storage.read("equities/daily/SPY").collect(), frame)

0 commit comments

Comments
 (0)