Skip to content

Commit e09a0e8

Browse files
fix(massive): follow aggregate next_url pages
OHLCV responses stopped after the first page. Requests larger than the 50,000-row limit silently returned only the earliest bars. Co-authored-by: quantum-like <[email protected]>
1 parent 55f6523 commit e09a0e8

2 files changed

Lines changed: 46 additions & 1 deletion

File tree

‎src/ml4t/data/providers/polygon.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ def _fetch_raw_data(
338338
)
339339
raise ProviderError(provider=self.name, message=f"API error: {error_msg}")
340340

341-
return data
341+
return self._collect_aggregate_pages(data)
342342

343343
except (
344344
AuthenticationError,
@@ -351,6 +351,27 @@ def _fetch_raw_data(
351351
except Exception as err:
352352
raise NetworkError(provider=self.name, message=f"Request failed: {endpoint}") from err
353353

354+
def _collect_aggregate_pages(self, data: dict[str, Any]) -> dict[str, Any]:
355+
"""Follow aggregate ``next_url`` cursors so a 50,000-row page is not the whole result."""
356+
next_url = data.get("next_url")
357+
if not isinstance(next_url, str) or not next_url:
358+
return data
359+
raw_results = data.get("results") or []
360+
results = (
361+
[item for item in raw_results if isinstance(item, dict)]
362+
if isinstance(raw_results, list)
363+
else []
364+
)
365+
seen: set[str] = set()
366+
while isinstance(next_url, str) and next_url and next_url not in seen:
367+
seen.add(next_url)
368+
page = self._get_json(next_url, {})
369+
results.extend(item for item in page.get("results", []) if isinstance(item, dict))
370+
next_url = page.get("next_url")
371+
if seen:
372+
return {**data, "results": results}
373+
return data
374+
354375
def _transform_data(self, raw_data: dict[str, Any], symbol: str) -> pl.DataFrame:
355376
"""Transform raw API response to Polars DataFrame."""
356377
if not raw_data.get("results"):

‎tests/test_polygon_provider.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,30 @@ def test_api_parameters(self):
386386
assert params.get("limit") == 50000
387387
assert params.get("apiKey") == "test_key"
388388

389+
def test_fetch_raw_data_follows_next_url(self):
390+
"""Aggregate responses continue through next_url instead of stopping at one page."""
391+
provider = MassiveProvider(api_key="test_key")
392+
first = MagicMock()
393+
first.status_code = 200
394+
first.json.return_value = {
395+
"status": "OK",
396+
"results": [{"t": 1, "o": 1.0, "h": 1.0, "l": 1.0, "c": 1.0, "v": 1.0}],
397+
"next_url": "https://api.massive.com/v2/aggs/next?cursor=abc",
398+
}
399+
second = MagicMock()
400+
second.status_code = 200
401+
second.json.return_value = {
402+
"status": "OK",
403+
"results": [{"t": 2, "o": 2.0, "h": 2.0, "l": 2.0, "c": 2.0, "v": 2.0}],
404+
}
405+
406+
with patch.object(provider.session, "get", side_effect=[first, second]) as mock_get:
407+
data = provider._fetch_raw_data("AAPL", "2024-01-01", "2024-03-01", "daily")
408+
409+
assert [row["t"] for row in data["results"]] == [1, 2]
410+
assert mock_get.call_count == 2
411+
assert mock_get.call_args_list[1].kwargs["params"]["apiKey"] == "test_key"
412+
389413

390414
class TestMassiveAssetClassRouting:
391415
"""Tests for Massive endpoint routing across asset classes."""

0 commit comments

Comments
 (0)