Skip to content

Commit a213df5

Browse files
stefan-jansenStefan Jansen
andauthored
fix(polymarket): migrate trade history to Data API v2 (#133)
* fix(polymarket): migrate trades to Data API v2 cursors * fix(deps): patch GitPython used by documentation qualification * fix(polymarket): preserve trade IDs across v2 numeric encoding * fix(ci): adopt reviewed Data 0.3 Python qualification scope * fix(deps): exclude vulnerable filelock releases * test(rate-limit): verify countdown independently of scheduler latency * test(rate-limit): check blocking wait without host timing assumptions --------- Co-authored-by: Stefan Jansen <[email protected]>
1 parent db9cda7 commit a213df5

10 files changed

Lines changed: 636 additions & 188 deletions

File tree

‎.github/workflows/ecosystem.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
qualification:
1818
permissions:
1919
contents: read
20-
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@066dc737a879faea51a5b8a900315441712ea89a # 2026-09-25 policy snapshot
20+
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@322df8513e6f7dceae8eab9d9697b3428ee62af6 # 2026-10-25 scoped policy
2121
with:
2222
import-package: ml4t.data
23-
prerelease-exception: python-315-polars
23+
prerelease-exception: python-315-data-databento-030

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ jobs:
6464
needs: preflight
6565
permissions:
6666
contents: read
67-
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@066dc737a879faea51a5b8a900315441712ea89a # 2026-09-25 policy snapshot
67+
uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@322df8513e6f7dceae8eab9d9697b3428ee62af6 # 2026-10-25 scoped policy
6868
with:
6969
import-package: ml4t.data
70-
prerelease-exception: python-315-polars
70+
prerelease-exception: python-315-data-databento-030
7171

7272
compatibility:
7373
name: Release Compatibility

‎docs/providers/polymarket.md‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,10 +221,16 @@ prices = provider.fetch_candles(
221221
## Trade History
222222

223223
`fetch_trades(condition_id, start=None, end=None)` reads the data API's
224-
`GET /trades?market=<condition id>`, which returns taker trades newest first, takes inclusive
225-
`start`/`end` bounds in Unix seconds, serves at most 10,000 trades per request and rejects offsets
226-
above 10,000. Beyond that depth the provider moves `end` to the oldest second seen and continues,
227-
skipping the trades of that second it already returned.
224+
`GET /v2/trades?condition=<condition id>`, which returns taker trades newest first in a
225+
`data`/`pagination` envelope. The provider follows opaque cursors, including after short or empty
226+
pages, and clamps `page_size` to 1-1000. Market history is limited to the server's rolling
227+
three-year window. The server ignores `start`/`end` for market requests, so the provider enforces
228+
these inclusive Unix-second bounds locally; older requested history is not guaranteed available.
229+
`max_pages` limits requests for sampling and does not guarantee full coverage.
230+
231+
`iter_trades()` yields raw snake_case v2 dictionaries. `fetch_trades()` preserves the normalized
232+
columns below and the existing trade identifiers. The CLOB price-history and Gamma market APIs
233+
remain separate from this [Data API migration](https://docs.polymarket.com/migrate/data-api-v1-to-v2).
228234

229235
| Column | Meaning |
230236
|--------|---------|

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ dependencies = [
9696
# Utilities
9797
"structlog>=23.0.0",
9898
"platformdirs>=4.0.0",
99-
"filelock>=3.19.1",
99+
"filelock>=4.2.1",
100100
"rich>=13.0.0",
101101
"aiofiles>=23.0.0", # Async file I/O for async storage backends
102102
# Exchange calendars for session-aware operations

‎src/ml4t/data/providers/polymarket.py‎

Lines changed: 100 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,8 @@ class PolymarketProvider(BaseProvider):
174174
# GET /prices-history rejects a startTs..endTs span longer than 15 days (HTTP 400).
175175
PRICE_HISTORY_MAX_WINDOW_SECONDS: ClassVar[int] = 15 * 86_400
176176

177-
# GET /trades serves up to 10000 trades per request and rejects offset > 10000 (HTTP 400).
178-
TRADES_MAX_LIMIT: ClassVar[int] = 10_000
179-
TRADES_MAX_OFFSET: ClassVar[int] = 10_000
177+
# Data API v2 trades use opaque cursors and at most 1000 rows per page.
178+
TRADES_MAX_LIMIT: ClassVar[int] = 1_000
180179
TRADES_PAGE_SIZE: ClassVar[int] = 1_000
181180

182181
# Map common frequency names to Polymarket interval values
@@ -1494,10 +1493,23 @@ def fetch_candles(
14941493

14951494
@staticmethod
14961495
def _trade_key(trade: dict[str, Any]) -> str:
1497-
fields = ("transactionHash", "asset", "proxyWallet", "side", "size", "price", "timestamp")
1498-
return hashlib.sha1(
1499-
"|".join(str(trade.get(field)) for field in fields).encode(), usedforsecurity=False
1500-
).hexdigest()
1496+
fields = (
1497+
("transaction_hash", "transactionHash"),
1498+
("token_id", "asset"),
1499+
("proxy_wallet", "proxyWallet"),
1500+
("side", "side"),
1501+
("size", "size"),
1502+
("price", "price"),
1503+
("timestamp", "timestamp"),
1504+
)
1505+
values = []
1506+
for field, legacy in fields:
1507+
value = trade.get(field, trade.get(legacy))
1508+
# V1 encodes whole quantities as integers; v2 uses floats for the same trades.
1509+
if field in ("size", "price") and isinstance(value, float) and value.is_integer():
1510+
value = int(value)
1511+
values.append(str(value))
1512+
return hashlib.sha1("|".join(values).encode(), usedforsecurity=False).hexdigest()
15011513

15021514
def iter_trades(
15031515
self,
@@ -1509,118 +1521,101 @@ def iter_trades(
15091521
) -> Iterator[dict[str, Any]]:
15101522
"""Iterate over the taker trades of one market, newest first.
15111523
1512-
``GET https://data-api.polymarket.com/trades`` returns trades newest first, takes
1513-
inclusive ``start``/``end`` bounds in Unix seconds, serves up to 10000 trades per
1514-
request and rejects ``offset`` above 10000. To reach older trades this method pages
1515-
by offset until the cap, then moves ``end`` to the oldest second seen and starts
1516-
again at offset 0, dropping the trades of that boundary second it already yielded.
1517-
Records carry no trade id; a trade is identified by its transaction hash, token,
1518-
wallet, side, size, price and time, so two otherwise identical fills in one
1519-
transaction are yielded once.
1524+
``GET /v2/trades`` returns snake_case records in a cursor-paginated envelope.
1525+
Market history is limited to the server's rolling three-year window. The server
1526+
ignores time bounds for market requests, so inclusive bounds are also enforced
1527+
locally. Short or empty pages can still carry a continuation cursor.
15201528
15211529
Args:
15221530
condition_id: Market condition id (``ticker`` of ``fetch_markets``).
15231531
start: Keep trades at or after this time (Unix seconds, ISO string, date or
15241532
datetime; naive is UTC).
15251533
end: Keep trades at or before this time.
1526-
page_size: Trades per request (1-10000).
1527-
max_pages: Optional cap on the number of requests, for sampling.
1534+
page_size: First-page size (clamped to 1-1000).
1535+
max_pages: Optional positive request cap for sampling, not full coverage.
15281536
15291537
Yields:
1530-
Raw data-api trade dictionaries.
1538+
Raw snake_case Data API v2 trade dictionaries. Identical fills with the same
1539+
transaction, token, wallet, side, size, price and time are yielded once.
15311540
15321541
Raises:
1533-
DataValidationError: The server repeated a page (offset ignored) or one second
1534-
holds more trades than one offset window can reach.
1542+
DataValidationError: Malformed records/envelope or a repeated cursor.
15351543
"""
1544+
if max_pages is not None and max_pages < 1:
1545+
raise ValueError("max_pages must be positive")
15361546
limit = max(1, min(int(page_size), self.TRADES_MAX_LIMIT))
1537-
start_seconds = to_unix_seconds(start)
1538-
window_end = to_unix_seconds(end)
1547+
start_seconds, end_seconds = to_unix_seconds(start), to_unix_seconds(end)
1548+
if start_seconds is not None and end_seconds is not None and start_seconds > end_seconds:
1549+
raise ValueError("start must be at or before end")
15391550
resource = f"trades for {condition_id}"
1540-
boundary_keys: set[str] = set()
1551+
params: dict[str, Any] = {"condition": condition_id, "limit": limit}
1552+
if start_seconds is not None:
1553+
params["start"] = start_seconds
1554+
if end_seconds is not None:
1555+
params["end"] = end_seconds
1556+
seen_cursors: set[str] = set()
1557+
seen_keys: set[str] = set()
15411558
pages = 0
15421559
while True:
1543-
offset = 0
1544-
window_keys = set(boundary_keys)
1545-
oldest: int | None = None
1546-
oldest_keys: set[str] = set()
1547-
previous_page: list[str] | None = None
1548-
while True:
1549-
params: dict[str, Any] = {"market": condition_id, "limit": limit, "offset": offset}
1550-
if start_seconds is not None:
1551-
params["start"] = start_seconds
1552-
if window_end is not None:
1553-
params["end"] = window_end
1554-
response = self._request(
1555-
f"{self.DATA_API_URL}/trades", resource=resource, params=params
1560+
response = self._request(
1561+
f"{self.DATA_API_URL}/v2/trades", resource=resource, params=params
1562+
)
1563+
try:
1564+
payload = response.json()
1565+
except ValueError as err:
1566+
raise DataValidationError(
1567+
provider="polymarket", message=f"Malformed JSON response for {resource}"
1568+
) from err
1569+
pagination = payload.get("pagination") if isinstance(payload, dict) else None
1570+
trades = payload.get("data") if isinstance(payload, dict) else None
1571+
if not isinstance(trades, list) or not isinstance(pagination, dict):
1572+
raise DataValidationError(
1573+
provider="polymarket", message=f"Expected v2 data/pagination for {resource}"
15561574
)
1575+
has_more, cursor = pagination.get("has_more"), pagination.get("next_cursor")
1576+
if (
1577+
not isinstance(has_more, bool)
1578+
or (has_more and (not isinstance(cursor, str) or not cursor))
1579+
or (not has_more and cursor is not None)
1580+
):
1581+
raise DataValidationError(
1582+
provider="polymarket", message=f"Malformed v2 pagination for {resource}"
1583+
)
1584+
if cursor is not None and cursor in seen_cursors:
1585+
raise DataValidationError(
1586+
provider="polymarket", message=f"Trade cursor repeated for {resource}"
1587+
)
1588+
pages += 1
1589+
newest: int | None = None
1590+
for trade in trades:
15571591
try:
1558-
trades = response.json()
1559-
except ValueError as err:
1560-
raise DataValidationError(
1561-
provider="polymarket", message=f"Malformed JSON response for {resource}"
1562-
) from err
1563-
if not isinstance(trades, list):
1564-
raise DataValidationError(
1565-
provider="polymarket",
1566-
message=f"Expected a JSON list for {resource}, got {type(trades).__name__}",
1567-
)
1568-
pages += 1
1569-
page_keys: list[str] = []
1570-
for trade in trades:
15711592
if not isinstance(trade, dict) or isinstance(trade.get("timestamp"), bool):
1572-
raise DataValidationError(
1573-
provider="polymarket",
1574-
message=f"Malformed trade entry for {resource}: {trade!r}",
1575-
field="trades",
1576-
)
1577-
try:
1578-
seconds = int(trade["timestamp"])
1579-
except (KeyError, TypeError, ValueError) as err:
1580-
raise DataValidationError(
1581-
provider="polymarket",
1582-
message=f"Trade without a timestamp for {resource}: {trade!r}",
1583-
field="timestamp",
1584-
) from err
1585-
key = self._trade_key(trade)
1586-
page_keys.append(key)
1587-
if oldest is None or seconds < oldest:
1588-
oldest, oldest_keys = seconds, {key}
1589-
elif seconds == oldest:
1590-
oldest_keys.add(key)
1591-
if key in window_keys:
1592-
continue
1593-
window_keys.add(key)
1594-
if start_seconds is not None and seconds < start_seconds:
1595-
continue
1596-
if window_end is not None and seconds > window_end:
1597-
continue
1598-
yield trade
1599-
if len(trades) < limit or (max_pages is not None and pages >= max_pages):
1600-
return
1601-
if offset > 0 and page_keys == previous_page:
1593+
raise ValueError("trade must contain an integer timestamp")
1594+
seconds = int(trade["timestamp"])
1595+
if isinstance(trade["timestamp"], float) and seconds != trade["timestamp"]:
1596+
raise ValueError("non-integer timestamp")
1597+
except (KeyError, TypeError, ValueError) as err:
16021598
raise DataValidationError(
16031599
provider="polymarket",
1604-
message=f"Trade page repeated for {resource}; the server ignored offset",
1605-
field="offset",
1606-
value=offset,
1607-
)
1608-
previous_page = page_keys
1609-
offset += len(trades)
1610-
if offset > self.TRADES_MAX_OFFSET:
1611-
break
1612-
if oldest is None or (window_end is not None and oldest >= window_end):
1613-
raise DataValidationError(
1614-
provider="polymarket",
1615-
message=(
1616-
f"More than {self.TRADES_MAX_OFFSET + limit} trades at second "
1617-
f"{window_end} for {resource}; cannot page past it"
1618-
),
1619-
field="end",
1620-
value=window_end,
1621-
)
1622-
window_end = oldest
1623-
boundary_keys = oldest_keys
1600+
message=f"Malformed trade timestamp for {resource}: {trade!r}",
1601+
field="timestamp",
1602+
) from err
1603+
newest = seconds if newest is None else max(newest, seconds)
1604+
key = self._trade_key(trade)
1605+
if key in seen_keys:
1606+
continue
1607+
seen_keys.add(key)
1608+
if start_seconds is not None and seconds < start_seconds:
1609+
continue
1610+
if end_seconds is not None and seconds > end_seconds:
1611+
continue
1612+
yield trade
1613+
if not has_more or (max_pages is not None and pages >= max_pages):
1614+
return
1615+
if start_seconds is not None and newest is not None and newest < start_seconds:
1616+
return
1617+
seen_cursors.add(cursor)
1618+
params["cursor"] = cursor
16241619

16251620
def fetch_trades(
16261621
self,
@@ -1667,13 +1662,13 @@ def fetch_trades(
16671662

16681663
def _normalize_trade(self, trade: dict[str, Any], condition_id: str) -> tuple[Any, ...]:
16691664
try:
1670-
outcome_index = trade["outcomeIndex"]
1665+
outcome_index = trade["outcome_index"]
16711666
outcome_price = float(trade["price"])
16721667
size = float(trade["size"])
16731668
side = str(trade["side"]).upper()
16741669
timestamp = datetime.fromtimestamp(int(trade["timestamp"]), UTC)
16751670
if isinstance(outcome_index, bool) or outcome_index not in (0, 1):
1676-
raise ValueError(f"outcomeIndex {outcome_index!r} is not 0 or 1")
1671+
raise ValueError(f"outcome_index {outcome_index!r} is not 0 or 1")
16771672
if side not in ("BUY", "SELL"):
16781673
raise ValueError(f"side {side!r} is not BUY or SELL")
16791674
except (KeyError, TypeError, ValueError) as err:
@@ -1687,7 +1682,7 @@ def _normalize_trade(self, trade: dict[str, Any], condition_id: str) -> tuple[An
16871682
taker_yes = first_outcome == (side == "BUY")
16881683
return (
16891684
self._trade_key(trade),
1690-
trade.get("conditionId") or condition_id,
1685+
trade.get("condition_id") or condition_id,
16911686
timestamp,
16921687
yes_price,
16931688
size,
@@ -1697,9 +1692,9 @@ def _normalize_trade(self, trade: dict[str, Any], condition_id: str) -> tuple[An
16971692
int(outcome_index),
16981693
side,
16991694
outcome_price,
1700-
None if trade.get("asset") is None else str(trade["asset"]),
1701-
trade.get("proxyWallet"),
1702-
trade.get("transactionHash"),
1695+
None if trade.get("token_id") is None else str(trade["token_id"]),
1696+
trade.get("proxy_wallet"),
1697+
trade.get("transaction_hash"),
17031698
)
17041699

17051700
def close(self) -> None:

0 commit comments

Comments
 (0)