Skip to content

Commit f19f132

Browse files
committed
feat: per-key request accounting and upstream signal capture
Asked whether remaining quota could be read from the responses. Measured against the live endpoint: it cannot. A successful 200 carries no rate-limit headers at all, and the console exposes no usage/balance/limits route. What a 200 actually carries (whole header list, 2026-10-18): CF-RAY, Cache-Control, Cf-Placement, Connection, Content-Type, Date, Server, Transfer-Encoding, x-opencode-log-id, x-zen-model No x-ratelimit-limit / -remaining / -reset, no retry-after, no quota. Console probes: /api/user is the only authenticated route (id, email, role); /api/usage, /api/zen/balance, /api/credits, /api/limits, /zen/v1/usage and the rest all 404. Upstream has an open feature request for precisely this (anomalyco/opencode#55816, 2026-10-15). So the router keeps its own per-key counters, which is the practical stand-in: - requests_since_reset - requests sent since that key last answered 429. This is the number you actually want when deciding whether to park a key, since upstream exposes no remaining figure. Rolls to 0 on every 429. - requests_total - lifetime count for the connection. Keeps climbing on a 429, because that request was still sent upstream. - last_status, last_ok_in, last_429_in with the last Retry-After value. - Captures the two headers upstream does send: x-opencode-log-id (the trace id support asks for) and x-zen-model (the model upstream actually served, which makes a silent substitution visible). Shown on the provider page per connection, alongside the cooling badge, with a note that these are local counters rather than upstream quota. /admin/api/ providers/<id> gained a usage array next to cooling. Tests: 104 passing (was 101).
1 parent dcbc4aa commit f19f132

4 files changed

Lines changed: 214 additions & 4 deletions

File tree

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,4 +185,12 @@ Tests do not send real requests to Gemini (`requests.post` is mocked).
185185
is always flagged — `X-Samet-Router: stale-response` and a `samet_router` block in the body with
186186
the original timestamp, age and `retry_after`. With nothing cached the router returns an honest
187187
429 carrying `Retry-After`.
188+
- **Quota visibility is local, not upstream.** Measured 2026-10-18: a successful 200 carries no
189+
rate-limit headers at all — the only upstream signals are `x-opencode-log-id` (trace id, quote it
190+
to support) and `x-zen-model` (the model upstream actually served, so a silent substitution shows
191+
up). The console exposes no usage/balance/limits route; `/api/user` is the only authenticated
192+
endpoint and returns just `id`/`email`/`role`. Upstream has an open feature request for exactly
193+
this (`anomalyco/opencode#55816`). So the router keeps its own per-key counters and shows them on
194+
the provider page: `requests_since_reset` (since that key last answered 429 — the practical stand-in
195+
for remaining quota), `requests_total`, last status, and the two captured upstream headers.
188196
- This is a hobby/startup prototype; add auth + rate limiting + DB for production.

‎app.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,98 @@ def cache_get_response(provider: str, model: str) -> dict | None:
500500
return hit
501501

502502

503+
# ---------------------------------------------------------------------------
504+
# Per-key request accounting
505+
# ---------------------------------------------------------------------------
506+
# Measured 2026-10-18: a successful 200 carries no rate-limit headers at all
507+
# (only cloudflare plumbing plus x-opencode-log-id and x-zen-model), and the
508+
# console exposes no usage/balance/limits endpoint - /api/user is the only
509+
# authenticated route and returns just id/email/role. Upstream published a
510+
# feature request for exactly this (anomalyco/opencode#55816). So remaining
511+
# quota cannot be read from the wire; these counters are the local proxy for it.
512+
513+
_key_usage: dict[str, dict] = {}
514+
USAGE_KEYS = ("requests_since_reset", "requests_total", "last_429_at",
515+
"last_429_retry_after", "last_ok_at", "upstream_log_id",
516+
"upstream_model", "last_status")
517+
518+
519+
def _usage_slot(entry: dict) -> dict:
520+
ident = _cooldown_key(entry)
521+
slot = _key_usage.get(ident)
522+
if slot is None:
523+
slot = _key_usage[ident] = {
524+
"requests_since_reset": 0, "requests_total": 0,
525+
"last_429_at": 0, "last_429_retry_after": 0.0,
526+
"last_ok_at": 0, "upstream_log_id": "", "upstream_model": "",
527+
"last_status": 0,
528+
}
529+
return slot
530+
531+
532+
def note_request_sent(entry: dict) -> None:
533+
slot = _usage_slot(entry)
534+
slot["requests_since_reset"] += 1
535+
slot["requests_total"] += 1
536+
537+
538+
def note_rate_limited(entry: dict, retry_after: float) -> None:
539+
slot = _usage_slot(entry)
540+
slot["last_429_at"] = time.time()
541+
slot["last_429_retry_after"] = retry_after
542+
slot["last_status"] = 429
543+
slot["requests_since_reset"] = 0 # quota window just rolled over
544+
545+
546+
def note_upstream_result(entry: dict, resp) -> None:
547+
"""Record what upstream said, including the two headers it does send."""
548+
slot = _usage_slot(entry)
549+
status = int(getattr(resp, "status_code", 0) or 0)
550+
slot["last_status"] = status
551+
headers = getattr(resp, "headers", None)
552+
if headers:
553+
try:
554+
lower = {str(k).lower(): v for k, v in headers.items()}
555+
except (AttributeError, TypeError):
556+
lower = {}
557+
# trace id is what opencode support asks for
558+
if lower.get("x-opencode-log-id"):
559+
slot["upstream_log_id"] = str(lower["x-opencode-log-id"])
560+
# echoes the model upstream actually served, so a silent substitution shows up
561+
if lower.get("x-zen-model"):
562+
slot["upstream_model"] = str(lower["x-zen-model"])
563+
if status == 200:
564+
slot["last_ok_at"] = time.time()
565+
566+
567+
def key_usage_report(provider: str) -> list[dict]:
568+
"""Per-key accounting for the admin UI."""
569+
with _lock:
570+
data = load_data()
571+
now = time.time()
572+
out = []
573+
for entry in data.get("provider_keys", []):
574+
if not isinstance(entry, dict) or (entry.get("provider") or "") != provider:
575+
continue
576+
ident = _cooldown_key(entry)
577+
slot = _key_usage.get(ident)
578+
row = {
579+
"id": entry.get("id"), "label": entry.get("label"),
580+
"requests_since_reset": (slot or {}).get("requests_since_reset", 0),
581+
"requests_total": (slot or {}).get("requests_total", 0),
582+
"last_status": (slot or {}).get("last_status", 0),
583+
"upstream_log_id": (slot or {}).get("upstream_log_id", ""),
584+
"upstream_model": (slot or {}).get("upstream_model", ""),
585+
"last_429_at": (slot or {}).get("last_429_at", 0),
586+
"last_429_in": (max(0.0, now - (slot or {}).get("last_429_at", 0))
587+
if (slot or {}).get("last_429_at") else 0),
588+
"last_ok_in": (max(0.0, now - (slot or {}).get("last_ok_at", 0))
589+
if (slot or {}).get("last_ok_at") else 0),
590+
}
591+
out.append(row)
592+
return out
593+
594+
503595
def served_from_cache(provider_id: str, upstream_model: str, router_model: str,
504596
cooling: AllKeysCoolingDown, anthropic: bool = False):
505597
"""Serve the last good response while every key is rate-limited.
@@ -3254,6 +3346,7 @@ def _post(url: str):
32543346

32553347
try:
32563348
if protocol == "openai":
3349+
note_request_sent(entry)
32573350
if oc_endpoint and oc_endpoint != "chat":
32583351
resp = _post(f"{base}/{opencode_route_path(oc_endpoint)}")
32593352
# upstream publishes no endpoint map, so on its explicit
@@ -3269,6 +3362,7 @@ def _post(url: str):
32693362
else:
32703363
resp = _post(openai_chat_url(base))
32713364
elif protocol == "anthropic":
3365+
note_request_sent(entry)
32723366
resp = _post(anthropic_messages_url(base))
32733367
else:
32743368
url = gemini_generate_url(base, gemini_model) + f"?key={key}"
@@ -3277,10 +3371,12 @@ def _post(url: str):
32773371
kwargs["headers"] = upstream_headers("gemini", key, merged_extra)
32783372
if proxy:
32793373
kwargs["proxies"] = proxy
3374+
note_request_sent(entry)
32803375
resp = requests.post(url, **kwargs)
32813376
except requests.RequestException as exc:
32823377
last_exc = exc
32833378
continue
3379+
note_upstream_result(entry, resp)
32843380
if resp.status_code == 200:
32853381
_advance_provider_index(protocol, len(pool), provider)
32863382
return entry, resp
@@ -3289,6 +3385,7 @@ def _post(url: str):
32893385
# explicit Retry-After when upstream sent one (OpenCode's daily free
32903386
# quota resets at 00:00 UTC and reports ~19h, far past our 5 min).
32913387
secs = mark_key_cooling(entry, retry_after_seconds(resp))
3388+
note_rate_limited(entry, secs)
32923389
if i < len(pool) - 1:
32933390
_close_quietly(resp)
32943391
last_exc = RuntimeError(f"upstream 429: {resp.text[:300]}")
@@ -3812,6 +3909,7 @@ def _provider_detail(provider_id: str) -> dict:
38123909
"settings": get_provider_settings(data, pid),
38133910
"connections": [_masked_provider_key(k) for k in keys],
38143911
"cooling": cooling_key_report(pid),
3912+
"usage": key_usage_report(pid),
38153913
"models": [{"name": m.get("name", ""),
38163914
"upstream_model": str(m.get("upstream_model") or m.get("gemini_model") or ""),
38173915
"description": m.get("description", ""),

‎templates/provider.html‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ <h3>Available Models</h3>
249249
$('pd-headers-badge').classList.toggle('hidden',
250250
!(d.settings.headers_active && (d.settings.custom_headers || []).length));
251251

252-
renderConnections(d.connections || [], d.cooling || []);
252+
renderConnections(d.connections || [], d.cooling || [], d.usage || []);
253253
renderModels(d.models || [], d.short, d.id);
254254
renderSuggestions(d.suggestions || []);
255255
document.querySelector('.pd-suggest-title').textContent = pdSuggestTitle(d.id);
@@ -261,8 +261,9 @@ <h3>Available Models</h3>
261261
} catch (e) {}
262262
}
263263

264-
function renderConnections(conns, cooling) {
264+
function renderConnections(conns, cooling, usage) {
265265
const cool = new Map((cooling || []).map(c => [c.id, c.retry_after]));
266+
const use = new Map((usage || []).map(u => [u.id, u]));
266267
if (!conns.length) {
267268
$('conn-left').innerHTML = `<div class="conn-empty"><span class="conn-lock">${PD_LOCK}</span> No connections yet</div>`;
268269
return;
@@ -275,11 +276,19 @@ <h3>Available Models</h3>
275276
const mins = left ? Math.ceil(left / 60) : 0;
276277
const badge = left
277278
? `<span class="tag" title="rate limited: skipped until the cooldown ends"
278-
style="background:var(--warn-a,rgba(245,165,36,.14));border-color:rgba(245,165,36,.4);color:#f5a524"
279+
style="background:rgba(245,165,36,.14);border-color:rgba(245,165,36,.4);color:#f5a524"
279280
>cooling ~${mins}m</span>`
280281
: '';
282+
const u = use.get(k.id) || {};
283+
const since = u.requests_since_reset ?? 0;
284+
const total = u.requests_total ?? 0;
285+
const meta = [];
286+
meta.push(`<span title="requests sent since this key last hit a rate limit">since reset <b>${since}</b></span>`);
287+
if (total && total !== since) meta.push(`<span>total ${total}</span>`);
288+
if (u.upstream_model) meta.push(`<span title="model upstream actually served">served ${esc(u.upstream_model)}</span>`);
289+
if (u.upstream_log_id) meta.push(`<span title="upstream trace id (quote to support)">log ${esc(u.upstream_log_id.slice(0, 8))}</span>`);
281290
return `
282-
<div class="conn-item">
291+
<div class="conn-item" style="flex-wrap:wrap">
283292
<span class="conn-dot"${left ? ' style="background:#f5a524"' : ''}></span>
284293
<span class="conn-label">${esc(k.label)}</span>
285294
${isOauth
@@ -288,6 +297,7 @@ <h3>Available Models</h3>
288297
${badge}
289298
${sub ? `<span class="conn-meta">${esc(sub)}</span>` : ''}
290299
<span class="conn-meta">prio ${k.priority ?? 1}</span>
300+
<span class="conn-meta" style="width:100%;display:flex;gap:12px;flex-wrap:wrap;margin-top:2px">${meta.join('')}</span>
291301
<button class="conn-del" onclick="delConnection('${k.id}')">Delete</button>
292302
</div>`;
293303
}).join('');
@@ -296,6 +306,9 @@ <h3>Available Models</h3>
296306
rate limited and being skipped. When every key is cooling, the router replays
297307
the last good response for this provider instead of failing.</p>`;
298308
}
309+
$('conn-left').innerHTML += `<p class="hint" style="margin-top:6px">OpenCode sends no
310+
quota or balance headers, so these counters are the local proxy: <code>since reset</code>
311+
counts requests since this key last answered 429.</p>`;
299312
}
300313
async function delConnection(id) {
301314
await fetch('/admin/api/provider-keys/' + id, { method: 'DELETE' });

‎test_router.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1702,6 +1702,97 @@ def json(self):
17021702
assert r.headers["X-Samet-Router"] == "stale-response"
17031703

17041704

1705+
# ---------------------------------------------------------------------------
1706+
# Per-key request accounting
1707+
# ---------------------------------------------------------------------------
1708+
1709+
1710+
def test_key_usage_counts_and_resets_on_429(tmp_path, monkeypatch):
1711+
c = make_client(tmp_path)
1712+
_reset_circuit()
1713+
router._key_usage.clear()
1714+
c.post("/admin/api/provider-keys", json={"key": "sk-a", "label": "a",
1715+
"provider": "openai"})
1716+
c.post("/admin/api/models", json={"name": "m", "provider": "openai",
1717+
"upstream_model": "gpt-x"})
1718+
1719+
class R200:
1720+
status_code = 200
1721+
text = "ok"
1722+
headers = {"x-opencode-log-id": "abc12345-def",
1723+
"x-zen-model": "gpt-x-real"}
1724+
1725+
def json(self):
1726+
return fake_openai_chat("ok")
1727+
1728+
class R429:
1729+
status_code = 429
1730+
text = "slow"
1731+
headers = {"retry-after": "600"}
1732+
1733+
def json(self):
1734+
return {"error": {"message": "Rate limit exceeded"}}
1735+
1736+
monkeypatch.setattr(router.requests, "post", lambda url, **kw: R200())
1737+
for _ in range(3):
1738+
assert c.post("/v1/chat/completions", json={
1739+
"model": "m", "messages": [{"role": "user", "content": "hi"}]}).status_code == 200
1740+
1741+
usage = c.get("/admin/api/providers/openai").get_json()["usage"]
1742+
row = usage[0]
1743+
assert row["requests_since_reset"] == 3
1744+
assert row["requests_total"] == 3
1745+
assert row["last_status"] == 200
1746+
# the two headers upstream does send are captured
1747+
assert row["upstream_log_id"] == "abc12345-def"
1748+
assert row["upstream_model"] == "gpt-x-real"
1749+
assert row["last_ok_in"] < 5
1750+
1751+
# a 429 rolls the quota window over
1752+
monkeypatch.setattr(router.requests, "post", lambda url, **kw: R429())
1753+
c.post("/v1/chat/completions", json={
1754+
"model": "m", "messages": [{"role": "user", "content": "hi"}]})
1755+
row = c.get("/admin/api/providers/openai").get_json()["usage"][0]
1756+
assert row["requests_since_reset"] == 0
1757+
# total keeps climbing: the 429 request itself was still sent upstream
1758+
assert row["requests_total"] == 4
1759+
assert row["last_status"] == 429
1760+
assert 0 < row["last_429_in"] < 5
1761+
1762+
1763+
def test_key_usage_survives_missing_headers(tmp_path, monkeypatch):
1764+
c = make_client(tmp_path)
1765+
_reset_circuit()
1766+
router._key_usage.clear()
1767+
c.post("/admin/api/provider-keys", json={"key": "sk-a", "provider": "openai"})
1768+
c.post("/admin/api/models", json={"name": "m", "provider": "openai",
1769+
"upstream_model": "gpt-x"})
1770+
1771+
class Bare:
1772+
status_code = 200
1773+
text = "ok"
1774+
1775+
def json(self):
1776+
return fake_openai_chat("ok")
1777+
1778+
monkeypatch.setattr(router.requests, "post", lambda url, **kw: Bare())
1779+
assert c.post("/v1/chat/completions", json={
1780+
"model": "m", "messages": [{"role": "user", "content": "hi"}]}).status_code == 200
1781+
row = c.get("/admin/api/providers/openai").get_json()["usage"][0]
1782+
assert row["requests_total"] == 1
1783+
assert row["upstream_log_id"] == ""
1784+
assert row["upstream_model"] == ""
1785+
1786+
1787+
def test_key_usage_zeroed_for_fresh_keys(tmp_path):
1788+
c = make_client(tmp_path)
1789+
c.post("/admin/api/provider-keys", json={"key": "sk-new", "provider": "openai"})
1790+
row = c.get("/admin/api/providers/openai").get_json()["usage"][0]
1791+
assert row["requests_since_reset"] == 0
1792+
assert row["requests_total"] == 0
1793+
assert row["last_429_in"] == 0
1794+
1795+
17051796
def test_retry_after_parsing(tmp_path):
17061797
class R:
17071798
def __init__(self, h):

0 commit comments

Comments
 (0)