Skip to content

Commit 25be299

Browse files
anandgupta42claude
andcommitted
fix: [#838] webfetch 404 cache — extend TTL + normalize URLs across tracking-param variations
Telemetry-2026-05-21 showed 486 residual webfetch 404 errors (down 78% from March's 2,222 — the existing failure cache is doing its job). The remaining cases mostly escape because of two gaps: 1. TTL was 5 minutes. Real sessions often run 30+ minutes; the same dead URL re-hit the network after the first 5-minute window. 2. Cache key was the raw URL string. LLM-generated URLs frequently include tracking params (utm_source, fbclid, gclid, etc.) or fragments — each variation got its own cache entry, so a known-bad URL with different tracking params re-hit the network on every call. Changes - Extend FAILURE_CACHE_TTL from 5 minutes to 30 minutes. URLs that 404 rarely self-heal within a session; the longer window catches the slow-burn case without measurably increasing the risk of caching a transient failure. - Add `normalizeUrlForCache()` and use it for every cache read/write. Strips universally-tracking params (utm_*, ref*, fbclid, gclid, msclkid, yclid, twclid, li_fat_id, mc_cid, mc_eid, _ga, _gl, igshid, mibextid, __cf_chl_*), strips URL fragments, lowercases the hostname, and sorts remaining params. Conservative list — only params known to be tracking-only are stripped, so functional params (page, id, q, etc.) survive untouched. - Export `_resetFailureCache` and `_failureCacheSize` for tests. Tests New test/tool/webfetch-cache.test.ts (15 tests): - strips utm_*, fbclid, gclid, msclkid family - preserves functional params (page, id, q, etc.) - sorts params for stable cache key - strips fragments - lowercases hostname (case-insensitive per RFC 3986) - preserves path case (paths ARE case-sensitive per HTTP) - collapses Cloudflare __cf_chl_* challenge tokens - 6 variations of the same logical URL collapse to one cache key - different functional params and paths do NOT collapse - invalid URL strings pass through unchanged Validation - `tsgo --noEmit`: pass. - `bun test test/tool/webfetch-cache.test.ts test/tool/webfetch.test.ts`: 19 pass / 0 fail (15 new + 4 existing). - Marker check: pass — altimate_change markers applied. Expected telemetry impact - Further reduction in residual webfetch 404 errors (currently 486 in 14d). Specifically helps long sessions revisiting dead doc URLs multiple times, and LLMs pasting the same URL with different tracking suffixes. - No change for first-time fetches of legitimately new URLs. Closes #838 Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 476d234 commit 25be299

2 files changed

Lines changed: 228 additions & 4 deletions

File tree

‎packages/opencode/src/tool/webfetch.ts‎

Lines changed: 92 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,91 @@ const RETRYABLE_STATUSES = new Set([403, 406])
1717

1818
// altimate_change start — session-level URL failure cache (#471)
1919
// Prevents repeated fetches to URLs that already returned 404/410 in this session.
20+
//
21+
// 2026-05-22 follow-up to telemetry-2026-05-21 (486 residual webfetch 404s,
22+
// down from March's 2,222 but still meaningful):
23+
// - Extended TTL from 5 min to 30 min. URLs that 404 rarely self-heal
24+
// within a session, and many sessions run longer than 5 min so the
25+
// short window was letting agents re-hit the same dead URL multiple
26+
// times in one session.
27+
// - Cache key is the URL with tracking-only params (utm_*, ref, fbclid,
28+
// gclid, mc_*, _ga, _gl, igshid, mibextid, __cf_chl_*) stripped, so
29+
// LLM-generated tracking variations of a known-bad URL all hit cache.
30+
// Functional query params (page, id, q, etc.) are preserved.
2031
const failedUrls = new Map<string, { status: number; timestamp: number }>()
21-
const FAILURE_CACHE_TTL = 5 * 60 * 1000 // 5 minutes
32+
const FAILURE_CACHE_TTL = 30 * 60 * 1000 // 30 minutes
33+
34+
/**
35+
* Param names that are universally tracking-only — stripping them never
36+
* changes the document the URL points to. Conservative list; favoured over
37+
* a regex sweep because false positives (e.g. stripping `?page=2`) would
38+
* cause real fetches to incorrectly hit the cache.
39+
*/
40+
const TRACKING_PARAMS = new Set([
41+
"utm_source",
42+
"utm_medium",
43+
"utm_campaign",
44+
"utm_term",
45+
"utm_content",
46+
"utm_id",
47+
"ref",
48+
"ref_src",
49+
"ref_url",
50+
"referrer",
51+
"fbclid",
52+
"gclid",
53+
"dclid",
54+
"msclkid",
55+
"yclid",
56+
"twclid",
57+
"li_fat_id",
58+
"mc_cid",
59+
"mc_eid",
60+
"_ga",
61+
"_gl",
62+
"igshid",
63+
"mibextid",
64+
"__cf_chl_tk",
65+
"__cf_chl_jschl_tk__",
66+
"vero_conv",
67+
"vero_id",
68+
])
69+
70+
function isTrackingParamPrefix(name: string): boolean {
71+
return name.startsWith("__cf_chl_") || name.startsWith("utm_")
72+
}
73+
74+
/**
75+
* Normalize a URL for cache lookup. Strips tracking-only query params, sorts
76+
* the remaining params for stable ordering, and lowercases the host. Returns
77+
* the input unchanged if URL parsing fails.
78+
*/
79+
export function normalizeUrlForCache(url: string): string {
80+
try {
81+
const parsed = new URL(url)
82+
parsed.hostname = parsed.hostname.toLowerCase()
83+
const kept: [string, string][] = []
84+
for (const [k, v] of parsed.searchParams) {
85+
if (TRACKING_PARAMS.has(k) || isTrackingParamPrefix(k)) continue
86+
kept.push([k, v])
87+
}
88+
kept.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
89+
parsed.search = ""
90+
for (const [k, v] of kept) parsed.searchParams.append(k, v)
91+
// Also strip fragment — fragments don't change the fetched resource.
92+
parsed.hash = ""
93+
return parsed.toString()
94+
} catch {
95+
return url
96+
}
97+
}
2298

2399
function isUrlCachedFailure(url: string): { status: number } | null {
24-
const entry = failedUrls.get(url)
100+
const key = normalizeUrlForCache(url)
101+
const entry = failedUrls.get(key)
25102
if (!entry) return null
26103
if (Date.now() - entry.timestamp > FAILURE_CACHE_TTL) {
27-
failedUrls.delete(url)
104+
failedUrls.delete(key)
28105
return null
29106
}
30107
return { status: entry.status }
@@ -34,15 +111,26 @@ const MAX_CACHED_URLS = 500
34111

35112
function cacheUrlFailure(url: string, status: number): void {
36113
if (status === 404 || status === 410 || status === 451) {
114+
const key = normalizeUrlForCache(url)
37115
if (failedUrls.size >= MAX_CACHED_URLS) {
38116
// Evict oldest entry (Map preserves insertion order)
39117
const oldest = failedUrls.keys().next().value
40118
if (oldest) failedUrls.delete(oldest)
41119
}
42-
failedUrls.set(url, { status, timestamp: Date.now() })
120+
failedUrls.set(key, { status, timestamp: Date.now() })
43121
}
44122
}
45123

124+
/** Reset the failure cache (testing only). */
125+
export function _resetFailureCache(): void {
126+
failedUrls.clear()
127+
}
128+
129+
/** Read the current failure cache size (testing only). */
130+
export function _failureCacheSize(): number {
131+
return failedUrls.size
132+
}
133+
46134
/** Strip query string from URL to avoid leaking auth tokens in error messages. */
47135
function sanitizeUrl(url: string): string {
48136
try {
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import { describe, test, expect, beforeEach } from "bun:test"
2+
import {
3+
normalizeUrlForCache,
4+
_resetFailureCache,
5+
_failureCacheSize,
6+
} from "../../src/tool/webfetch"
7+
8+
/**
9+
* Webfetch URL failure cache.
10+
*
11+
* Background: telemetry-2026-05-21 showed 486 residual webfetch 404 errors
12+
* (down 78% from March's 2,222, so the existing cache is working — these
13+
* are the cases that still escape it). The cache previously had a 5-min TTL
14+
* and didn't normalize URLs, so tracking-param variations of a known-bad URL
15+
* missed cache. This file pins the normalization contract that now
16+
* collapses common LLM-generated URL variations onto one cache entry.
17+
*/
18+
19+
beforeEach(() => {
20+
_resetFailureCache()
21+
})
22+
23+
describe("normalizeUrlForCache", () => {
24+
test("returns identical URLs unchanged (apart from sorting empty params)", () => {
25+
expect(normalizeUrlForCache("https://example.com/path")).toBe("https://example.com/path")
26+
})
27+
28+
test("strips utm_* tracking params", () => {
29+
expect(
30+
normalizeUrlForCache("https://example.com/docs?utm_source=twitter&utm_medium=social"),
31+
).toBe("https://example.com/docs")
32+
})
33+
34+
test("strips fbclid, gclid, msclkid, etc.", () => {
35+
expect(normalizeUrlForCache("https://example.com/a?fbclid=abc123")).toBe(
36+
"https://example.com/a",
37+
)
38+
expect(normalizeUrlForCache("https://example.com/a?gclid=xyz")).toBe("https://example.com/a")
39+
expect(normalizeUrlForCache("https://example.com/a?msclkid=qqq")).toBe(
40+
"https://example.com/a",
41+
)
42+
})
43+
44+
test("preserves functional query params", () => {
45+
expect(normalizeUrlForCache("https://example.com/api?page=2&limit=20")).toBe(
46+
"https://example.com/api?limit=20&page=2",
47+
)
48+
expect(normalizeUrlForCache("https://example.com/search?q=hello+world")).toBe(
49+
"https://example.com/search?q=hello+world",
50+
)
51+
})
52+
53+
test("preserves functional params when mixed with tracking params", () => {
54+
expect(
55+
normalizeUrlForCache(
56+
"https://example.com/post?id=42&utm_source=email&utm_campaign=monthly&ref=fred",
57+
),
58+
).toBe("https://example.com/post?id=42")
59+
})
60+
61+
test("sorts query params for stable cache key", () => {
62+
expect(normalizeUrlForCache("https://example.com/x?b=2&a=1")).toBe(
63+
"https://example.com/x?a=1&b=2",
64+
)
65+
expect(normalizeUrlForCache("https://example.com/x?a=1&b=2")).toBe(
66+
"https://example.com/x?a=1&b=2",
67+
)
68+
})
69+
70+
test("strips URL fragments", () => {
71+
expect(normalizeUrlForCache("https://example.com/page#section-3")).toBe(
72+
"https://example.com/page",
73+
)
74+
expect(normalizeUrlForCache("https://example.com/page?id=1#section-3")).toBe(
75+
"https://example.com/page?id=1",
76+
)
77+
})
78+
79+
test("lowercases hostname (case-insensitive per RFC 3986)", () => {
80+
expect(normalizeUrlForCache("https://EXAMPLE.COM/path")).toBe("https://example.com/path")
81+
expect(normalizeUrlForCache("https://Example.Com/Path")).toBe("https://example.com/Path")
82+
})
83+
84+
test("preserves path case (paths ARE case-sensitive per HTTP)", () => {
85+
expect(normalizeUrlForCache("https://example.com/Foo/Bar")).toBe("https://example.com/Foo/Bar")
86+
})
87+
88+
test("collapses Cloudflare challenge tokens (__cf_chl_* prefix)", () => {
89+
expect(
90+
normalizeUrlForCache(
91+
"https://example.com/page?__cf_chl_tk=abc&__cf_chl_jschl_tk__=xyz",
92+
),
93+
).toBe("https://example.com/page")
94+
})
95+
96+
test("returns input verbatim if URL parsing fails", () => {
97+
expect(normalizeUrlForCache("not a url at all")).toBe("not a url at all")
98+
})
99+
100+
test("multiple variations of the same logical URL collapse to one cache key", () => {
101+
const variations = [
102+
"https://docs.example.com/api/v1?utm_source=docs",
103+
"https://docs.example.com/api/v1?utm_source=twitter&fbclid=q",
104+
"https://docs.example.com/api/v1?gclid=z",
105+
"https://docs.example.com/api/v1#anchor",
106+
"https://DOCS.EXAMPLE.COM/api/v1",
107+
"https://docs.example.com/api/v1?utm_campaign=monthly",
108+
]
109+
const normalized = variations.map(normalizeUrlForCache)
110+
const unique = new Set(normalized)
111+
expect(unique.size).toBe(1)
112+
expect(unique.values().next().value).toBe("https://docs.example.com/api/v1")
113+
})
114+
115+
test("different functional params do NOT collapse", () => {
116+
const a = normalizeUrlForCache("https://example.com/a?page=1")
117+
const b = normalizeUrlForCache("https://example.com/a?page=2")
118+
expect(a).not.toBe(b)
119+
})
120+
121+
test("different paths do NOT collapse", () => {
122+
expect(normalizeUrlForCache("https://example.com/v1/api")).not.toBe(
123+
normalizeUrlForCache("https://example.com/v2/api"),
124+
)
125+
})
126+
})
127+
128+
describe("failure cache helpers", () => {
129+
test("_resetFailureCache empties the cache", () => {
130+
expect(_failureCacheSize()).toBe(0)
131+
// Add an entry via direct manipulation isn't exposed; this test just
132+
// exercises the helpers exist and return sane values.
133+
_resetFailureCache()
134+
expect(_failureCacheSize()).toBe(0)
135+
})
136+
})

0 commit comments

Comments
 (0)