Skip to content

Commit f880a9d

Browse files
author
Bertan Ari
committed
test: replace flaky timing-based scaling assertions with deterministic work counter
The two leaked tool-call scaling tests measured wall-clock elapsed time and asserted the 4x-input ratio stayed under 10. On shared CI runners GC pauses and contention breached that even though complexity is linear (observed 14.55 and 10.02). Count characters the parser scans instead: exact, machine-independent, and still ~16x under a reintroduced quadratic prefix re-scan.
1 parent 92517a8 commit f880a9d

1 file changed

Lines changed: 32 additions & 22 deletions

File tree

‎src/api/providers/__tests__/vscode-lm.spec.ts‎

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1983,58 +1983,68 @@ describe("leaked tool-call parser scaling", () => {
19831983
const todo = () =>
19841984
`<in${"voke"} name="update_todo_list"><param${"eter"} name="todos">x</param${"eter"}></in${"voke"}>`
19851985

1986-
/** Median wall-clock cost of `run`, which is far steadier than a single sample under CI noise. */
1987-
const medianMs = (run: () => void): number => {
1988-
const samples: number[] = []
1989-
for (let iter = 0; iter < 5; iter += 1) {
1990-
const start = performance.now()
1986+
/**
1987+
* Characters the parser copies out of the message while scanning. Wall-clock timing flaked on
1988+
* shared CI runners, so work performed is counted instead: it is exact and machine-independent.
1989+
*/
1990+
const charactersScanned = (run: () => void): number => {
1991+
const originalSlice = String.prototype.slice
1992+
let scanned = 0
1993+
String.prototype.slice = function (this: string, start?: number, end?: number): string {
1994+
const piece = originalSlice.call(this, start, end)
1995+
scanned += piece.length
1996+
return piece
1997+
}
1998+
try {
19911999
run()
1992-
samples.push(performance.now() - start)
2000+
} finally {
2001+
String.prototype.slice = originalSlice
19932002
}
1994-
return samples.sort((left, right) => left - right)[2]
2003+
return scanned
19952004
}
19962005

19972006
/**
1998-
* Cost at 4x input over cost at 1x. Linear work lands near 4; the quadratic scans this pins
1999-
* landed near 16 or worse. Absolute timings are machine-dependent, so only the ratio is asserted.
2007+
* Work at 4x input over work at 1x. Linear scanning lands near 4; the quadratic prefix re-scan
2008+
* this pins landed near 16. Only the ratio is asserted, never an absolute count.
20002009
*/
20012010
const growthFactor = (build: (size: number) => string, baseSize: number, run: (input: string) => void): number => {
20022011
const small = build(baseSize)
20032012
const large = build(baseSize * 4)
2004-
// Warm both paths so JIT compilation is not charged to the first measurement.
2005-
run(small)
2006-
run(large)
20072013
return (
2008-
medianMs(() => run(large)) /
2014+
charactersScanned(() => run(large)) /
20092015
Math.max(
2010-
medianMs(() => run(small)),
2011-
0.001,
2016+
charactersScanned(() => run(small)),
2017+
1,
20122018
)
20132019
)
20142020
}
20152021

2016-
it("scans ordinary wrapped output in time linear in message length", () => {
2022+
/** Linear work must grow with the input, so a collapsed ratio near 1 fails too. */
2023+
const expectLinearGrowth = (growth: number) => {
2024+
expect(growth).toBeGreaterThan(3)
2025+
expect(growth).toBeLessThan(6)
2026+
}
2027+
2028+
it("scans ordinary wrapped output doing work linear in message length", () => {
20172029
const build = (count: number) => `<function${"_calls"}>\n${`${todo()}\n`.repeat(count)}</function${"_calls"}>\n`
2018-
const growth = growthFactor(build, 150, (input) => extractLeakedToolCalls(input, tools))
20192030

2020-
// Re-splitting the prefix per candidate made this ~16x; 10 leaves room for CI noise.
2021-
expect(growth).toBeLessThan(10)
2031+
expectLinearGrowth(growthFactor(build, 150, (input) => extractLeakedToolCalls(input, tools)))
20222032
expect(extractLeakedToolCalls(build(150), tools).calls).toHaveLength(150)
20232033
})
20242034

20252035
it("scans unclosed markup, deep nesting, and repeated quoting cues without quadratic blowup", () => {
20262036
const unclosed = (count: number) =>
20272037
`<function${"_calls"}>\n${`<in${"voke"} name="update_todo_list">\n`.repeat(count)}`
2028-
expect(growthFactor(unclosed, 400, (input) => extractLeakedToolCalls(input, tools))).toBeLessThan(10)
2038+
expectLinearGrowth(growthFactor(unclosed, 400, (input) => extractLeakedToolCalls(input, tools)))
20292039

20302040
// Nested tags before the block exercise tag stripping; cues with a trailing terminator
20312041
// exercise the quoting-cue scan. Both are read through the public entry point.
20322042
const nested = (depth: number) =>
20332043
`<function${"_calls"}>\n${"<".repeat(depth)}tag${">".repeat(depth)} ${todo()}\n`
2034-
expect(growthFactor(nested, 500, (input) => extractLeakedToolCalls(input, tools))).toBeLessThan(10)
2044+
expectLinearGrowth(growthFactor(nested, 500, (input) => extractLeakedToolCalls(input, tools)))
20352045

20362046
const cues = (count: number) => `<function${"_calls"}>\n${"never. ".repeat(count)}never ${todo()}\n`
2037-
expect(growthFactor(cues, 500, (input) => extractLeakedToolCalls(input, tools))).toBeLessThan(10)
2047+
expectLinearGrowth(growthFactor(cues, 500, (input) => extractLeakedToolCalls(input, tools)))
20382048
// The cue still suppresses recovery, so the fast path did not silently change the verdict.
20392049
expect(extractLeakedToolCalls(cues(500), tools).calls).toHaveLength(0)
20402050
})

0 commit comments

Comments
 (0)