Skip to content

Commit bd6f7f5

Browse files
test(output-economy): make routine-workflow ref-surfacing depend on rendered output
Address review on #1190: - Drop the raw mutation-confirm/failure MCP data samples that leaked e4/e5 and mislabeled the surface; e4/e5 now surface only from the rendered CLI settled-diff, so dropping added refs genuinely raises the fallback count. - Track the failure once as its projection-invariant normalized payload (workflow.failure.shared.json) instead of duplicate cli/mcp raw copies. - Reuse the shared REF_TOKEN_PATTERN from economy-metrics.ts. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent fec74bd commit bd6f7f5

3 files changed

Lines changed: 18 additions & 28 deletions

File tree

‎test/output-economy/economy-metrics.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
export type EconomySample = { text: string } | { data: unknown };
22

3+
// Rendered @ref token grammar (optionally generation-pinned, e.g. `@e12~s3`).
4+
// Shared so ref counting and the routine-workflow oracle agree on what a ref is.
5+
export const REF_TOKEN_PATTERN = /@e\d+(?:~s\d+)?/g;
6+
37
export type EconomyMetrics = {
48
bytes: number;
59
lines: number;
@@ -13,7 +17,7 @@ export function measureEconomySample(sample: EconomySample): EconomyMetrics {
1317
return {
1418
bytes: Buffer.byteLength(sample.text),
1519
lines: sample.text.length === 0 ? 0 : sample.text.split('\n').length,
16-
refs: new Set(sample.text.match(/@e\d+(?:~s\d+)?/g) ?? []).size,
20+
refs: new Set(sample.text.match(REF_TOKEN_PATTERN) ?? []).size,
1721
hints: (sample.text.match(/(?:^|\n)hint:/gi) ?? []).length,
1822
shape: 'text',
1923
};

‎test/output-economy/output-economy.baseline.json‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,6 @@
139139
"hints": 0,
140140
"shape": "text"
141141
},
142-
"workflow.mutation-confirm.mcp.json": {
143-
"bytes": 431,
144-
"lines": 1,
145-
"refs": 3,
146-
"hints": 0,
147-
"shape": "$.message:string|$.ref:string|$.settle.captures:number|$.settle.diff.lines:array|$.settle.diff.lines[].kind:string|$.settle.diff.lines[].ref:string|$.settle.diff.lines[].text:string|$.settle.diff.lines[]:object|$.settle.diff.summary.additions:number|$.settle.diff.summary.removals:number|$.settle.diff.summary.unchanged:number|$.settle.diff.summary:object|$.settle.diff:object|$.settle.quietMs:number|$.settle.refsGeneration:number|$.settle.settled:boolean|$.settle.timeoutMs:number|$.settle.waitedMs:number|$.settle:object|$.x:number|$.y:number|$:object"
148-
},
149142
"workflow.mutation-tail.cli.text": {
150143
"bytes": 161,
151144
"lines": 6,
@@ -160,14 +153,7 @@
160153
"hints": 0,
161154
"shape": "text"
162155
},
163-
"workflow.failure.cli.json": {
164-
"bytes": 328,
165-
"lines": 1,
166-
"refs": 1,
167-
"hints": 1,
168-
"shape": "$.code:string|$.details.reason:string|$.details.ref:string|$.details.refsGeneration:number|$.details.session:string|$.details.timeoutMs:number|$.details:object|$.hint:string|$.message:string|$.retriable:boolean|$:object"
169-
},
170-
"workflow.failure.mcp.json": {
156+
"workflow.failure.shared.json": {
171157
"bytes": 328,
172158
"lines": 1,
173159
"refs": 1,

‎test/output-economy/routine-workflow.ts‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { attachRefs, type RawSnapshotNode } from '../../src/kernel/snapshot.ts';
99
import { snapshotCliOutput } from '../../src/commands/capture/output.ts';
1010
import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts';
1111
import { createCommandToolExecutor } from '../../src/mcp/command-tools.ts';
12-
import type { EconomySample } from './economy-metrics.ts';
12+
import { REF_TOKEN_PATTERN, type EconomySample } from './economy-metrics.ts';
1313

1414
// A routine workflow-level oracle (#1180). PR #1174 pinned per-surface output
1515
// budgets; this pairs those bytes with the follow-up behavior they enable, so a
@@ -179,7 +179,10 @@ const RETRY_RESULT: CommandRequestResult = {
179179
},
180180
};
181181

182-
export type WorkflowProjection = 'cli' | 'mcp';
182+
// 'cli'/'mcp' are rendered per-projection; 'shared' is the projection-invariant
183+
// structured payload both surfaces carry (the normalized error's recovery
184+
// fields are identical in the CLI --json body and the MCP structuredContent).
185+
export type WorkflowProjection = 'cli' | 'mcp' | 'shared';
183186

184187
export type WorkflowStepKind = 'orient' | 'recheck' | 'mutation' | 'read' | 'failure' | 'retry';
185188

@@ -271,10 +274,11 @@ export async function renderRoutineWorkflow(): Promise<{
271274
command: 'press @e3 --settle',
272275
kind: 'mutation',
273276
targetRef: '@e3',
274-
samples: {
275-
cli: { text: interactionText(SETTLE_CONFIRM_RESULT) },
276-
mcp: { data: SETTLE_CONFIRM_RESULT },
277-
},
277+
// Only the rendered CLI settled-diff carries e4/e5. If that formatter
278+
// stops emitting added refs, `read` and `mutation-tail` lose their
279+
// targets and the fallback-observation count rises — so this step is a
280+
// genuine formatter guard, not a scan of the raw fixture object.
281+
samples: { cli: { text: interactionText(SETTLE_CONFIRM_RESULT) } },
278282
},
279283
{
280284
id: 'mutation-tail',
@@ -301,7 +305,7 @@ export async function renderRoutineWorkflow(): Promise<{
301305
command: 'press @e6 --settle',
302306
kind: 'failure',
303307
targetRef: '@e6',
304-
samples: { cli: { data: error }, mcp: { data: error } },
308+
samples: { shared: { data: error } },
305309
},
306310
{
307311
id: 'retry',
@@ -327,13 +331,9 @@ function workflowSamples(steps: WorkflowStep[]): Record<string, EconomySample> {
327331
return Object.fromEntries(entries);
328332
}
329333

330-
const REF_PATTERN = /@?e\d+/g;
331-
332334
function refsInSample(sample: EconomySample): string[] {
333335
const serialized = 'text' in sample ? sample.text : JSON.stringify(sample.data);
334-
return (serialized.match(REF_PATTERN) ?? []).map((ref) =>
335-
ref.startsWith('@') ? ref : `@${ref}`,
336-
);
336+
return serialized.match(REF_TOKEN_PATTERN) ?? [];
337337
}
338338

339339
function measureRoutineWorkflow(

0 commit comments

Comments
 (0)