Skip to content

Commit 9fb94ee

Browse files
fix(apple-runner): stop an inline status probe clearing an outstanding command charge
A `status` or `uptime` reply is served inline — outside the runner's journal and off its serial command queue — so it can answer while the command it probes is still executing. The old accounting settled by count, so that inline reply discharged the abandoned mutation's charge and a graceful shutdown handed off a runner still holding it. Charges now carry the `commandId` their exchange put on the wire: - an inline answer settles only its own charge; - a queued answer still forgives one abandoned residue (the serial queue makes it evidence the earlier handling finished, #2552); - `completed`/`failed` from the journal discharges the abandoned charge its `statusCommandId` names, exactly once, and cannot consume another command's; - `accepted`/`started`/unknown and probe failure keep the charge, so the runner stays on the kill path. `runner-readiness-routing.test.ts` pins the `readinessProbe` trait and the terminal-state set to the Swift declarations they describe. Regression rows in `runner-recovery-wiring.test.ts` enter through `runAppleRunnerCommand` and assert the production handoff decision; they fail on the pre-fix accounting. Closes #2965 Co-authored-by: Apex <[email protected]>
1 parent 13652ee commit 9fb94ee

9 files changed

Lines changed: 947 additions & 74 deletions
Lines changed: 350 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,350 @@
1+
import { describe, expect, test } from 'vitest';
2+
import { AppError } from '@agent-device/kernel/errors';
3+
import { RunnerCommandAccounting } from '../runner-command-accounting.ts';
4+
5+
// The settlement table #2965 asked to resolve before implementation. Each row records the outstanding
6+
// charges and the handoff verdict, so a rule that quietly stops naming the command it settles is a red
7+
// row rather than a runner handed off mid-mutation.
8+
9+
function abandonedTransportFailure(): AppError {
10+
return new AppError('COMMAND_FAILED', 'fetch failed');
11+
}
12+
13+
function busyRefusal(): AppError {
14+
return new AppError('COMMAND_FAILED', 'runner is draining', {
15+
runnerErrorCode: 'RUNNER_BUSY',
16+
runner: { ok: false, error: { code: 'RUNNER_BUSY', message: 'runner is draining' } },
17+
});
18+
}
19+
20+
describe('charging an exchange', () => {
21+
test('refuses handoff for a charge whose answer has not arrived', () => {
22+
const charges = new RunnerCommandAccounting();
23+
charges.charge('cmd-a', 'queued');
24+
25+
expect(charges.hasOutstandingCharges).toBe(true);
26+
expect(charges.hasAbandonedCharges).toBe(false);
27+
});
28+
29+
test('keys a charge by the lane it was served on, not by the id alone', () => {
30+
// `status` is sent no id, so its charge carries the blank id and can only name another probe. The
31+
// newest-first lookup makes this order the load-bearing one: the blank queued charge is the newest,
32+
// so a key without the lane hands the probe's answer to it and spends the probe's own proof on it —
33+
// discharging a queued exchange the probe says nothing about, and forgiving its residue too (#2965).
34+
const charges = new RunnerCommandAccounting();
35+
charges.charge(undefined, 'inline');
36+
charges.markAbandoned(undefined, 'inline');
37+
charges.charge(undefined, 'queued');
38+
charges.settleAnswered(undefined, 'inline');
39+
40+
expect(charges.outstandingChargeCount).toBe(1);
41+
expect(charges.hasAbandonedCharges).toBe(false);
42+
});
43+
test('journal evidence cannot reach a probe charge, which has no journal entry', () => {
44+
// The runner accepts a command into its journal from `enqueueAccepted`, past the inline arms, so a
45+
// lifecycle state can only ever name queued execution (#2965).
46+
const probe = new RunnerCommandAccounting();
47+
probe.charge('cmd-probe', 'inline');
48+
probe.markAbandoned('cmd-probe', 'inline');
49+
50+
expect(probe.settleTerminalEvidence('cmd-probe')).toEqual({
51+
settled: false,
52+
refused: 'no_abandoned_charge',
53+
});
54+
expect(probe.hasOutstandingCharges).toBe(true);
55+
});
56+
});
57+
58+
describe('an answered queued exchange', () => {
59+
test('settles its own charge and leaves a clean runner eligible', () => {
60+
const charges = new RunnerCommandAccounting();
61+
charges.charge('cmd-a', 'queued');
62+
charges.settleAnswered('cmd-a', 'queued');
63+
64+
expect(charges.hasOutstandingCharges).toBe(false);
65+
});
66+
67+
test('forgives one abandoned charge, because the serial queue proves the earlier work finished', () => {
68+
const charges = new RunnerCommandAccounting();
69+
charges.charge('cmd-a', 'queued');
70+
charges.markAbandoned('cmd-a', 'queued');
71+
charges.charge('cmd-b', 'queued');
72+
charges.settleAnswered('cmd-b', 'queued');
73+
74+
expect(charges.hasOutstandingCharges).toBe(false);
75+
});
76+
77+
test('leaves one residue per extra abandoned exchange', () => {
78+
const charges = new RunnerCommandAccounting();
79+
charges.charge('cmd-a', 'queued');
80+
charges.markAbandoned('cmd-a', 'queued');
81+
charges.charge('cmd-b', 'queued');
82+
charges.markAbandoned('cmd-b', 'queued');
83+
charges.charge('cmd-c', 'queued');
84+
charges.settleAnswered('cmd-c', 'queued');
85+
86+
// A and B are both abandoned and C answered: the conservative rule forgives one residue and keeps
87+
// the other, so a runner with a run of dropped exchanges stays on the kill path.
88+
expect(charges.outstandingChargeCount).toBe(1);
89+
expect(charges.hasAbandonedCharges).toBe(true);
90+
expect(charges.hasOutstandingCharges).toBe(true);
91+
});
92+
93+
test('settles a late answer to an abandoned exchange without forgiving a second residue', () => {
94+
const charges = new RunnerCommandAccounting();
95+
charges.charge('cmd-a', 'queued');
96+
charges.markAbandoned('cmd-a', 'queued');
97+
charges.charge('cmd-b', 'queued');
98+
charges.markAbandoned('cmd-b', 'queued');
99+
// A's own exchange answers after its send path reported failure: that is A's evidence, so B keeps
100+
// its charge rather than being forgiven by an answer that says nothing about it.
101+
charges.settleAnswered('cmd-a', 'queued');
102+
103+
expect(charges.outstandingChargeCount).toBe(1);
104+
expect(charges.hasAbandonedCharges).toBe(true);
105+
});
106+
107+
test('settles an answered structured refusal the way it settles a healthy reply', () => {
108+
const charges = new RunnerCommandAccounting();
109+
charges.charge('cmd-a', 'queued');
110+
charges.charge('cmd-b', 'queued');
111+
charges.markAbandoned('cmd-b', 'queued');
112+
charges.settleExchange('cmd-a', 'queued', busyRefusal());
113+
114+
expect(charges.hasOutstandingCharges).toBe(false);
115+
});
116+
117+
test('settles nothing when the id names no charge', () => {
118+
const charges = new RunnerCommandAccounting();
119+
charges.charge('cmd-a', 'queued');
120+
charges.settleAnswered('cmd-unknown', 'queued');
121+
122+
expect(charges.outstandingChargeCount).toBe(1);
123+
});
124+
});
125+
126+
describe('a transport-shaped failure', () => {
127+
test('keeps the charge and marks it abandoned', () => {
128+
const charges = new RunnerCommandAccounting();
129+
charges.charge('cmd-a', 'queued');
130+
charges.settleExchange('cmd-a', 'queued', abandonedTransportFailure());
131+
132+
expect(charges.hasOutstandingCharges).toBe(true);
133+
expect(charges.hasAbandonedCharges).toBe(true);
134+
});
135+
136+
test('marks no other command when its id holds no charge', () => {
137+
const charges = new RunnerCommandAccounting();
138+
charges.charge('cmd-a', 'queued');
139+
charges.settleAnswered('cmd-a', 'queued');
140+
charges.settleExchange('cmd-gone', 'queued', abandonedTransportFailure());
141+
142+
// Forgiving an unrelated charge here would clear the runner whose send actually failed.
143+
expect(charges.hasOutstandingCharges).toBe(false);
144+
});
145+
});
146+
147+
describe('an inline probe answer', () => {
148+
test('settles only its own charge and never an abandoned command', () => {
149+
const charges = new RunnerCommandAccounting();
150+
charges.charge('cmd-mutation', 'queued');
151+
charges.markAbandoned('cmd-mutation', 'queued');
152+
charges.charge(undefined, 'inline');
153+
charges.settleAnswered(undefined, 'inline');
154+
155+
// The runner served the probe off its XCTest channel while the mutation it abandoned may still run:
156+
// the probe's reply is no evidence about queued work (#2965).
157+
expect(charges.outstandingChargeCount).toBe(1);
158+
expect(charges.hasAbandonedCharges).toBe(true);
159+
expect(charges.hasOutstandingCharges).toBe(true);
160+
});
161+
162+
test('is refused by the handoff verdict while the probed command stays charged', () => {
163+
const charges = new RunnerCommandAccounting();
164+
charges.charge('cmd-mutation', 'queued');
165+
charges.markAbandoned('cmd-mutation', 'queued');
166+
charges.charge(undefined, 'inline');
167+
charges.settleAnswered(undefined, 'inline');
168+
169+
expect(charges.hasOutstandingCharges).toBe(true);
170+
});
171+
172+
test('never consumes a numbered exchange settlement, whichever order they settle in', () => {
173+
const byCommand = new RunnerCommandAccounting();
174+
byCommand.charge('cmd-mutation', 'queued');
175+
byCommand.markAbandoned('cmd-mutation', 'queued');
176+
byCommand.charge(undefined, 'inline');
177+
byCommand.settleAnswered(undefined, 'inline');
178+
179+
const byNumber = new RunnerCommandAccounting();
180+
byNumber.charge('cmd-mutation', 'queued');
181+
byNumber.markAbandoned('cmd-mutation', 'queued');
182+
byNumber.charge(undefined, 'inline');
183+
byNumber.settleAnswered('cmd-mutation', 'queued');
184+
185+
expect(byCommand.outstandingChargeCount).toBe(1);
186+
// The numbered answer takes the numbered charge only; the probe stays owed its own reply.
187+
expect(byNumber.outstandingChargeCount).toBe(1);
188+
expect(byNumber.hasAbandonedCharges).toBe(false);
189+
});
190+
});
191+
192+
describe('an answered probe and a dropped probe', () => {
193+
test('clears the dropped probe the next probe answered, so a healthy runner is not latched busy', () => {
194+
// Recovery sends `status`, and `status` is sent no id, so its charge is keyed blank and no terminal
195+
// verdict can ever name it. If a probe answer forgave nothing, one dropped probe would keep every
196+
// later handoff refused for the life of the session. Probe traffic proves only probe traffic
197+
// landed, so the residue forgiven here is a probe's, never the mutation's.
198+
const charges = new RunnerCommandAccounting();
199+
charges.charge(undefined, 'inline');
200+
charges.markAbandoned(undefined, 'inline');
201+
charges.charge(undefined, 'inline');
202+
charges.settleAnswered(undefined, 'inline');
203+
204+
expect(charges.hasOutstandingCharges).toBe(false);
205+
});
206+
207+
test('keeps a dropped probe charged while only queued answers land', () => {
208+
const charges = new RunnerCommandAccounting();
209+
charges.charge(undefined, 'inline');
210+
charges.markAbandoned(undefined, 'inline');
211+
charges.charge('cmd-a', 'queued');
212+
charges.settleAnswered('cmd-a', 'queued');
213+
214+
expect(charges.outstandingChargeCount).toBe(1);
215+
expect(charges.hasAbandonedCharges).toBe(true);
216+
});
217+
218+
test('settles its own id-keyed charge without touching another probe run', () => {
219+
// `uptime` is sent an id, so each health probe settles itself and an abandoned one is forgiven only
220+
// by the next `uptime` that answers.
221+
const charges = new RunnerCommandAccounting();
222+
charges.charge('cmd-probe-1', 'inline');
223+
charges.markAbandoned('cmd-probe-1', 'inline');
224+
charges.charge('cmd-probe-2', 'inline');
225+
charges.settleAnswered('cmd-probe-2', 'inline');
226+
227+
expect(charges.hasOutstandingCharges).toBe(false);
228+
});
229+
});
230+
231+
describe('terminal status evidence', () => {
232+
test('settles the abandoned charge it names, exactly once', () => {
233+
const charges = new RunnerCommandAccounting();
234+
charges.charge('cmd-a', 'queued');
235+
charges.markAbandoned('cmd-a', 'queued');
236+
237+
expect(charges.settleTerminalEvidence('cmd-a')).toEqual({
238+
settled: true,
239+
refused: undefined,
240+
});
241+
expect(charges.hasOutstandingCharges).toBe(false);
242+
});
243+
244+
test('a repeat observation settles nothing a second time', () => {
245+
const charges = new RunnerCommandAccounting();
246+
charges.charge('cmd-a', 'queued');
247+
charges.markAbandoned('cmd-a', 'queued');
248+
charges.settleTerminalEvidence('cmd-a');
249+
250+
expect(charges.settleTerminalEvidence('cmd-a')).toEqual({
251+
settled: false,
252+
refused: 'no_abandoned_charge',
253+
});
254+
});
255+
256+
test('cannot consume another command charge', () => {
257+
const charges = new RunnerCommandAccounting();
258+
charges.charge('cmd-a', 'queued');
259+
charges.markAbandoned('cmd-a', 'queued');
260+
charges.charge('cmd-b', 'queued');
261+
charges.markAbandoned('cmd-b', 'queued');
262+
263+
charges.settleTerminalEvidence('cmd-a');
264+
265+
expect(charges.outstandingChargeCount).toBe(1);
266+
expect(charges.hasAbandonedCharges).toBe(true);
267+
expect(charges.settleTerminalEvidence('cmd-a')).toEqual({
268+
settled: false,
269+
refused: 'no_abandoned_charge',
270+
});
271+
expect(charges.outstandingChargeCount).toBe(1);
272+
});
273+
274+
test('leaves an exchange still being awaited charged until its own answer settles', () => {
275+
const charges = new RunnerCommandAccounting();
276+
charges.charge('cmd-a', 'queued');
277+
charges.markAbandoned('cmd-a', 'queued');
278+
// A read-only retry re-sends the same id, so both attempts share one logical command. Terminal
279+
// evidence discharges the abandoned attempt; a live wait stays charged for its own answer.
280+
charges.charge('cmd-a', 'queued');
281+
282+
expect(charges.settleTerminalEvidence('cmd-a')).toEqual({
283+
settled: true,
284+
refused: undefined,
285+
});
286+
expect(charges.outstandingChargeCount).toBe(1);
287+
expect(charges.hasAbandonedCharges).toBe(false);
288+
289+
charges.settleAnswered('cmd-a', 'queued');
290+
expect(charges.hasOutstandingCharges).toBe(false);
291+
});
292+
293+
test('settles a charge a queued answer already forgave as nothing', () => {
294+
const charges = new RunnerCommandAccounting();
295+
charges.charge('cmd-a', 'queued');
296+
charges.markAbandoned('cmd-a', 'queued');
297+
charges.charge('cmd-b', 'queued');
298+
charges.settleAnswered('cmd-b', 'queued');
299+
300+
expect(charges.settleTerminalEvidence('cmd-a')).toEqual({
301+
settled: false,
302+
refused: 'no_abandoned_charge',
303+
});
304+
});
305+
306+
test('ignores an id the evidence does not carry', () => {
307+
const charges = new RunnerCommandAccounting();
308+
charges.charge('cmd-a', 'queued');
309+
charges.markAbandoned('cmd-a', 'queued');
310+
311+
expect(charges.settleTerminalEvidence(undefined)).toEqual({
312+
settled: false,
313+
refused: 'no_abandoned_charge',
314+
});
315+
expect(charges.outstandingChargeCount).toBe(1);
316+
});
317+
318+
test('settles every abandoned attempt of the command the evidence names', () => {
319+
const charges = new RunnerCommandAccounting();
320+
charges.charge('cmd-a', 'queued');
321+
charges.markAbandoned('cmd-a', 'queued');
322+
charges.charge('cmd-a', 'queued');
323+
charges.markAbandoned('cmd-a', 'queued');
324+
325+
expect(charges.settleTerminalEvidence('cmd-a').settled).toBe(true);
326+
expect(charges.hasOutstandingCharges).toBe(false);
327+
});
328+
});
329+
330+
describe('the abandoned-then-terminal sequence the issue asks about', () => {
331+
test('keeps the surviving charge named when a queued answer forgives one of two abandonments', () => {
332+
// A abandons, B abandons, C answers. The conservative rule forgives the oldest residue, so B stays
333+
// charged — and B's own terminal evidence, not A's, is what discharges it.
334+
const charges = new RunnerCommandAccounting();
335+
charges.charge('cmd-a', 'queued');
336+
charges.markAbandoned('cmd-a', 'queued');
337+
charges.charge('cmd-b', 'queued');
338+
charges.markAbandoned('cmd-b', 'queued');
339+
charges.charge('cmd-c', 'queued');
340+
charges.settleAnswered('cmd-c', 'queued');
341+
342+
expect(charges.hasAbandonedCharges).toBe(true);
343+
expect(charges.settleTerminalEvidence('cmd-a')).toEqual({
344+
settled: false,
345+
refused: 'no_abandoned_charge',
346+
});
347+
expect(charges.settleTerminalEvidence('cmd-b').settled).toBe(true);
348+
expect(charges.hasOutstandingCharges).toBe(false);
349+
});
350+
});

‎packages/platform-apple/src/runner/__tests__/runner-command-retry.test.ts‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
import { beforeEach, test, vi } from 'vitest';
22
import assert from 'node:assert/strict';
33
import { IOS_SIMULATOR } from './device-fixtures.ts';
4-
import { createTestRequestCancellation, runnerConnectFailure } from './runner-session-fixtures.ts';
4+
import {
5+
createTestRequestCancellation,
6+
makeRunnerSession,
7+
runnerConnectFailure,
8+
} from './runner-session-fixtures.ts';
59
import { AppError } from '@agent-device/kernel/errors';
610
import { Deadline } from '../host.ts';
711
import { appleRunnerTestHost } from '../test-host.ts';
8-
import type { RunnerSession } from '../runner-session-types.ts';
912

1013
const {
1114
mockEnsureRunnerSession,
@@ -1090,21 +1093,6 @@ function assertDiagnosticDecision(expected: {
10901093
);
10911094
}
10921095

1093-
function makeRunnerSession(overrides: Partial<RunnerSession> = {}): RunnerSession {
1094-
return {
1095-
sessionId: `session-${overrides.port ?? 8100}`,
1096-
device: IOS_SIMULATOR,
1097-
deviceId: IOS_SIMULATOR.id,
1098-
port: 8100,
1099-
xctestrunPath: '/tmp/runner.xctestrun',
1100-
jsonPath: '/tmp/runner.json',
1101-
testPromise: Promise.resolve({ exitCode: 0, stdout: '', stderr: '' }),
1102-
child: { pid: 1234, exitCode: null },
1103-
state: 'ready',
1104-
...overrides,
1105-
} as RunnerSession;
1106-
}
1107-
11081096
function makeRunnerArtifact(
11091097
overrides: Partial<RunnerXctestrunArtifact> = {},
11101098
): RunnerXctestrunArtifact {

0 commit comments

Comments
 (0)