Skip to content

Commit 3bbe6a8

Browse files
committed
fix(ios): keep a pruned commandId spent and answer a joined send from one decision
Pruning an entry kept the entry's admission fact with the body it shared, so a pruned id became executable again and a delayed resend could drive the device a second time. Pruning now leaves the id in a bounded closed set that answers admission with the typed refusal, and a pruned refusal keeps its id free for the busy resend (ADR 0005). The join verdict and registering the send for the execution's answer were two steps, so a close that landed between them left the send waiting on a delivery that had already gone out. They are one operation the journal's lock covers.
1 parent e1c2cae commit 3bbe6a8

5 files changed

Lines changed: 328 additions & 130 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift‎

Lines changed: 68 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ struct RunnerCommandJournalEntry {
3636
enum RunnerCommandAdmission {
3737
/// Nothing is running for this id and nothing has finished: the caller owns the one execution.
3838
case execute
39-
/// An execution is claimed and has not finished: the caller joins its outcome and runs nothing.
39+
/// An execution is claimed and has not finished: the caller runs nothing and is answered by that
40+
/// execution. `admit(command:registeringJoin:)` has already registered the send it gave this verdict.
4041
case join
4142
/// Execution finished, and the work reached the device: the caller answers from the journal and
4243
/// never executes this command again.
@@ -50,19 +51,44 @@ enum RunnerCommandAdmission {
5051
final class RunnerCommandJournal {
5152
private let lock = NSLock()
5253
private let maxEntries = 64
54+
/// How many closed `commandId`s are remembered as bare ids once their entry is pruned. The window is
55+
/// 64x the entry window because a marker holds no outcome body, only the id.
56+
private let maxClosedCommandIds = 4096
5357
private let maxResponseJsonBytes = 16 * 1024
5458
private var entries: [String: RunnerCommandJournalEntry] = [:]
5559
private var order: [String] = []
60+
/// Ids whose dispatched execution closed and whose entry has since been pruned: the admission half of
61+
/// what the entry recorded, kept without a body so bounded memory cannot make a spent id executable
62+
/// again. It settles admission only. `status` still answers `notAccepted` for such an id, because the
63+
/// outcome the entry held is gone and only the entry can report one.
64+
private var closedCommandIds: Set<String> = []
65+
private var closedOrder: [String] = []
5666

67+
/// Settles one send, and when the answer is to join a running execution, registers that send for the
68+
/// answer inside the same locked operation. No transition can close the entry between the two, which is
69+
/// what keeps a joined send from waiting on a delivery that already went out. `register` runs on the
70+
/// calling thread while the journal's lock is held: it may take a lock of its own, and must not call
71+
/// back into this journal. The wire route passes one; a caller that only reads the verdict — a journal
72+
/// test, or a probe that will not itself answer for the execution — leaves it unregistered and must not
73+
/// treat a `.join` verdict as a send that will be answered.
5774
@discardableResult
58-
func accept(command: Command) -> RunnerCommandAdmission {
75+
func accept(
76+
command: Command,
77+
registeringJoin register: (String) -> Void = { _ in }
78+
) -> RunnerCommandAdmission {
5979
guard let commandId = command.commandId?.trimmedNonEmpty else { return .execute }
6080
lock.lock()
6181
defer { lock.unlock() }
6282
guard let entry = entries[commandId] else {
83+
// The entry can be pruned; the fact it recorded is not, so an id that spent itself on the device
84+
// never executes again.
85+
if closedCommandIds.contains(commandId) {
86+
return .terminal(outcome: alreadyFinishedRefusal(command: command, commandId: commandId, outcomeDropped: true))
87+
}
6388
return claim(command: command, commandId: commandId)
6489
}
6590
if !entry.state.isTerminal {
91+
register(commandId)
6692
return .join
6793
}
6894
// Nothing of this command ever reached the device, so the id carries no effect to protect and the
@@ -119,19 +145,6 @@ final class RunnerCommandJournal {
119145
)
120146
}
121147

122-
/// The answer owed a send that joined an execution which has since closed: the outcome the runner
123-
/// recorded, whether or not the work reached the device. Registering that send as a waiter instead
124-
/// would leave it waiting on a delivery that already went out. `accept` answers a closed id on its own
125-
/// terms, so this is narrower than admission.
126-
func closedOutcome(for command: Command) -> Response? {
127-
guard let commandId = command.commandId?.trimmedNonEmpty else { return nil }
128-
lock.lock()
129-
let entry = entries[commandId]
130-
lock.unlock()
131-
guard let entry, entry.state.isTerminal else { return nil }
132-
return recordedOutcome(for: entry)
133-
}
134-
135148
func status(normalizedCommandId commandId: String) -> DataPayload {
136149
lock.lock()
137150
let entry = entries[commandId]
@@ -200,16 +213,26 @@ final class RunnerCommandJournal {
200213
pruneIfNeeded()
201214
}
202215

203-
/// Drops the oldest closed entries. An entry whose execution is claimed but not finished is never
204-
/// dropped, however deep the backlog: `status` answers `notAccepted` only for an id this journal does
205-
/// not hold, and the daemon cannot read that as a safe terminal state — it would invalidate a session
206-
/// whose runner is still executing the command it lost the answer to.
216+
/// Drops the oldest closed entries, keeping each one's id as the fact that settles admission for it.
217+
/// An entry whose execution is claimed but not finished is never dropped, however deep the backlog:
218+
/// `status` answers `notAccepted` only for an id this journal does not hold, and the daemon cannot read
219+
/// that as a safe terminal state — it would invalidate a session whose runner is still executing the
220+
/// command it lost the answer to.
207221
private func pruneIfNeeded() {
208222
while order.count > maxEntries {
209223
guard let removable = order.first(where: { entries[$0]?.state.isTerminal == true }) else {
210224
return
211225
}
212226
order.removeAll { $0 == removable }
227+
// Only work that reached the device spends its id. A pruned refusal leaves the id free, which is
228+
// what the busy resend of the same `commandId` depends on (ADR 0005).
229+
if entries[removable]?.dispatched == true {
230+
closedCommandIds.insert(removable)
231+
closedOrder.append(removable)
232+
while closedOrder.count > maxClosedCommandIds {
233+
closedCommandIds.remove(closedOrder.removeFirst())
234+
}
235+
}
213236
entries.removeValue(forKey: removable)
214237
}
215238
}
@@ -225,12 +248,36 @@ final class RunnerCommandJournal {
225248
if let error = entry.error {
226249
return Response(ok: false, error: error)
227250
}
228-
return Response(
251+
return alreadyFinishedRefusal(
252+
commandId: entry.commandId,
253+
commandName: entry.command,
254+
outcomeDropped: false
255+
)
256+
}
257+
258+
private func alreadyFinishedRefusal(
259+
command: Command,
260+
commandId: String,
261+
outcomeDropped: Bool
262+
) -> Response {
263+
alreadyFinishedRefusal(
264+
commandId: commandId,
265+
commandName: command.command.rawValue,
266+
outcomeDropped: outcomeDropped
267+
)
268+
}
269+
270+
private func alreadyFinishedRefusal(
271+
commandId: String,
272+
commandName: String,
273+
outcomeDropped: Bool
274+
) -> Response {
275+
Response(
229276
ok: false,
230277
error: ErrorPayload(
231278
code: RunnerTests.RunnerWireErrorCode.commandAlreadyFinished,
232279
message:
233-
"The runner already finished \(entry.command) with id \(entry.commandId) and kept no response to replay.",
280+
"The runner already finished \(commandName) with id \(commandId) and kept no response to replay\(outcomeDropped ? ", because that record has since been dropped" : "").",
234281
hint:
235282
"Read the screen with snapshot before sending anything else: the finished command's effect is already on the device."
236283
)

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift‎

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -167,16 +167,25 @@ extension RunnerTests {
167167
completion((jsonResponse(status: 200, response: response), false))
168168
return
169169
}
170-
// The journal decides whether this send may run the command at all. Reading that verdict here, at
171-
// the one route an inbound request takes, is what keeps one `commandId` to one execution: a
170+
// The journal decides here, at the one route an inbound request takes, whether this send may run
171+
// the command at all, and when the answer is to join an execution it registers this send for that
172+
// execution's answer in the same operation. That is what keeps one `commandId` to one execution: a
172173
// command that already finished is answered from the journal, so a repeat that arrives after its
173-
// answer was delivered cannot drive the device a second time.
174-
switch commandJournal.accept(command: command) {
174+
// answer was delivered cannot drive the device a second time, and a send told to join can never be
175+
// left behind by a delivery that went out first.
176+
let admission = commandJournal.accept(command: command) { [weak self] commandId in
177+
self?.registerJoiner(for: commandId, completion: completion)
178+
}
179+
switch admission {
175180
case .join:
176-
// Re-sends of a still-executing commandId (the daemon's transport retry loop) attach to the
177-
// in-flight execution and receive its response instead of piling a second execution onto the
178-
// main queue behind it (#1105 capture pileup).
179-
attachToInFlightExecution(command: command, completion: completion)
181+
// Re-sends of a still-executing commandId (the daemon's transport retry loop) were registered
182+
// onto the in-flight execution above and receive its response instead of piling a second
183+
// execution onto the main queue behind it (#1105 capture pileup).
184+
NSLog(
185+
"AGENT_DEVICE_RUNNER_COMMAND_COALESCED command=%@ commandId=%@",
186+
command.command.rawValue,
187+
command.commandId ?? ""
188+
)
180189
case .terminal(let outcome):
181190
NSLog(
182191
"AGENT_DEVICE_RUNNER_COMMAND_ALREADY_FINISHED command=%@ commandId=%@",
@@ -265,31 +274,22 @@ extension RunnerTests {
265274

266275
// MARK: - In-Flight Command Coalescing
267276

268-
/// Attaches this send to the in-flight execution the journal just told it to join, so the execution's
269-
/// own delivery answers it. The journal closes an entry before its delivery drains this waiter list, and
270-
/// both take `inFlightCommandLock`, so checking here under that lock decides which answer this send is
271-
/// owed exactly once: one that arrives after the entry closed answers from the journal instead of
272-
/// waiting for a delivery that already went out.
273-
func attachToInFlightExecution(
274-
command: Command,
277+
/// Records this send as a waiter of the in-flight execution the journal just settled it onto, so the
278+
/// execution's own delivery answers it. The journal calls this while it holds its own lock and after it
279+
/// checked that the entry is still running, which is what makes the two indivisible: the transition that
280+
/// closes an entry needs that lock, and one close drains this list only after it has closed the entry, so
281+
/// no send can be recorded here for a delivery that already went out. Takes only
282+
/// `inFlightCommandLock`, and never the journal's, so nothing here can re-enter the journal.
283+
func registerJoiner(
284+
for commandId: String,
275285
completion: @escaping @Sendable ((data: Data, shouldFinish: Bool)) -> Void
276286
) {
277-
guard let commandId = command.commandId?.trimmedNonEmpty else { return }
278287
inFlightCommandLock.lock()
279-
if let outcome = commandJournal.closedOutcome(for: command) {
280-
inFlightCommandLock.unlock()
281-
completion((jsonResponse(status: 200, response: outcome), false))
282-
return
283-
}
284288
inFlightCommandWaiters[commandId, default: []].append(completion)
285289
inFlightCommandLock.unlock()
286-
NSLog(
287-
"AGENT_DEVICE_RUNNER_COMMAND_COALESCED command=%@ commandId=%@",
288-
command.command.rawValue,
289-
commandId
290-
)
291290
}
292291

292+
/// Answers the send that owns this execution, and every send the journal settled onto it.
293293
func deliverCommandResult(
294294
command: Command,
295295
result: (data: Data, shouldFinish: Bool),

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift‎

Lines changed: 62 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,19 @@ extension RunnerTests {
200200
XCTAssertNil(largeReadStatus.lifecycleResponseJson)
201201
}
202202

203+
/// Fills the 64-entry retention window the way a busy session does: closed commands, oldest first. The
204+
/// transport's own retention tests need the same pressure, so this is the one implementation.
205+
func runnerJournalFinishOtherCommands(_ journal: RunnerCommandJournal, _ count: Int) {
206+
for index in 0..<count {
207+
let settled = runnerJournalCommand("readText", id: "settled-\(index)")
208+
journal.accept(command: settled)
209+
journal.finish(
210+
command: settled,
211+
response: Response(ok: true, data: DataPayload(text: "x"))
212+
)
213+
}
214+
}
215+
203216
/// The 64-entry window must never cost the runner the ability to answer "is this command still
204217
/// running?". A queued or executing command is the one entry pruning may not drop: `status` reports
205218
/// `notAccepted` for an id it does not hold, and the daemon cannot read that as a safe terminal
@@ -211,14 +224,7 @@ extension RunnerTests {
211224
journal.accept(command: running)
212225
journal.start(command: running)
213226

214-
for index in 0..<64 {
215-
let settled = runnerJournalCommand("readText", id: "settled-\(index)")
216-
journal.accept(command: settled)
217-
journal.finish(
218-
command: settled,
219-
response: Response(ok: true, data: DataPayload(text: "x"))
220-
)
221-
}
227+
runnerJournalFinishOtherCommands(journal, 64)
222228

223229
let status = journal.status(normalizedCommandId: "running-tap")
224230
XCTAssertEqual(
@@ -239,14 +245,7 @@ extension RunnerTests {
239245
command: early,
240246
response: Response(ok: true, data: DataPayload(message: "tapped"))
241247
)
242-
for index in 0..<64 {
243-
let settled = runnerJournalCommand("readText", id: "settled-\(index)")
244-
journal.accept(command: settled)
245-
journal.finish(
246-
command: settled,
247-
response: Response(ok: true, data: DataPayload(text: "x"))
248-
)
249-
}
248+
runnerJournalFinishOtherCommands(journal, 64)
250249
XCTAssertEqual(
251250
journal.status(normalizedCommandId: "pruned-tap").lifecycleState,
252251
RunnerCommandLifecycleState.notAccepted.rawValue
@@ -261,6 +260,53 @@ extension RunnerTests {
261260
)
262261
}
263262

263+
/// Bounded memory has to drop something, and what it may drop is the outcome body, never the fact that
264+
/// the id was spent. Otherwise the retention window itself becomes the bug the journal exists to
265+
/// prevent: finish a dispatched tap, let 64 later commands push its entry out, and a delayed resend of
266+
/// that tap would be admitted as a new execution and tap the device a second time.
267+
func testJournalNeverReAdmitsAPrunedCommandsId() throws {
268+
let journal = RunnerCommandJournal()
269+
let tap = runnerJournalCommand("tap", id: "spent-tap")
270+
journal.accept(command: tap)
271+
journal.markDispatched(command: tap)
272+
journal.finish(
273+
command: tap,
274+
response: Response(ok: true, data: DataPayload(message: "tapped"))
275+
)
276+
runnerJournalFinishOtherCommands(journal, 64)
277+
278+
guard case .terminal(let outcome) = journal.accept(command: tap) else {
279+
return XCTFail("a pruned id whose work reached the device must never be admitted again")
280+
}
281+
XCTAssertFalse(outcome.ok)
282+
XCTAssertEqual(
283+
outcome.error?.code,
284+
RunnerTests.RunnerWireErrorCode.commandAlreadyFinished,
285+
"the pruned id's outcome body is gone, so the answer is the typed refusal, got \(String(describing: outcome.error?.code))"
286+
)
287+
}
288+
289+
/// The one id pruning must leave free: a command the runner refused before dispatch left nothing on the
290+
/// device, and its read-only resend arrives with the same `commandId` (ADR 0005). A closed marker for it
291+
/// would refuse that resend and lose a command the runner never ran.
292+
func testJournalReAdmitsAPrunedIdWhoseExecutionNeverDispatched() throws {
293+
let journal = RunnerCommandJournal()
294+
let refused = runnerJournalCommand("readText", id: "refused-before-dispatch")
295+
journal.accept(command: refused)
296+
journal.finish(
297+
command: refused,
298+
response: Response(
299+
ok: false,
300+
error: ErrorPayload(code: "RUNNER_BUSY", message: "The iOS runner is still finishing a previous command.")
301+
)
302+
)
303+
runnerJournalFinishOtherCommands(journal, 64)
304+
305+
guard case .execute = journal.accept(command: refused) else {
306+
return XCTFail("an id the runner refused before dispatch carries no effect and must stay executable")
307+
}
308+
}
309+
264310
/// The daemon resends a read-only command the runner refused with `RUNNER_BUSY`, under its busy
265311
/// budget and with the same `commandId` (ADR 0005). A terminal record closes only ids whose execution
266312
/// began, so a refusal that dispatched nothing has to leave the id executable once the runner is idle,

0 commit comments

Comments
 (0)