|
| 1 | +import { describe, expect, test } from 'vitest'; |
| 2 | +import { AppError } from '@agent-device/kernel/errors'; |
| 3 | +import { RunnerCommandAccounting } from '../runner-command-accounting.ts'; |
| 4 | + |
| 5 | +// The settlement table #2965 asked to resolve before implementation. Each row records the outstanding |
| 6 | +// charges and the handoff verdict, so a rule that quietly stops naming the command it settles is a red |
| 7 | +// row rather than a runner handed off mid-mutation. |
| 8 | + |
| 9 | +function abandonedTransportFailure(): AppError { |
| 10 | + return new AppError('COMMAND_FAILED', 'fetch failed'); |
| 11 | +} |
| 12 | + |
| 13 | +function busyRefusal(): AppError { |
| 14 | + return new AppError('COMMAND_FAILED', 'runner is draining', { |
| 15 | + runnerErrorCode: 'RUNNER_BUSY', |
| 16 | + runner: { ok: false, error: { code: 'RUNNER_BUSY', message: 'runner is draining' } }, |
| 17 | + }); |
| 18 | +} |
| 19 | + |
| 20 | +describe('charging an exchange', () => { |
| 21 | + test('refuses handoff for a charge whose answer has not arrived', () => { |
| 22 | + const charges = new RunnerCommandAccounting(); |
| 23 | + charges.charge('cmd-a', 'queued'); |
| 24 | + |
| 25 | + expect(charges.hasOutstandingCharges).toBe(true); |
| 26 | + expect(charges.hasAbandonedCharges).toBe(false); |
| 27 | + }); |
| 28 | + |
| 29 | + test('keys a charge by the lane it was served on, not by the id alone', () => { |
| 30 | + // `status` is sent no id, so its charge carries the blank id and can only name another probe. The |
| 31 | + // newest-first lookup makes this order the load-bearing one: the blank queued charge is the newest, |
| 32 | + // so a key without the lane hands the probe's answer to it and spends the probe's own proof on it — |
| 33 | + // discharging a queued exchange the probe says nothing about, and forgiving its residue too (#2965). |
| 34 | + const charges = new RunnerCommandAccounting(); |
| 35 | + charges.charge(undefined, 'inline'); |
| 36 | + charges.markAbandoned(undefined, 'inline'); |
| 37 | + charges.charge(undefined, 'queued'); |
| 38 | + charges.settleAnswered(undefined, 'inline'); |
| 39 | + |
| 40 | + expect(charges.outstandingChargeCount).toBe(1); |
| 41 | + expect(charges.hasAbandonedCharges).toBe(false); |
| 42 | + }); |
| 43 | + test('journal evidence cannot reach a probe charge, which has no journal entry', () => { |
| 44 | + // The runner accepts a command into its journal from `enqueueAccepted`, past the inline arms, so a |
| 45 | + // lifecycle state can only ever name queued execution (#2965). |
| 46 | + const probe = new RunnerCommandAccounting(); |
| 47 | + probe.charge('cmd-probe', 'inline'); |
| 48 | + probe.markAbandoned('cmd-probe', 'inline'); |
| 49 | + |
| 50 | + expect(probe.settleTerminalEvidence('cmd-probe')).toEqual({ |
| 51 | + settled: false, |
| 52 | + refused: 'no_abandoned_charge', |
| 53 | + }); |
| 54 | + expect(probe.hasOutstandingCharges).toBe(true); |
| 55 | + }); |
| 56 | +}); |
| 57 | + |
| 58 | +describe('an answered queued exchange', () => { |
| 59 | + test('settles its own charge and leaves a clean runner eligible', () => { |
| 60 | + const charges = new RunnerCommandAccounting(); |
| 61 | + charges.charge('cmd-a', 'queued'); |
| 62 | + charges.settleAnswered('cmd-a', 'queued'); |
| 63 | + |
| 64 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 65 | + }); |
| 66 | + |
| 67 | + test('forgives one abandoned charge, because the serial queue proves the earlier work finished', () => { |
| 68 | + const charges = new RunnerCommandAccounting(); |
| 69 | + charges.charge('cmd-a', 'queued'); |
| 70 | + charges.markAbandoned('cmd-a', 'queued'); |
| 71 | + charges.charge('cmd-b', 'queued'); |
| 72 | + charges.settleAnswered('cmd-b', 'queued'); |
| 73 | + |
| 74 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 75 | + }); |
| 76 | + |
| 77 | + test('leaves one residue per extra abandoned exchange', () => { |
| 78 | + const charges = new RunnerCommandAccounting(); |
| 79 | + charges.charge('cmd-a', 'queued'); |
| 80 | + charges.markAbandoned('cmd-a', 'queued'); |
| 81 | + charges.charge('cmd-b', 'queued'); |
| 82 | + charges.markAbandoned('cmd-b', 'queued'); |
| 83 | + charges.charge('cmd-c', 'queued'); |
| 84 | + charges.settleAnswered('cmd-c', 'queued'); |
| 85 | + |
| 86 | + // A and B are both abandoned and C answered: the conservative rule forgives one residue and keeps |
| 87 | + // the other, so a runner with a run of dropped exchanges stays on the kill path. |
| 88 | + expect(charges.outstandingChargeCount).toBe(1); |
| 89 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 90 | + expect(charges.hasOutstandingCharges).toBe(true); |
| 91 | + }); |
| 92 | + |
| 93 | + test('settles a late answer to an abandoned exchange without forgiving a second residue', () => { |
| 94 | + const charges = new RunnerCommandAccounting(); |
| 95 | + charges.charge('cmd-a', 'queued'); |
| 96 | + charges.markAbandoned('cmd-a', 'queued'); |
| 97 | + charges.charge('cmd-b', 'queued'); |
| 98 | + charges.markAbandoned('cmd-b', 'queued'); |
| 99 | + // A's own exchange answers after its send path reported failure: that is A's evidence, so B keeps |
| 100 | + // its charge rather than being forgiven by an answer that says nothing about it. |
| 101 | + charges.settleAnswered('cmd-a', 'queued'); |
| 102 | + |
| 103 | + expect(charges.outstandingChargeCount).toBe(1); |
| 104 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 105 | + }); |
| 106 | + |
| 107 | + test('settles an answered structured refusal the way it settles a healthy reply', () => { |
| 108 | + const charges = new RunnerCommandAccounting(); |
| 109 | + charges.charge('cmd-a', 'queued'); |
| 110 | + charges.charge('cmd-b', 'queued'); |
| 111 | + charges.markAbandoned('cmd-b', 'queued'); |
| 112 | + charges.settleExchange('cmd-a', 'queued', busyRefusal()); |
| 113 | + |
| 114 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 115 | + }); |
| 116 | + |
| 117 | + test('settles nothing when the id names no charge', () => { |
| 118 | + const charges = new RunnerCommandAccounting(); |
| 119 | + charges.charge('cmd-a', 'queued'); |
| 120 | + charges.settleAnswered('cmd-unknown', 'queued'); |
| 121 | + |
| 122 | + expect(charges.outstandingChargeCount).toBe(1); |
| 123 | + }); |
| 124 | +}); |
| 125 | + |
| 126 | +describe('a transport-shaped failure', () => { |
| 127 | + test('keeps the charge and marks it abandoned', () => { |
| 128 | + const charges = new RunnerCommandAccounting(); |
| 129 | + charges.charge('cmd-a', 'queued'); |
| 130 | + charges.settleExchange('cmd-a', 'queued', abandonedTransportFailure()); |
| 131 | + |
| 132 | + expect(charges.hasOutstandingCharges).toBe(true); |
| 133 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 134 | + }); |
| 135 | + |
| 136 | + test('marks no other command when its id holds no charge', () => { |
| 137 | + const charges = new RunnerCommandAccounting(); |
| 138 | + charges.charge('cmd-a', 'queued'); |
| 139 | + charges.settleAnswered('cmd-a', 'queued'); |
| 140 | + charges.settleExchange('cmd-gone', 'queued', abandonedTransportFailure()); |
| 141 | + |
| 142 | + // Forgiving an unrelated charge here would clear the runner whose send actually failed. |
| 143 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 144 | + }); |
| 145 | +}); |
| 146 | + |
| 147 | +describe('an inline probe answer', () => { |
| 148 | + test('settles only its own charge and never an abandoned command', () => { |
| 149 | + const charges = new RunnerCommandAccounting(); |
| 150 | + charges.charge('cmd-mutation', 'queued'); |
| 151 | + charges.markAbandoned('cmd-mutation', 'queued'); |
| 152 | + charges.charge(undefined, 'inline'); |
| 153 | + charges.settleAnswered(undefined, 'inline'); |
| 154 | + |
| 155 | + // The runner served the probe off its XCTest channel while the mutation it abandoned may still run: |
| 156 | + // the probe's reply is no evidence about queued work (#2965). |
| 157 | + expect(charges.outstandingChargeCount).toBe(1); |
| 158 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 159 | + expect(charges.hasOutstandingCharges).toBe(true); |
| 160 | + }); |
| 161 | + |
| 162 | + test('is refused by the handoff verdict while the probed command stays charged', () => { |
| 163 | + const charges = new RunnerCommandAccounting(); |
| 164 | + charges.charge('cmd-mutation', 'queued'); |
| 165 | + charges.markAbandoned('cmd-mutation', 'queued'); |
| 166 | + charges.charge(undefined, 'inline'); |
| 167 | + charges.settleAnswered(undefined, 'inline'); |
| 168 | + |
| 169 | + expect(charges.hasOutstandingCharges).toBe(true); |
| 170 | + }); |
| 171 | + |
| 172 | + test('never consumes a numbered exchange settlement, whichever order they settle in', () => { |
| 173 | + const byCommand = new RunnerCommandAccounting(); |
| 174 | + byCommand.charge('cmd-mutation', 'queued'); |
| 175 | + byCommand.markAbandoned('cmd-mutation', 'queued'); |
| 176 | + byCommand.charge(undefined, 'inline'); |
| 177 | + byCommand.settleAnswered(undefined, 'inline'); |
| 178 | + |
| 179 | + const byNumber = new RunnerCommandAccounting(); |
| 180 | + byNumber.charge('cmd-mutation', 'queued'); |
| 181 | + byNumber.markAbandoned('cmd-mutation', 'queued'); |
| 182 | + byNumber.charge(undefined, 'inline'); |
| 183 | + byNumber.settleAnswered('cmd-mutation', 'queued'); |
| 184 | + |
| 185 | + expect(byCommand.outstandingChargeCount).toBe(1); |
| 186 | + // The numbered answer takes the numbered charge only; the probe stays owed its own reply. |
| 187 | + expect(byNumber.outstandingChargeCount).toBe(1); |
| 188 | + expect(byNumber.hasAbandonedCharges).toBe(false); |
| 189 | + }); |
| 190 | +}); |
| 191 | + |
| 192 | +describe('an answered probe and a dropped probe', () => { |
| 193 | + test('clears the dropped probe the next probe answered, so a healthy runner is not latched busy', () => { |
| 194 | + // Recovery sends `status`, and `status` is sent no id, so its charge is keyed blank and no terminal |
| 195 | + // verdict can ever name it. If a probe answer forgave nothing, one dropped probe would keep every |
| 196 | + // later handoff refused for the life of the session. Probe traffic proves only probe traffic |
| 197 | + // landed, so the residue forgiven here is a probe's, never the mutation's. |
| 198 | + const charges = new RunnerCommandAccounting(); |
| 199 | + charges.charge(undefined, 'inline'); |
| 200 | + charges.markAbandoned(undefined, 'inline'); |
| 201 | + charges.charge(undefined, 'inline'); |
| 202 | + charges.settleAnswered(undefined, 'inline'); |
| 203 | + |
| 204 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 205 | + }); |
| 206 | + |
| 207 | + test('keeps a dropped probe charged while only queued answers land', () => { |
| 208 | + const charges = new RunnerCommandAccounting(); |
| 209 | + charges.charge(undefined, 'inline'); |
| 210 | + charges.markAbandoned(undefined, 'inline'); |
| 211 | + charges.charge('cmd-a', 'queued'); |
| 212 | + charges.settleAnswered('cmd-a', 'queued'); |
| 213 | + |
| 214 | + expect(charges.outstandingChargeCount).toBe(1); |
| 215 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 216 | + }); |
| 217 | + |
| 218 | + test('settles its own id-keyed charge without touching another probe run', () => { |
| 219 | + // `uptime` is sent an id, so each health probe settles itself and an abandoned one is forgiven only |
| 220 | + // by the next `uptime` that answers. |
| 221 | + const charges = new RunnerCommandAccounting(); |
| 222 | + charges.charge('cmd-probe-1', 'inline'); |
| 223 | + charges.markAbandoned('cmd-probe-1', 'inline'); |
| 224 | + charges.charge('cmd-probe-2', 'inline'); |
| 225 | + charges.settleAnswered('cmd-probe-2', 'inline'); |
| 226 | + |
| 227 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 228 | + }); |
| 229 | +}); |
| 230 | + |
| 231 | +describe('terminal status evidence', () => { |
| 232 | + test('settles the abandoned charge it names, exactly once', () => { |
| 233 | + const charges = new RunnerCommandAccounting(); |
| 234 | + charges.charge('cmd-a', 'queued'); |
| 235 | + charges.markAbandoned('cmd-a', 'queued'); |
| 236 | + |
| 237 | + expect(charges.settleTerminalEvidence('cmd-a')).toEqual({ |
| 238 | + settled: true, |
| 239 | + refused: undefined, |
| 240 | + }); |
| 241 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 242 | + }); |
| 243 | + |
| 244 | + test('a repeat observation settles nothing a second time', () => { |
| 245 | + const charges = new RunnerCommandAccounting(); |
| 246 | + charges.charge('cmd-a', 'queued'); |
| 247 | + charges.markAbandoned('cmd-a', 'queued'); |
| 248 | + charges.settleTerminalEvidence('cmd-a'); |
| 249 | + |
| 250 | + expect(charges.settleTerminalEvidence('cmd-a')).toEqual({ |
| 251 | + settled: false, |
| 252 | + refused: 'no_abandoned_charge', |
| 253 | + }); |
| 254 | + }); |
| 255 | + |
| 256 | + test('cannot consume another command charge', () => { |
| 257 | + const charges = new RunnerCommandAccounting(); |
| 258 | + charges.charge('cmd-a', 'queued'); |
| 259 | + charges.markAbandoned('cmd-a', 'queued'); |
| 260 | + charges.charge('cmd-b', 'queued'); |
| 261 | + charges.markAbandoned('cmd-b', 'queued'); |
| 262 | + |
| 263 | + charges.settleTerminalEvidence('cmd-a'); |
| 264 | + |
| 265 | + expect(charges.outstandingChargeCount).toBe(1); |
| 266 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 267 | + expect(charges.settleTerminalEvidence('cmd-a')).toEqual({ |
| 268 | + settled: false, |
| 269 | + refused: 'no_abandoned_charge', |
| 270 | + }); |
| 271 | + expect(charges.outstandingChargeCount).toBe(1); |
| 272 | + }); |
| 273 | + |
| 274 | + test('leaves an exchange still being awaited charged until its own answer settles', () => { |
| 275 | + const charges = new RunnerCommandAccounting(); |
| 276 | + charges.charge('cmd-a', 'queued'); |
| 277 | + charges.markAbandoned('cmd-a', 'queued'); |
| 278 | + // A read-only retry re-sends the same id, so both attempts share one logical command. Terminal |
| 279 | + // evidence discharges the abandoned attempt; a live wait stays charged for its own answer. |
| 280 | + charges.charge('cmd-a', 'queued'); |
| 281 | + |
| 282 | + expect(charges.settleTerminalEvidence('cmd-a')).toEqual({ |
| 283 | + settled: true, |
| 284 | + refused: undefined, |
| 285 | + }); |
| 286 | + expect(charges.outstandingChargeCount).toBe(1); |
| 287 | + expect(charges.hasAbandonedCharges).toBe(false); |
| 288 | + |
| 289 | + charges.settleAnswered('cmd-a', 'queued'); |
| 290 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 291 | + }); |
| 292 | + |
| 293 | + test('settles a charge a queued answer already forgave as nothing', () => { |
| 294 | + const charges = new RunnerCommandAccounting(); |
| 295 | + charges.charge('cmd-a', 'queued'); |
| 296 | + charges.markAbandoned('cmd-a', 'queued'); |
| 297 | + charges.charge('cmd-b', 'queued'); |
| 298 | + charges.settleAnswered('cmd-b', 'queued'); |
| 299 | + |
| 300 | + expect(charges.settleTerminalEvidence('cmd-a')).toEqual({ |
| 301 | + settled: false, |
| 302 | + refused: 'no_abandoned_charge', |
| 303 | + }); |
| 304 | + }); |
| 305 | + |
| 306 | + test('ignores an id the evidence does not carry', () => { |
| 307 | + const charges = new RunnerCommandAccounting(); |
| 308 | + charges.charge('cmd-a', 'queued'); |
| 309 | + charges.markAbandoned('cmd-a', 'queued'); |
| 310 | + |
| 311 | + expect(charges.settleTerminalEvidence(undefined)).toEqual({ |
| 312 | + settled: false, |
| 313 | + refused: 'no_abandoned_charge', |
| 314 | + }); |
| 315 | + expect(charges.outstandingChargeCount).toBe(1); |
| 316 | + }); |
| 317 | + |
| 318 | + test('settles every abandoned attempt of the command the evidence names', () => { |
| 319 | + const charges = new RunnerCommandAccounting(); |
| 320 | + charges.charge('cmd-a', 'queued'); |
| 321 | + charges.markAbandoned('cmd-a', 'queued'); |
| 322 | + charges.charge('cmd-a', 'queued'); |
| 323 | + charges.markAbandoned('cmd-a', 'queued'); |
| 324 | + |
| 325 | + expect(charges.settleTerminalEvidence('cmd-a').settled).toBe(true); |
| 326 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 327 | + }); |
| 328 | +}); |
| 329 | + |
| 330 | +describe('the abandoned-then-terminal sequence the issue asks about', () => { |
| 331 | + test('keeps the surviving charge named when a queued answer forgives one of two abandonments', () => { |
| 332 | + // A abandons, B abandons, C answers. The conservative rule forgives the oldest residue, so B stays |
| 333 | + // charged — and B's own terminal evidence, not A's, is what discharges it. |
| 334 | + const charges = new RunnerCommandAccounting(); |
| 335 | + charges.charge('cmd-a', 'queued'); |
| 336 | + charges.markAbandoned('cmd-a', 'queued'); |
| 337 | + charges.charge('cmd-b', 'queued'); |
| 338 | + charges.markAbandoned('cmd-b', 'queued'); |
| 339 | + charges.charge('cmd-c', 'queued'); |
| 340 | + charges.settleAnswered('cmd-c', 'queued'); |
| 341 | + |
| 342 | + expect(charges.hasAbandonedCharges).toBe(true); |
| 343 | + expect(charges.settleTerminalEvidence('cmd-a')).toEqual({ |
| 344 | + settled: false, |
| 345 | + refused: 'no_abandoned_charge', |
| 346 | + }); |
| 347 | + expect(charges.settleTerminalEvidence('cmd-b').settled).toBe(true); |
| 348 | + expect(charges.hasOutstandingCharges).toBe(false); |
| 349 | + }); |
| 350 | +}); |
0 commit comments