@@ -454,6 +454,7 @@ async fn state_built(
454454 . await
455455 . unwrap ( ) ;
456456 let state = AppState {
457+ inflight_requests : Arc :: new ( std:: sync:: atomic:: AtomicUsize :: new ( 0 ) ) ,
457458 rt,
458459 catalog : Arc :: new ( RwLock :: new ( providers:: catalog ( ) ) ) ,
459460 checkpoints : tools:: checkpoint_pool ( data. join ( "checkpoints" ) ) ,
@@ -2649,20 +2650,84 @@ async fn pull_request_rpcs_return_typed_unavailable_not_die_defect() {
26492650 }
26502651}
26512652
2653+ /// The applied-ack carries a sequence — and ONLY for a command that was applied.
2654+ ///
2655+ /// REWRITTEN FOR #433/#446, and the rewrite is the point, so it is spelled out
2656+ /// rather than quietly done.
2657+ ///
2658+ /// This test used to dispatch `{"type": "noop"}` — a type that appears nowhere
2659+ /// in `DispatchableClientOrchestrationCommand` — and assert
2660+ /// `exit._tag == "Success"` with a numeric sequence. It passed. It passed
2661+ /// because the dispatcher's catch-all was `_ => {}` falling through to the
2662+ /// applied-ack, which is precisely the defect #433 enumerated and #446 traced to
2663+ /// its generator. So the suite was not merely blind to those 15 silent no-ops:
2664+ /// it PINNED them. Any worker who fixed the catch-all got a red test that read
2665+ /// like a regression, which is a good way to make a real fix look wrong twice
2666+ /// and get reverted.
2667+ ///
2668+ /// That is why this is a rewrite and not a deletion. The property the old name
2669+ /// promised — "dispatch acks with sequence" — is real and still worth holding.
2670+ /// What was wrong was the input chosen to demonstrate it: an unimplemented
2671+ /// command is the one case where the ack must NOT happen. So the test now
2672+ /// asserts the property on a command that is genuinely implemented, and adds
2673+ /// the boundary that the old version had inverted.
2674+ ///
2675+ /// Both halves in one test on purpose: the ack and the refusal are the same
2676+ /// decision read from two sides, and splitting them invites one to be updated
2677+ /// without the other.
26522678#[ tokio:: test]
2653- async fn dispatch_acks_with_sequence ( ) {
2679+ async fn dispatch_acks_an_implemented_command_with_a_sequence_and_refuses_an_unimplemented_one ( ) {
26542680 let ( state, _d) = test_state ( ) . await ;
2681+
2682+ // HALF ONE — an IMPLEMENTED command still acks, and still carries the
2683+ // sequence. `project.meta.update` is handled at server_main.rs and defaults
2684+ // `projectId` to the seeded `p-workspace`, so it exercises the real applied
2685+ // path rather than an error path that happens to also return.
26552686 let ( tx, mut rx) = mpsc:: unbounded_channel ( ) ;
26562687 request (
26572688 & state,
26582689 & tx,
26592690 "orchestration.dispatchCommand" ,
2660- json ! ( { "input" : { "type" : "noop" } } ) ,
2691+ json ! ( { "input" : { "type" : "project.meta.update" , "patch" : { "name" : "renamed" } } } ) ,
26612692 )
26622693 . await ;
26632694 let frames = drain ( & mut rx) ;
2664- assert_eq ! ( frames[ 0 ] [ "exit" ] [ "_tag" ] , "Success" ) ;
2665- assert ! ( frames[ 0 ] [ "exit" ] [ "value" ] [ "sequence" ] . is_number( ) ) ;
2695+ assert_eq ! (
2696+ frames[ 0 ] [ "exit" ] [ "_tag" ] , "Success" ,
2697+ "an implemented command must still ack as applied: {:?}" ,
2698+ frames[ 0 ]
2699+ ) ;
2700+ assert ! (
2701+ frames[ 0 ] [ "exit" ] [ "value" ] [ "sequence" ] . is_number( ) ,
2702+ "the applied-ack must still carry the durable sequence: {:?}" ,
2703+ frames[ 0 ]
2704+ ) ;
2705+
2706+ // HALF TWO — the boundary the old test had backwards. An unimplemented
2707+ // command must be REFUSED, not acked. `noop` is the exact input the previous
2708+ // version asserted Success on.
2709+ let ( tx2, mut rx2) = mpsc:: unbounded_channel ( ) ;
2710+ request (
2711+ & state,
2712+ & tx2,
2713+ "orchestration.dispatchCommand" ,
2714+ json ! ( { "input" : { "type" : "noop" } } ) ,
2715+ )
2716+ . await ;
2717+ let frames2 = drain ( & mut rx2) ;
2718+ assert_ne ! (
2719+ frames2[ 0 ] [ "exit" ] [ "_tag" ] , "Success" ,
2720+ "#433/#446: an unimplemented command acked as APPLIED. The client applies \
2721+ optimistically and the ack confirms, so the user's change is shown and \
2722+ then silently gone on the next reload. Got: {:?}",
2723+ frames2[ 0 ]
2724+ ) ;
2725+ let rendered = frames2[ 0 ] . to_string ( ) ;
2726+ assert ! (
2727+ rendered. contains( "noop" ) ,
2728+ "the refusal must name the command type it refused, or the user cannot \
2729+ report which action did nothing: {rendered}"
2730+ ) ;
26662731}
26672732
26682733/// #35: the advertised catalog is the reconciled registry, not two
@@ -9120,6 +9185,7 @@ async fn the_shell_snapshot_reads_projects_from_the_durable_store() {
91209185 Err ( e) => panic ! ( "project store unreadable: {e}" ) ,
91219186 }
91229187 let state_b = AppState {
9188+ inflight_requests : Arc :: new ( std:: sync:: atomic:: AtomicUsize :: new ( 0 ) ) ,
91239189 rt : rt_b,
91249190 catalog : state_a. catalog . clone ( ) ,
91259191 checkpoints : tools:: checkpoint_pool ( data. join ( "checkpoints" ) ) ,
@@ -11915,3 +11981,117 @@ async fn the_read_loop_answers_a_ping_while_slow_requests_are_still_running() {
1191511981
1191611982 server. abort ( ) ;
1191711983}
11984+
11985+ /// PROOF (#476): a disconnecting client takes its in-flight requests with it.
11986+ ///
11987+ /// THE DEFECT THIS PINS, AND IT WAS MINE. The first fix for #432 replaced the
11988+ /// read loop's inline `dispatch_ws_frame(...).await` with a bare
11989+ /// `tokio::spawn`. That un-starves the reader — the #432 test above proves it —
11990+ /// and in the same line it detaches the work. `orchestration.subscribeThread`
11991+ /// attaches a durable tail and streams for the life of the thread, so a client
11992+ /// that disconnects mid-stream left a task writing into a `tx` whose receiver
11993+ /// was gone, holding an `AppState` clone, forever. Reconnect and there are two.
11994+ /// Serialization had been the backpressure, and removing it put nothing in its
11995+ /// place.
11996+ ///
11997+ /// clau-f11b caught it and claude-4e91 had already written the better shape: a
11998+ /// `JoinSet` OWNED by `handle_socket`, so the tasks cannot outlive the
11999+ /// connection. That is what this asserts.
12000+ ///
12001+ /// WHY THE #432 TEST CANNOT COVER THIS. #432 asks "does the loop keep READING"
12002+ /// — the bare spawn passes it. #476 asks "is the spawned work OWNED", and the
12003+ /// two implementations are byte-identical on the wire; the difference only ever
12004+ /// shows up as a leak, much later, in a process nobody is looking at. So this
12005+ /// asserts on `AppState::inflight_requests`, which is an OBSERVABLE and not a
12006+ /// control: nothing reads it to make a decision, so its presence cannot change
12007+ /// the behaviour it measures.
12008+ ///
12009+ /// NOT A TIMER. The assertion is that the count RETURNS TO ITS PRE-CONNECT
12010+ /// VALUE, polled under a bounded deadline. A test that slept and then checked
12011+ /// would pass on a detached task that merely happened to finish.
12012+ #[ tokio:: test( flavor = "multi_thread" , worker_threads = 4 ) ]
12013+ async fn a_disconnecting_client_takes_its_in_flight_requests_with_it ( ) {
12014+ use futures_util:: { SinkExt , StreamExt } ;
12015+
12016+ let ( state, _dir) = test_state ( ) . await ;
12017+ let counter = state. inflight_requests . clone ( ) ;
12018+ let app = super :: build_app ( state) ;
12019+
12020+ let listener = tokio:: net:: TcpListener :: bind ( "127.0.0.1:0" ) . await . unwrap ( ) ;
12021+ let addr = listener. local_addr ( ) . unwrap ( ) ;
12022+ let server = tokio:: spawn ( async move {
12023+ let _ = axum:: serve ( listener, app) . await ;
12024+ } ) ;
12025+
12026+ assert_eq ! (
12027+ counter. load( std:: sync:: atomic:: Ordering :: SeqCst ) ,
12028+ 0 ,
12029+ "nothing is in flight before a client connects"
12030+ ) ;
12031+
12032+ let ( mut sock, _resp) = tokio_tungstenite:: connect_async ( format ! ( "ws://{addr}/ws" ) )
12033+ . await
12034+ . expect ( "WS upgrade" ) ;
12035+
12036+ // Three slow-but-real requests, so there is genuinely something in flight
12037+ // when the socket goes away. Same handler the #432 test uses (it forks `ps`)
12038+ // and for the same reason: a test-only sleep method would prove the backdoor.
12039+ for i in 0 ..3 {
12040+ sock. send ( tokio_tungstenite:: tungstenite:: Message :: Text (
12041+ json ! ( {
12042+ "_tag" : "Request" ,
12043+ "id" : format!( "doomed-{i}" ) ,
12044+ "tag" : "server.getProcessDiagnostics" ,
12045+ "payload" : { } ,
12046+ } )
12047+ . to_string ( )
12048+ . into ( ) ,
12049+ ) )
12050+ . await
12051+ . expect ( "send" ) ;
12052+ }
12053+
12054+ // Wait until the server has actually PICKED THEM UP, so the drop below is a
12055+ // disconnect-with-work-in-flight and not a race the test wins by accident.
12056+ // If this never rises, the test is not exercising #476 at all and says so
12057+ // rather than passing vacuously.
12058+ let deadline = tokio:: time:: Instant :: now ( ) + std:: time:: Duration :: from_secs ( 30 ) ;
12059+ let mut peak = 0usize ;
12060+ while tokio:: time:: Instant :: now ( ) < deadline {
12061+ peak = peak. max ( counter. load ( std:: sync:: atomic:: Ordering :: SeqCst ) ) ;
12062+ if peak > 0 {
12063+ break ;
12064+ }
12065+ tokio:: task:: yield_now ( ) . await ;
12066+ }
12067+ assert ! (
12068+ peak > 0 ,
12069+ "#476: no request task was ever observed in flight, so this test never \
12070+ reached the condition it exists to check. Either the spawn is gone or \
12071+ the counter is not wired — both make the assertion below vacuous."
12072+ ) ;
12073+
12074+ // THE DISCONNECT.
12075+ drop ( sock) ;
12076+
12077+ // THE ASSERTION: in-flight returns to zero because the tasks were OWNED, not
12078+ // because they happened to finish. Bounded poll, no sleep-then-check.
12079+ let deadline = tokio:: time:: Instant :: now ( ) + std:: time:: Duration :: from_secs ( 30 ) ;
12080+ let mut last = usize:: MAX ;
12081+ while tokio:: time:: Instant :: now ( ) < deadline {
12082+ last = counter. load ( std:: sync:: atomic:: Ordering :: SeqCst ) ;
12083+ if last == 0 {
12084+ break ;
12085+ }
12086+ tokio:: task:: yield_now ( ) . await ;
12087+ }
12088+ assert_eq ! (
12089+ last, 0 ,
12090+ "#476: {last} request task(s) outlived the socket that created them. A \
12091+ bare `tokio::spawn` detaches, so subscribeThread keeps streaming into a \
12092+ `tx` whose receiver is gone and holds an AppState clone per dead client. \
12093+ The JoinSet owned by handle_socket drops them with the connection."
12094+ ) ;
12095+
12096+ server. abort ( ) ;
12097+ }
0 commit comments