Skip to content

Commit e4bc554

Browse files
feat(agent): enforce cancellation-aware switch boundary for streaming model fallback (issue #210)
ModelFallbackInterceptor matched only top-level CancellationException, so a wrapped cancellation still dispatched another provider call, and streaming failures were passed through unchanged regardless of observable output. - Detect cancellations through the cause chain (depth-bounded against cyclic chains) and terminate without switching; the unwrapped identity is kept before and after the first emitted event. Reactive paths never touch the pooled thread's interrupt flag; the synchronous path keeps its thread-aware semantics. - Switch a failing stream to the next candidate only before any event (text delta, tool call, usage, response) has been emitted; afterwards the failure terminates the stream so two models' output is never concatenated. - Check the request's ExecutionGuard before every attempt, the primary included; a dead run is terminal on both the synchronous and the streaming path. - Dispatch the streaming switch on boundedElastic so a non-streaming fallback candidate cannot park the thread that delivered the failure. - Give each subscription its own switch state, extend the aggregate failure message with the attempt sequence, and document the recommended fallback-outer/retry-inner composition order.
1 parent efd12ae commit e4bc554

4 files changed

Lines changed: 743 additions & 14 deletions

File tree

‎argi-orchestration/argi-agent-framework/src/main/java/io/github/agentic/ai/agent/orchestration/interceptor/modelfallback/ModelFallbackInterceptor.java‎

Lines changed: 217 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,38 @@
2121
import io.github.agentic.ai.agent.interceptor.ModelResponse;
2222

2323
import io.github.agentic.ai.api.chat.ChatClient;
24+
import io.github.agentic.ai.api.chat.ChatEvent;
25+
import io.github.agentic.ai.runtime.execution.ExecutionGuard;
2426

2527
import java.util.ArrayList;
2628
import java.util.List;
2729
import java.util.Objects;
2830
import java.util.concurrent.CancellationException;
31+
import java.util.concurrent.atomic.AtomicBoolean;
2932

3033
import org.slf4j.Logger;
3134
import org.slf4j.LoggerFactory;
3235

36+
import reactor.adapter.JdkFlowAdapter;
37+
import reactor.core.publisher.Flux;
38+
import reactor.core.scheduler.Schedulers;
39+
3340
/**
3441
* Automatic fallback to alternative models on errors.
3542
*
3643
* Retries failed model calls with alternative models in sequence until success or all
37-
* models exhausted. Streaming responses are passed through unchanged. Errors emitted by
38-
* the publisher after subscription are not handled by this interceptor.
44+
* models exhausted. A failure of a streaming response switches to the next candidate
45+
* only when it arrives before any event has been emitted downstream; once a text delta,
46+
* tool call, usage or response event has been observed, the failure terminates the
47+
* stream, so output of two models is never concatenated. A cancellation — however
48+
* deeply it is wrapped in the cause chain — keeps its unwrapped identity on either side
49+
* of that boundary, and it, an interruption, or a run whose {@link ExecutionGuard} is
50+
* no longer active never triggers another candidate. Each subscription of a streaming
51+
* response owns its switch sequence.
52+
*
53+
* When combined with {@code ModelRetryInterceptor}, register this interceptor first
54+
* (outermost), so each model is retried to exhaustion before the sequence fails over to
55+
* the next candidate.
3956
*
4057
* Example: ModelFallbackInterceptor interceptor = ModelFallbackInterceptor.builder()
4158
* .addFallbackClient(primaryAlternative) .addFallbackClient(secondaryAlternative)
@@ -45,6 +62,9 @@ public class ModelFallbackInterceptor extends ModelInterceptor {
4562

4663
private static final Logger log = LoggerFactory.getLogger(ModelFallbackInterceptor.class);
4764

65+
/** Bounds the cause-chain walks; a cyclic chain must not spin the thread forever. */
66+
private static final int MAX_CAUSE_DEPTH = 32;
67+
4868
private final List<ChatClient> fallbackClients;
4969

5070
private ModelFallbackInterceptor(Builder builder) {
@@ -59,9 +79,13 @@ public static Builder builder() {
5979
public ModelResponse interceptModel(ModelRequest request, ModelCallHandler handler) {
6080
Exception lastException = null;
6181

82+
// A cancelled or expired run is terminal: no provider call at all, the primary
83+
// included, mirroring the per-attempt guard check of the retry interceptor.
84+
assertRunActive(request);
85+
6286
// Try primary model first
6387
try {
64-
return handler.call(request);
88+
return withSwitchBoundary(request, handler, handler.call(request), 0);
6589
}
6690
catch (Exception e) {
6791
propagateCancellation(e);
@@ -71,11 +95,15 @@ public ModelResponse interceptModel(ModelRequest request, ModelCallHandler handl
7195

7296
// Try fallback models in sequence
7397
for (int i = 0; i < fallbackClients.size(); i++) {
98+
// A cancelled or expired run is terminal: no further provider call.
99+
assertRunActive(request);
74100
try {
75101
log.info("Trying fallback client {} of {}", i + 1, fallbackClients.size());
76102
// Reuse the request pipeline so system messages, tools and usage are
77103
// retained.
78-
return handler.call(ModelRequest.builder(request).chatClient(fallbackClients.get(i)).build());
104+
ModelResponse fallback = handler
105+
.call(ModelRequest.builder(request).chatClient(fallbackClients.get(i)).build());
106+
return withSwitchBoundary(request, handler, fallback, i + 1);
79107
}
80108
catch (Exception e) {
81109
propagateCancellation(e);
@@ -85,18 +113,197 @@ public ModelResponse interceptModel(ModelRequest request, ModelCallHandler handl
85113
}
86114

87115
// All models failed
88-
throw new RuntimeException("All clients failed after " + (fallbackClients.size() + 1) + " attempts",
116+
throw allClientsFailed(lastException);
117+
}
118+
119+
/**
120+
* Attaches the switch boundary to a candidate's streaming response. A
121+
* non-streaming response is returned unchanged. Every subscription of the returned
122+
* publisher owns its own switch state, so a repeated subscription replays the
123+
* captured candidate stream instead of resuming the fallback sequence of an earlier
124+
* one.
125+
*/
126+
private ModelResponse withSwitchBoundary(ModelRequest request, ModelCallHandler handler, ModelResponse response,
127+
int nextClientIndex) {
128+
if (!response.isStreaming()) {
129+
return response;
130+
}
131+
return ModelResponse.stream(JdkFlowAdapter.publisherToFlowPublisher(
132+
switchingFluxDeferred(request, handler, JdkFlowAdapter.flowPublisherToFlux(response.getStream()),
133+
nextClientIndex)));
134+
}
135+
136+
/**
137+
* The deferred form of the switch boundary: fresh switch state per subscription and
138+
* a subscription-time interrupt check, mirroring the retry interceptor.
139+
*/
140+
private Flux<ChatEvent> switchingFluxDeferred(ModelRequest request, ModelCallHandler handler,
141+
Flux<ChatEvent> candidateFlux, int nextClientIndex) {
142+
return Flux.defer(() -> {
143+
// Mirrors the retry interceptor: a subscribing thread that is already
144+
// interrupted must not start (or resume) a provider sequence.
145+
if (Thread.currentThread().isInterrupted()) {
146+
return Flux.<ChatEvent>error(new CancellationException("Model call interrupted"));
147+
}
148+
return switchingFlux(request, handler, candidateFlux, nextClientIndex);
149+
});
150+
}
151+
152+
/**
153+
* Wraps one candidate's stream with the switch boundary: a failure that arrives
154+
* before any event has been emitted downstream switches to the next candidate,
155+
* because no observable effect has escaped yet. Once an event — text delta, tool
156+
* call, usage or response — has been emitted, the failure terminates the stream: a
157+
* second model's output must never be appended to a partially delivered one. A
158+
* cancellation keeps its unwrapped identity on both sides of that boundary.
159+
*/
160+
private Flux<ChatEvent> switchingFlux(ModelRequest request, ModelCallHandler handler, Flux<ChatEvent> candidateFlux,
161+
int nextClientIndex) {
162+
AtomicBoolean eventEmitted = new AtomicBoolean(false);
163+
return candidateFlux.doOnNext(event -> eventEmitted.set(true)).onErrorResume(error -> {
164+
if (error instanceof Error) {
165+
return Flux.error(error);
166+
}
167+
// The error signal itself carries the cancellation truth on a reactive
168+
// thread; the thread's interrupt flag belongs to the pool, not to this run,
169+
// and is deliberately not consulted here.
170+
CancellationException cancellation = findWrappedCancellation(error);
171+
if (cancellation != null) {
172+
log.info("Streaming candidate failed with a cancellation; terminating the fallback sequence: {}",
173+
cancellation.getMessage());
174+
return Flux.error(cancellation);
175+
}
176+
if (eventEmitted.get()) {
177+
return Flux.error(error);
178+
}
179+
// Dispatch off the signal thread: a non-streaming fallback candidate would
180+
// otherwise park the thread that delivered the failure, which for a Netty
181+
// event loop stalls every other stream multiplexed on it.
182+
return Flux.defer(() -> nextClientFlux(request, handler, nextClientIndex, toException(error)))
183+
.subscribeOn(Schedulers.boundedElastic());
184+
});
185+
}
186+
187+
/**
188+
* Streams the next fallback candidate, advancing until one succeeds or the sequence
189+
* is exhausted. A cancellation, however wrapped, and a dead execution guard end the
190+
* sequence instead of dispatching another provider call.
191+
*/
192+
private Flux<ChatEvent> nextClientFlux(ModelRequest request, ModelCallHandler handler, int clientIndex,
193+
Exception lastException) {
194+
if (clientIndex >= fallbackClients.size()) {
195+
return Flux.error(allClientsFailed(lastException));
196+
}
197+
try {
198+
assertRunActive(request);
199+
}
200+
catch (Exception lostRun) {
201+
return Flux.error(lostRun);
202+
}
203+
log.info("Trying fallback client {} of {}", clientIndex + 1, fallbackClients.size());
204+
try {
205+
ModelResponse next = handler
206+
.call(ModelRequest.builder(request).chatClient(fallbackClients.get(clientIndex)).build());
207+
if (next.isStreaming()) {
208+
return switchingFluxDeferred(request, handler, JdkFlowAdapter.flowPublisherToFlux(next.getStream()),
209+
clientIndex + 1);
210+
}
211+
return Flux.just(ChatEvent.response(next.getChatResponse()));
212+
}
213+
catch (Exception e) {
214+
CancellationException cancellation = findWrappedCancellation(e);
215+
if (cancellation != null) {
216+
return Flux.error(cancellation);
217+
}
218+
log.warn("Fallback model {} failed: {}", clientIndex + 1, e.getMessage());
219+
return nextClientFlux(request, handler, clientIndex + 1, e);
220+
}
221+
}
222+
223+
private static void assertRunActive(ModelRequest request) {
224+
request.getExecutionGuard().ifPresent(ExecutionGuard::assertActive);
225+
}
226+
227+
private RuntimeException allClientsFailed(Exception lastException) {
228+
StringBuilder sequence = new StringBuilder("primary");
229+
for (int i = 1; i <= fallbackClients.size(); i++) {
230+
sequence.append(", fallback-").append(i);
231+
}
232+
return new RuntimeException(
233+
"All clients failed after " + (fallbackClients.size() + 1) + " attempts (" + sequence + ")",
89234
lastException);
90235
}
91236

237+
private static Exception toException(Throwable error) {
238+
return error instanceof Exception exception ? exception : new RuntimeException(error);
239+
}
240+
92241
private static void propagateCancellation(Exception failure) {
93-
if (failure instanceof CancellationException cancelled) {
94-
throw cancelled;
242+
CancellationException cancellation = cancellation(failure);
243+
if (cancellation != null) {
244+
throw cancellation;
245+
}
246+
}
247+
248+
/**
249+
* Finds the cancellation of a failed attempt, however deeply it is wrapped, without
250+
* touching thread state. A wrapped cancellation is still a cancellation: switching
251+
* to another model would dispatch a call the caller already aborted (issue #210).
252+
* The walk is depth-bounded, because a cyclic cause chain (constructible through
253+
* {@code Throwable.initCause} on two nodes) would otherwise spin the thread
254+
* forever.
255+
*/
256+
private static CancellationException findWrappedCancellation(Throwable failure) {
257+
Throwable cause = failure;
258+
for (int depth = 0; cause != null && depth < MAX_CAUSE_DEPTH; depth++) {
259+
if (cause instanceof CancellationException cancelled) {
260+
return cancelled;
261+
}
262+
if (cause instanceof InterruptedException interrupted) {
263+
CancellationException cancelled = new CancellationException("Model call interrupted");
264+
cancelled.initCause(interrupted);
265+
return cancelled;
266+
}
267+
cause = cause.getCause() == cause ? null : cause.getCause();
268+
}
269+
return null;
270+
}
271+
272+
/**
273+
* The cancellation of a synchronous attempt. On the caller's own thread the
274+
* interrupt flag carries the same meaning: an interruption in the chain restores
275+
* the flag so upstream blocking work also terminates, and a bare interrupt flag
276+
* converts to a cancellation. Reactive paths use
277+
* {@link #findWrappedCancellation(Throwable)} instead, because a pooled signal
278+
* thread's flag belongs to the pool, not to this run.
279+
*/
280+
private static CancellationException cancellation(Throwable failure) {
281+
CancellationException wrapped = findWrappedCancellation(failure);
282+
if (wrapped != null) {
283+
if (chainsInterruption(failure)) {
284+
Thread.currentThread().interrupt();
285+
}
286+
return wrapped;
287+
}
288+
if (!Thread.currentThread().isInterrupted()) {
289+
return null;
290+
}
291+
CancellationException cancelled = new CancellationException("Model call interrupted");
292+
if (failure != null) {
293+
cancelled.initCause(failure);
95294
}
96-
if (Thread.currentThread().isInterrupted() || failure instanceof InterruptedException) {
97-
Thread.currentThread().interrupt();
98-
throw new CancellationException("Model call interrupted");
295+
return cancelled;
296+
}
297+
298+
private static boolean chainsInterruption(Throwable failure) {
299+
Throwable cause = failure;
300+
for (int depth = 0; cause != null && depth < MAX_CAUSE_DEPTH; depth++) {
301+
if (cause instanceof InterruptedException) {
302+
return true;
303+
}
304+
cause = cause.getCause() == cause ? null : cause.getCause();
99305
}
306+
return false;
100307
}
101308

102309
@Override

‎argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/interceptors/ModelFallbackInterceptorStreamingTest.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,19 +51,28 @@ void streamingPrimaryResponseDoesNotCallFallback() {
5151
ModelResponse result = interceptor(fallback).interceptModel(ModelRequest.builder().messages(List.of()).build(),
5252
request -> primary);
5353

54-
assertSame(primary, result);
54+
// The response is wrapped by the switch boundary, but its events are delivered
55+
// unchanged and the fallback client is never invoked.
56+
assertEquals(true, result.isStreaming());
57+
Flux<ChatEvent> stream = JdkFlowAdapter.flowPublisherToFlux(result.getStream());
58+
assertEquals("primary", stream.blockLast(Duration.ofSeconds(2)).response().text());
5559
assertEquals(0, fallback.calls);
5660
}
5761

5862
@Test
59-
void publisherErrorsRemainStreamingErrors() {
63+
void publisherErrorsExhaustingAllCandidatesFailWithAggregate() {
6064
CountingModel fallback = new CountingModel("fallback");
6165
RuntimeException failure = new RuntimeException("stream failed");
6266
ModelResponse result = interceptor(fallback).interceptModel(ModelRequest.builder().messages(List.of()).build(),
6367
request -> ModelResponse.stream(JdkFlowAdapter.publisherToFlowPublisher(Flux.error(failure))));
6468

6569
Flux<ChatEvent> stream = JdkFlowAdapter.flowPublisherToFlux(result.getStream());
66-
assertSame(failure, assertThrows(RuntimeException.class, () -> stream.blockLast(Duration.ofSeconds(2))));
70+
// The handler fails for every candidate before any event is emitted, so the
71+
// switch boundary walks the whole sequence and reports the aggregate.
72+
RuntimeException aggregate = assertThrows(RuntimeException.class,
73+
() -> stream.blockLast(Duration.ofSeconds(2)));
74+
assertEquals("All clients failed after 2 attempts (primary, fallback-1)", aggregate.getMessage());
75+
assertSame(failure, aggregate.getCause());
6776
assertEquals(0, fallback.calls);
6877
}
6978

‎argi-orchestration/argi-agent-framework/src/test/java/io/github/agentic/ai/agent/orchestration/interceptors/ModelFallbackNeutralClientTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ void fallbackExhaustionPreservesOrderAndFinalCause() {
165165
assertThatThrownBy(() -> interceptor.interceptModel(ModelRequest.builder().build(), request -> {
166166
attempts.add(request.getChatClient());
167167
throw request.getChatClient() == second ? last : new IllegalStateException("Unavailable");
168-
})).hasMessage("All clients failed after 3 attempts").hasCause(last);
168+
})).hasMessage("All clients failed after 3 attempts (primary, fallback-1, fallback-2)").hasCause(last);
169169
assertThat(attempts).containsExactly(null, first, second);
170170
assertThatThrownBy(() -> ModelFallbackInterceptor.builder().build())
171171
.isInstanceOf(IllegalArgumentException.class);

0 commit comments

Comments
 (0)