2121import io .github .agentic .ai .agent .interceptor .ModelResponse ;
2222
2323import io .github .agentic .ai .api .chat .ChatClient ;
24+ import io .github .agentic .ai .api .chat .ChatEvent ;
25+ import io .github .agentic .ai .runtime .execution .ExecutionGuard ;
2426
2527import java .util .ArrayList ;
2628import java .util .List ;
2729import java .util .Objects ;
2830import java .util .concurrent .CancellationException ;
31+ import java .util .concurrent .atomic .AtomicBoolean ;
2932
3033import org .slf4j .Logger ;
3134import org .slf4j .LoggerFactory ;
3235
36+ import reactor .adapter .JdkFlowAdapter ;
37+ import reactor .core .publisher .Flux ;
38+ import reactor .core .scheduler .Schedulers ;
39+
3340/**
3441 * Automatic fallback to alternative models on errors.
3542 *
3643 * Retries failed model calls with alternative models in sequence until success or all
37- * models exhausted. Streaming responses are passed through unchanged. Errors emitted by
38- * the publisher after subscription are not handled by this interceptor.
44+ * models exhausted. A failure of a streaming response switches to the next candidate
45+ * only when it arrives before any event has been emitted downstream; once a text delta,
46+ * tool call, usage or response event has been observed, the failure terminates the
47+ * stream, so output of two models is never concatenated. A cancellation — however
48+ * deeply it is wrapped in the cause chain — keeps its unwrapped identity on either side
49+ * of that boundary, and it, an interruption, or a run whose {@link ExecutionGuard} is
50+ * no longer active never triggers another candidate. Each subscription of a streaming
51+ * response owns its switch sequence.
52+ *
53+ * When combined with {@code ModelRetryInterceptor}, register this interceptor first
54+ * (outermost), so each model is retried to exhaustion before the sequence fails over to
55+ * the next candidate.
3956 *
4057 * Example: ModelFallbackInterceptor interceptor = ModelFallbackInterceptor.builder()
4158 * .addFallbackClient(primaryAlternative) .addFallbackClient(secondaryAlternative)
@@ -45,6 +62,9 @@ public class ModelFallbackInterceptor extends ModelInterceptor {
4562
4663 private static final Logger log = LoggerFactory .getLogger (ModelFallbackInterceptor .class );
4764
65+ /** Bounds the cause-chain walks; a cyclic chain must not spin the thread forever. */
66+ private static final int MAX_CAUSE_DEPTH = 32 ;
67+
4868 private final List <ChatClient > fallbackClients ;
4969
5070 private ModelFallbackInterceptor (Builder builder ) {
@@ -59,9 +79,13 @@ public static Builder builder() {
5979 public ModelResponse interceptModel (ModelRequest request , ModelCallHandler handler ) {
6080 Exception lastException = null ;
6181
82+ // A cancelled or expired run is terminal: no provider call at all, the primary
83+ // included, mirroring the per-attempt guard check of the retry interceptor.
84+ assertRunActive (request );
85+
6286 // Try primary model first
6387 try {
64- return handler .call (request );
88+ return withSwitchBoundary ( request , handler , handler .call (request ), 0 );
6589 }
6690 catch (Exception e ) {
6791 propagateCancellation (e );
@@ -71,11 +95,15 @@ public ModelResponse interceptModel(ModelRequest request, ModelCallHandler handl
7195
7296 // Try fallback models in sequence
7397 for (int i = 0 ; i < fallbackClients .size (); i ++) {
98+ // A cancelled or expired run is terminal: no further provider call.
99+ assertRunActive (request );
74100 try {
75101 log .info ("Trying fallback client {} of {}" , i + 1 , fallbackClients .size ());
76102 // Reuse the request pipeline so system messages, tools and usage are
77103 // retained.
78- return handler .call (ModelRequest .builder (request ).chatClient (fallbackClients .get (i )).build ());
104+ ModelResponse fallback = handler
105+ .call (ModelRequest .builder (request ).chatClient (fallbackClients .get (i )).build ());
106+ return withSwitchBoundary (request , handler , fallback , i + 1 );
79107 }
80108 catch (Exception e ) {
81109 propagateCancellation (e );
@@ -85,18 +113,197 @@ public ModelResponse interceptModel(ModelRequest request, ModelCallHandler handl
85113 }
86114
87115 // All models failed
88- throw new RuntimeException ("All clients failed after " + (fallbackClients .size () + 1 ) + " attempts" ,
116+ throw allClientsFailed (lastException );
117+ }
118+
119+ /**
120+ * Attaches the switch boundary to a candidate's streaming response. A
121+ * non-streaming response is returned unchanged. Every subscription of the returned
122+ * publisher owns its own switch state, so a repeated subscription replays the
123+ * captured candidate stream instead of resuming the fallback sequence of an earlier
124+ * one.
125+ */
126+ private ModelResponse withSwitchBoundary (ModelRequest request , ModelCallHandler handler , ModelResponse response ,
127+ int nextClientIndex ) {
128+ if (!response .isStreaming ()) {
129+ return response ;
130+ }
131+ return ModelResponse .stream (JdkFlowAdapter .publisherToFlowPublisher (
132+ switchingFluxDeferred (request , handler , JdkFlowAdapter .flowPublisherToFlux (response .getStream ()),
133+ nextClientIndex )));
134+ }
135+
136+ /**
137+ * The deferred form of the switch boundary: fresh switch state per subscription and
138+ * a subscription-time interrupt check, mirroring the retry interceptor.
139+ */
140+ private Flux <ChatEvent > switchingFluxDeferred (ModelRequest request , ModelCallHandler handler ,
141+ Flux <ChatEvent > candidateFlux , int nextClientIndex ) {
142+ return Flux .defer (() -> {
143+ // Mirrors the retry interceptor: a subscribing thread that is already
144+ // interrupted must not start (or resume) a provider sequence.
145+ if (Thread .currentThread ().isInterrupted ()) {
146+ return Flux .<ChatEvent >error (new CancellationException ("Model call interrupted" ));
147+ }
148+ return switchingFlux (request , handler , candidateFlux , nextClientIndex );
149+ });
150+ }
151+
152+ /**
153+ * Wraps one candidate's stream with the switch boundary: a failure that arrives
154+ * before any event has been emitted downstream switches to the next candidate,
155+ * because no observable effect has escaped yet. Once an event — text delta, tool
156+ * call, usage or response — has been emitted, the failure terminates the stream: a
157+ * second model's output must never be appended to a partially delivered one. A
158+ * cancellation keeps its unwrapped identity on both sides of that boundary.
159+ */
160+ private Flux <ChatEvent > switchingFlux (ModelRequest request , ModelCallHandler handler , Flux <ChatEvent > candidateFlux ,
161+ int nextClientIndex ) {
162+ AtomicBoolean eventEmitted = new AtomicBoolean (false );
163+ return candidateFlux .doOnNext (event -> eventEmitted .set (true )).onErrorResume (error -> {
164+ if (error instanceof Error ) {
165+ return Flux .error (error );
166+ }
167+ // The error signal itself carries the cancellation truth on a reactive
168+ // thread; the thread's interrupt flag belongs to the pool, not to this run,
169+ // and is deliberately not consulted here.
170+ CancellationException cancellation = findWrappedCancellation (error );
171+ if (cancellation != null ) {
172+ log .info ("Streaming candidate failed with a cancellation; terminating the fallback sequence: {}" ,
173+ cancellation .getMessage ());
174+ return Flux .error (cancellation );
175+ }
176+ if (eventEmitted .get ()) {
177+ return Flux .error (error );
178+ }
179+ // Dispatch off the signal thread: a non-streaming fallback candidate would
180+ // otherwise park the thread that delivered the failure, which for a Netty
181+ // event loop stalls every other stream multiplexed on it.
182+ return Flux .defer (() -> nextClientFlux (request , handler , nextClientIndex , toException (error )))
183+ .subscribeOn (Schedulers .boundedElastic ());
184+ });
185+ }
186+
187+ /**
188+ * Streams the next fallback candidate, advancing until one succeeds or the sequence
189+ * is exhausted. A cancellation, however wrapped, and a dead execution guard end the
190+ * sequence instead of dispatching another provider call.
191+ */
192+ private Flux <ChatEvent > nextClientFlux (ModelRequest request , ModelCallHandler handler , int clientIndex ,
193+ Exception lastException ) {
194+ if (clientIndex >= fallbackClients .size ()) {
195+ return Flux .error (allClientsFailed (lastException ));
196+ }
197+ try {
198+ assertRunActive (request );
199+ }
200+ catch (Exception lostRun ) {
201+ return Flux .error (lostRun );
202+ }
203+ log .info ("Trying fallback client {} of {}" , clientIndex + 1 , fallbackClients .size ());
204+ try {
205+ ModelResponse next = handler
206+ .call (ModelRequest .builder (request ).chatClient (fallbackClients .get (clientIndex )).build ());
207+ if (next .isStreaming ()) {
208+ return switchingFluxDeferred (request , handler , JdkFlowAdapter .flowPublisherToFlux (next .getStream ()),
209+ clientIndex + 1 );
210+ }
211+ return Flux .just (ChatEvent .response (next .getChatResponse ()));
212+ }
213+ catch (Exception e ) {
214+ CancellationException cancellation = findWrappedCancellation (e );
215+ if (cancellation != null ) {
216+ return Flux .error (cancellation );
217+ }
218+ log .warn ("Fallback model {} failed: {}" , clientIndex + 1 , e .getMessage ());
219+ return nextClientFlux (request , handler , clientIndex + 1 , e );
220+ }
221+ }
222+
223+ private static void assertRunActive (ModelRequest request ) {
224+ request .getExecutionGuard ().ifPresent (ExecutionGuard ::assertActive );
225+ }
226+
227+ private RuntimeException allClientsFailed (Exception lastException ) {
228+ StringBuilder sequence = new StringBuilder ("primary" );
229+ for (int i = 1 ; i <= fallbackClients .size (); i ++) {
230+ sequence .append (", fallback-" ).append (i );
231+ }
232+ return new RuntimeException (
233+ "All clients failed after " + (fallbackClients .size () + 1 ) + " attempts (" + sequence + ")" ,
89234 lastException );
90235 }
91236
237+ private static Exception toException (Throwable error ) {
238+ return error instanceof Exception exception ? exception : new RuntimeException (error );
239+ }
240+
92241 private static void propagateCancellation (Exception failure ) {
93- if (failure instanceof CancellationException cancelled ) {
94- throw cancelled ;
242+ CancellationException cancellation = cancellation (failure );
243+ if (cancellation != null ) {
244+ throw cancellation ;
245+ }
246+ }
247+
248+ /**
249+ * Finds the cancellation of a failed attempt, however deeply it is wrapped, without
250+ * touching thread state. A wrapped cancellation is still a cancellation: switching
251+ * to another model would dispatch a call the caller already aborted (issue #210).
252+ * The walk is depth-bounded, because a cyclic cause chain (constructible through
253+ * {@code Throwable.initCause} on two nodes) would otherwise spin the thread
254+ * forever.
255+ */
256+ private static CancellationException findWrappedCancellation (Throwable failure ) {
257+ Throwable cause = failure ;
258+ for (int depth = 0 ; cause != null && depth < MAX_CAUSE_DEPTH ; depth ++) {
259+ if (cause instanceof CancellationException cancelled ) {
260+ return cancelled ;
261+ }
262+ if (cause instanceof InterruptedException interrupted ) {
263+ CancellationException cancelled = new CancellationException ("Model call interrupted" );
264+ cancelled .initCause (interrupted );
265+ return cancelled ;
266+ }
267+ cause = cause .getCause () == cause ? null : cause .getCause ();
268+ }
269+ return null ;
270+ }
271+
272+ /**
273+ * The cancellation of a synchronous attempt. On the caller's own thread the
274+ * interrupt flag carries the same meaning: an interruption in the chain restores
275+ * the flag so upstream blocking work also terminates, and a bare interrupt flag
276+ * converts to a cancellation. Reactive paths use
277+ * {@link #findWrappedCancellation(Throwable)} instead, because a pooled signal
278+ * thread's flag belongs to the pool, not to this run.
279+ */
280+ private static CancellationException cancellation (Throwable failure ) {
281+ CancellationException wrapped = findWrappedCancellation (failure );
282+ if (wrapped != null ) {
283+ if (chainsInterruption (failure )) {
284+ Thread .currentThread ().interrupt ();
285+ }
286+ return wrapped ;
287+ }
288+ if (!Thread .currentThread ().isInterrupted ()) {
289+ return null ;
290+ }
291+ CancellationException cancelled = new CancellationException ("Model call interrupted" );
292+ if (failure != null ) {
293+ cancelled .initCause (failure );
95294 }
96- if (Thread .currentThread ().isInterrupted () || failure instanceof InterruptedException ) {
97- Thread .currentThread ().interrupt ();
98- throw new CancellationException ("Model call interrupted" );
295+ return cancelled ;
296+ }
297+
298+ private static boolean chainsInterruption (Throwable failure ) {
299+ Throwable cause = failure ;
300+ for (int depth = 0 ; cause != null && depth < MAX_CAUSE_DEPTH ; depth ++) {
301+ if (cause instanceof InterruptedException ) {
302+ return true ;
303+ }
304+ cause = cause .getCause () == cause ? null : cause .getCause ();
99305 }
306+ return false ;
100307 }
101308
102309 @ Override
0 commit comments