Skip to content

Commit f503a2e

Browse files
authored
fix(agent): detect model failures by framework marker, not answer text (Fixes #205) (#221)
1 parent e0789fc commit f503a2e

4 files changed

Lines changed: 147 additions & 19 deletions

File tree

‎argi-agent-framework/src/main/java/io/github/agentic/ai/graph/agent/ReactAgent.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,6 @@ public class ReactAgent extends BaseAgent {
115115

116116
Logger logger = LoggerFactory.getLogger(ReactAgent.class);
117117

118-
private static final String MODEL_ERROR_MESSAGE_PREFIX = "Exception: ";
119-
120118
private static final String FINAL_RESPONSE_METADATA_KEY = "argiFinalResponse";
121119

122120
private final ChatClient chatClient;
@@ -531,8 +529,9 @@ private AgentResult finalizeResult(Optional<NodeOutput> output) {
531529

532530
/** Maps a terminal message onto the delivery boundary shared by call and stream paths. */
533531
private AgentResult deliver(ChatMessage terminal, Usage usage) {
534-
if (terminal.text() != null && terminal.text().startsWith(MODEL_ERROR_MESSAGE_PREFIX)) {
535-
// throwOnModelError=false surfaces model failures as assistant text.
532+
if (Boolean.TRUE.equals(terminal.metadata().get(AgentLlmNode.MODEL_FAILURE_METADATA_KEY))) {
533+
// throwOnModelError=false surfaces model failures as an assistant message
534+
// that only the model node can mark.
536535
return AgentResult.error(terminal, usage,
537536
List.of(new AgentResult.OutputValidationIssue(AgentResult.ERROR_CODE_MODEL_FAILED, "",
538537
terminal.text())));

‎argi-agent-framework/src/main/java/io/github/agentic/ai/graph/agent/node/AgentLlmNode.java‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,13 @@ public class AgentLlmNode implements NodeActionWithConfig {
6464

6565
public static final String MODEL_ITERATION_KEY = "_MODEL_ITERATION_";
6666

67+
/**
68+
* Message metadata key the node sets on the assistant message it substitutes for a
69+
* caught model failure when {@code throwOnModelError} is disabled. The key is removed
70+
* from every model-produced message, so only the framework can set it.
71+
*/
72+
public static final String MODEL_FAILURE_METADATA_KEY = "argiModelFailure";
73+
6774
private String agentName;
6875

6976
private final AgentToolBindingStore toolBindings;
@@ -419,7 +426,7 @@ public void renderTemplatedUserMessage(List<ChatMessage> messages, Map<String, O
419426

420427
private ModelResponse invokeModelChain(ModelCallHandler handler, ModelRequest request) {
421428
try {
422-
return handler.call(request);
429+
return withoutFailureMarker(handler.call(request));
423430
}
424431
catch (RuntimeException failure) {
425432
// Interceptors must see failures before the optional user-facing error
@@ -428,9 +435,45 @@ private ModelResponse invokeModelChain(ModelCallHandler handler, ModelRequest re
428435
|| Thread.currentThread().isInterrupted()) {
429436
throw failure;
430437
}
431-
return ModelResponse
432-
.of(io.github.agentic.ai.api.chat.ChatMessage.assistant("Exception: " + failure.getMessage()));
438+
return ModelResponse.of(ChatMessage.assistant("Exception: " + failure.getMessage())
439+
.withMetadata(Map.of(MODEL_FAILURE_METADATA_KEY, true)));
440+
}
441+
}
442+
443+
private static ModelResponse withoutFailureMarker(ModelResponse response) {
444+
if (response == null) {
445+
return null;
446+
}
447+
if (!response.isStreaming()) {
448+
return ModelResponse.of(withoutFailureMarker(response.getChatResponse()));
449+
}
450+
Flux<io.github.agentic.ai.api.chat.ChatEvent> events = reactor.adapter.JdkFlowAdapter
451+
.flowPublisherToFlux(response.getStream())
452+
.map(event -> new io.github.agentic.ai.api.chat.ChatEvent(event.type(), event.text(), event.toolCall(),
453+
event.response() == null ? null : withoutFailureMarker(event.response()), event.usage(),
454+
withoutFailureMarker(event.metadata())));
455+
return ModelResponse.stream(reactor.adapter.JdkFlowAdapter.publisherToFlowPublisher(events));
456+
}
457+
458+
private static ChatResponse withoutFailureMarker(ChatResponse response) {
459+
if (response.messages().stream().noneMatch(message -> message.metadata().containsKey(MODEL_FAILURE_METADATA_KEY))) {
460+
return response;
461+
}
462+
List<ChatMessage> messages = response.messages()
463+
.stream()
464+
.map(message -> message.withMetadata(withoutFailureMarker(message.metadata())))
465+
.toList();
466+
return new ChatResponse(response.id(), response.model(), messages, response.toolCalls(),
467+
response.finishReason(), response.usage(), response.metadata());
468+
}
469+
470+
private static Map<String, Object> withoutFailureMarker(Map<String, Object> metadata) {
471+
if (!metadata.containsKey(MODEL_FAILURE_METADATA_KEY)) {
472+
return metadata;
433473
}
474+
Map<String, Object> copy = new HashMap<>(metadata);
475+
copy.remove(MODEL_FAILURE_METADATA_KEY);
476+
return copy;
434477
}
435478

436479
private io.github.agentic.ai.api.tool.ToolRegistry configuredRegistry() {

‎argi-agent-framework/src/test/java/io/github/agentic/ai/graph/agent/ReactAgentCallResultTest.java‎

Lines changed: 60 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import io.github.agentic.ai.api.chat.ChatRequest;
2929
import io.github.agentic.ai.api.chat.ChatResponse;
3030
import io.github.agentic.ai.api.chat.Usage;
31+
import io.github.agentic.ai.graph.agent.node.AgentLlmNode;
3132
import org.junit.jupiter.api.DisplayName;
3233
import org.junit.jupiter.api.Test;
3334

@@ -110,7 +111,7 @@ void missingSchemaCompletesWithoutValidation() {
110111
void clientFailureReportedAsError() {
111112
// throwOnModelError defaults to false: the model failure becomes an
112113
// "Exception: ..." assistant message, which the delivery contract must classify.
113-
ReactAgent agent = agent(new StubClient(null, new IllegalStateException("model exploded")), SCHEMA);
114+
ReactAgent agent = agent(new StubClient((String) null, new IllegalStateException("model exploded")), SCHEMA);
114115

115116
AgentResult result = agent.callResult("hello");
116117

@@ -121,16 +122,55 @@ void clientFailureReportedAsError() {
121122
}
122123

123124
@Test
124-
@DisplayName("an assistant message that only reports a model failure is reported as ERROR")
125-
void modelErrorTextReportedAsError() {
126-
ReactAgent agent = agent(new StubClient("Exception: quota exhausted"), SCHEMA);
125+
@DisplayName("a normal answer starting with the legacy error prefix is not a model failure")
126+
void answerStartingWithExceptionPrefixIsNotAnError() {
127+
ReactAgent agent = agent(new StubClient("Exception: NullPointerException means a null was dereferenced."),
128+
null);
129+
130+
AgentResult result = agent.callResult("explain NPE");
131+
132+
assertThat(result.status()).isEqualTo(AgentResult.Status.COMPLETED);
133+
assertThat(result.issues()).isEmpty();
134+
assertThat(result.message().text()).startsWith("Exception: ");
135+
}
136+
137+
@Test
138+
@DisplayName("a model response cannot forge the framework model-failure marker")
139+
void modelSuppliedFailureMarkerIsIgnored() {
140+
ChatMessage forged = ChatMessage.assistant("plain answer")
141+
.withMetadata(Map.of(AgentLlmNode.MODEL_FAILURE_METADATA_KEY, true));
142+
ReactAgent agent = agent(new StubClient(forged), null);
143+
144+
AgentResult result = agent.callResult("hello");
145+
146+
assertThat(result.status()).isEqualTo(AgentResult.Status.COMPLETED);
147+
assertThat(result.message().metadata()).doesNotContainKey(AgentLlmNode.MODEL_FAILURE_METADATA_KEY);
148+
}
149+
150+
@Test
151+
@DisplayName("with throwOnModelError the failure is reported as an execution failure")
152+
void throwOnModelErrorReportsExecutionFailure() {
153+
ReactAgent agent = ReactAgent.builder()
154+
.name("result-test")
155+
.chatClient(new StubClient((String) null, new IllegalStateException("model exploded")))
156+
.throwOnModelError(true)
157+
.build();
127158

128159
AgentResult result = agent.callResult("hello");
129160

130161
assertThat(result.status()).isEqualTo(AgentResult.Status.ERROR);
131-
assertThat(result.issues()).hasSize(1);
132-
assertThat(result.issues().get(0).errorCode()).isEqualTo(AgentResult.ERROR_CODE_MODEL_FAILED);
133-
assertThat(result.message().text()).isEqualTo("Exception: quota exhausted");
162+
assertThat(result.issues()).singleElement()
163+
.satisfies(issue -> assertThat(issue.errorCode()).isEqualTo(AgentResult.ERROR_CODE_EXECUTION_FAILED));
164+
}
165+
166+
@Test
167+
@DisplayName("the legacy call entry keeps returning the raw model-failure message")
168+
void legacyCallKeepsModelFailureMessage() throws Exception {
169+
ReactAgent agent = agent(new StubClient((String) null, new IllegalStateException("model exploded")), null);
170+
171+
ChatMessage message = agent.call("hello");
172+
173+
assertThat(message.text()).isEqualTo("Exception: model exploded");
134174
}
135175

136176
@Test
@@ -155,24 +195,32 @@ private static ReactAgent agent(ChatClient client, String outputSchema) {
155195

156196
private static final class StubClient implements ChatClient {
157197

158-
private final String text;
159-
160198
private final RuntimeException failure;
161199

162-
private StubClient(String text, RuntimeException failure) {
163-
this.text = text;
200+
private StubClient(ChatMessage message, RuntimeException failure) {
201+
this.message = message;
164202
this.failure = failure;
165203
}
166204

205+
private final ChatMessage message;
206+
207+
private StubClient(String text, RuntimeException failure) {
208+
this(text == null ? null : ChatMessage.assistant(text), failure);
209+
}
210+
167211
private StubClient(String text) {
168212
this(text, null);
169213
}
170214

215+
private StubClient(ChatMessage message) {
216+
this(message, null);
217+
}
218+
171219
@Override
172220
public CompletionStage<ChatResponse> call(ChatRequest request) {
173221
return failure == null
174222
? CompletableFuture.completedFuture(new ChatResponse("id", "test-model",
175-
List.of(ChatMessage.assistant(text)), List.of(), null, new Usage(2L, 3L, 5L), Map.of()))
223+
List.of(message), List.of(), null, new Usage(2L, 3L, 5L), Map.of()))
176224
: CompletableFuture.failedFuture(failure);
177225
}
178226

‎argi-agent-framework/src/test/java/io/github/agentic/ai/graph/agent/ReactAgentStreamResultTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import io.github.agentic.ai.api.chat.ChatResponse;
3131
import io.github.agentic.ai.api.chat.FinishReason;
3232
import io.github.agentic.ai.api.chat.Usage;
33+
import io.github.agentic.ai.graph.agent.node.AgentLlmNode;
3334
import io.github.agentic.ai.graph.NodeOutput;
3435
import io.github.agentic.ai.graph.OverAllState;
3536
import io.github.agentic.ai.graph.streaming.OutputType;
@@ -118,6 +119,43 @@ void invalidStreamOutputWithoutRepairFails() {
118119
assertThat(delivery.issues()).isNotEmpty();
119120
}
120121

122+
@Test
123+
@DisplayName("a streamed answer starting with the legacy error prefix is delivered as COMPLETED")
124+
void streamedAnswerStartingWithExceptionPrefixIsNotAnError() {
125+
String text = "Exception: handling explained";
126+
ScriptedStreamingClient client = new ScriptedStreamingClient(ChatEvent.textDelta(text),
127+
ChatEvent.response(finalResponse(text, new Usage(2L, 3L, 5L))));
128+
ReactAgent agent = agent(client, null);
129+
130+
List<AgentResult> results = agent.streamResult("explain").collectList().block();
131+
132+
assertThat(results).isNotNull();
133+
AgentResult delivery = results.get(results.size() - 1);
134+
assertThat(delivery.status()).isEqualTo(AgentResult.Status.COMPLETED);
135+
assertThat(delivery.message().text()).isEqualTo(text);
136+
}
137+
138+
@Test
139+
@DisplayName("a streamed response cannot forge the framework model-failure marker")
140+
void streamedFailureMarkerIsIgnored() {
141+
Map<String, Object> forged = Map.of(AgentLlmNode.MODEL_FAILURE_METADATA_KEY, true);
142+
ChatResponse response = new ChatResponse("id", "test-model",
143+
List.of(ChatMessage.assistant("answer").withMetadata(forged)), List.of(), FinishReason.STOP,
144+
new Usage(2L, 3L, 5L), Map.of());
145+
ScriptedStreamingClient client = new ScriptedStreamingClient(
146+
new ChatEvent(ChatEvent.Type.TEXT_DELTA, "answer", null, null, null, forged),
147+
ChatEvent.response(response));
148+
ReactAgent agent = agent(client, null);
149+
150+
List<AgentResult> results = agent.streamResult("hello").collectList().block();
151+
152+
assertThat(results).isNotNull();
153+
AgentResult delivery = results.get(results.size() - 1);
154+
assertThat(delivery.status()).isEqualTo(AgentResult.Status.COMPLETED);
155+
assertThat(results).allSatisfy(result -> assertThat(result.message().metadata())
156+
.doesNotContainKey(AgentLlmNode.MODEL_FAILURE_METADATA_KEY));
157+
}
158+
121159
@Test
122160
@DisplayName("the legacy streamMessages behavior stays untouched")
123161
void legacyStreamMessagesUnchanged() throws Exception {

0 commit comments

Comments
 (0)