Skip to content

Commit efd12ae

Browse files
authored
Merge pull request #366 from XianReallyHot-ZZH/feat/issue-207-repair-usage
fix(agent): fold structured-output repair usage into delivery totals (#207)
2 parents 4acbe77 + 5ea40de commit efd12ae

9 files changed

Lines changed: 784 additions & 26 deletions

File tree

‎argi-agent/argi-agent-api/src/main/java/io/github/agentic/ai/agent/AgentResult.java‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,18 @@
3737
* assistant message exists (for example after an interruption), the status is
3838
* {@link Status#INTERRUPTED} and {@link #message()} is {@code null}.
3939
* </p>
40+
*
41+
* <p>
42+
* {@link #usage()} is the total usage of the whole delivery: the accumulated model
43+
* turns plus any bounded structured-output repair re-ask, per-field, with a field
44+
* unknown on either side staying unknown instead of being summed as zero. When a
45+
* repair re-ask was issued, {@link #repairUsage()} carries the usage that single call
46+
* reported, so main-call and repair consumption stay distinguishable; an issued
47+
* repair whose consumption is unknown carries an all-unknown segment rather than a
48+
* fabricated zero.
49+
* </p>
4050
*/
41-
public record AgentResult(ChatMessage message, Status status, Usage usage, Object data,
51+
public record AgentResult(ChatMessage message, Status status, Usage usage, Usage repairUsage, Object data,
4252
List<OutputValidationIssue> issues) {
4353

4454
/** Error code for terminal output that is not valid JSON at all. */
@@ -96,20 +106,40 @@ public record OutputValidationIssue(String errorCode, String fieldPath, String m
96106
issues = issues == null ? List.of() : List.copyOf(issues);
97107
}
98108

109+
/**
110+
* Compatibility shape without the repair usage segment, equivalent to a delivery
111+
* where no repair re-ask was issued.
112+
*/
113+
public AgentResult(ChatMessage message, Status status, Usage usage, Object data,
114+
List<OutputValidationIssue> issues) {
115+
this(message, status, usage, null, data, issues);
116+
}
117+
99118
public static AgentResult completed(ChatMessage message, Usage usage, Object data) {
100-
return new AgentResult(message, Status.COMPLETED, usage, data, List.of());
119+
return new AgentResult(message, Status.COMPLETED, usage, null, data, List.of());
120+
}
121+
122+
/** Completion whose total includes an issued repair re-ask, with its reported segment. */
123+
public static AgentResult completed(ChatMessage message, Usage usage, Usage repairUsage, Object data) {
124+
return new AgentResult(message, Status.COMPLETED, usage, repairUsage, data, List.of());
101125
}
102126

103127
public static AgentResult validationFailed(ChatMessage message, Usage usage, List<OutputValidationIssue> issues) {
104-
return new AgentResult(message, Status.VALIDATION_FAILED, usage, null, issues);
128+
return new AgentResult(message, Status.VALIDATION_FAILED, usage, null, null, issues);
129+
}
130+
131+
/** Validation failure whose total includes an issued repair re-ask, with its reported segment. */
132+
public static AgentResult validationFailed(ChatMessage message, Usage usage, Usage repairUsage,
133+
List<OutputValidationIssue> issues) {
134+
return new AgentResult(message, Status.VALIDATION_FAILED, usage, repairUsage, null, issues);
105135
}
106136

107137
public static AgentResult interrupted(Usage usage) {
108-
return new AgentResult(null, Status.INTERRUPTED, usage, null, List.of());
138+
return new AgentResult(null, Status.INTERRUPTED, usage, null, null, List.of());
109139
}
110140

111141
public static AgentResult error(ChatMessage message, Usage usage, List<OutputValidationIssue> issues) {
112-
return new AgentResult(message, Status.ERROR, usage, null, issues);
142+
return new AgentResult(message, Status.ERROR, usage, null, null, issues);
113143
}
114144

115145
}

‎argi-agent/argi-agent-core/src/main/java/io/github/agentic/ai/agent/react/ReActAgent.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -693,11 +693,23 @@ private void admitPatch(String toolName, ToolResult result, Map<String, Object>
693693
private void finish(List<ChatMessage> messages, Map<String, Object> state, Usage usage,
694694
ChatMessage terminal, String terminalMessageId) {
695695
Map<String, Object> outputs = new LinkedHashMap<>(state);
696+
Usage deliveredUsage = usage;
696697
if (outputSchema != null && outputFinalizer != null) {
697698
AgentResult result = outputFinalizer.finalizeOutput(terminal, outputSchema, usage, context.control());
699+
if (result.repairUsage() != null) {
700+
// A repair re-ask happened after the last per-turn update: publish
701+
// the delivery-inclusive cumulative usage before anything consumes
702+
// or propagates the result, so UsageUpdate consumers and the result
703+
// — delivered or thrown — always agree on the same totals.
704+
emit(new AgentEvent.UsageUpdate(result.usage()));
705+
}
698706
if (result.status() != AgentResult.Status.COMPLETED) {
699707
throw new AgentExecutionException(result);
700708
}
709+
// The finalizer's usage is the delivery total: a repair re-ask spent
710+
// tokens after the last model turn, and the finished snapshot must carry
711+
// the same total the result reports instead of the pre-repair one.
712+
deliveredUsage = result.usage();
701713
// A successful repair replaces the delivered terminal: the message that
702714
// actually passed validation becomes the final message while the failed
703715
// original stays in history, and stream consumers see it through the
@@ -714,7 +726,7 @@ private void finish(List<ChatMessage> messages, Map<String, Object> state, Usage
714726
outputs.put("structuredOutput", result.data());
715727
}
716728
}
717-
emit(new AgentEvent.Finished(AgentOutcome.completed(messages, outputs, usage)));
729+
emit(new AgentEvent.Finished(AgentOutcome.completed(messages, outputs, deliveredUsage)));
718730
}
719731

720732
private void emit(AgentEvent event) {

‎argi-agent/argi-agent-core/src/main/java/io/github/agentic/ai/agent/result/StructuredOutputFinalizer.java‎

Lines changed: 56 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ public AgentResult finalizeOutput(ChatMessage terminal, String outputSchema, Usa
171171
ROOT_PATH, "Configured output schema is not a valid JSON Schema")));
172172
}
173173

174-
AgentResult first = validateAgainst(terminal, schema, usage);
174+
AgentResult first = validateAgainst(terminal, schema, usage, null);
175175
if (first.status() != AgentResult.Status.VALIDATION_FAILED || this.repairClient == null) {
176176
return first;
177177
}
@@ -180,39 +180,51 @@ public AgentResult finalizeOutput(ChatMessage terminal, String outputSchema, Usa
180180
// exceeded and ownership lost all propagate as execution stops.
181181
control.assertActive();
182182
}
183-
ChatMessage repaired = attemptRepair(terminal, outputSchema, first.issues(), control);
184-
if (repaired == null) {
185-
// The repair re-ask could not be performed: keep the original failure and
186-
// mark the distinction from a repair that ran but stayed invalid.
187-
List<AgentResult.OutputValidationIssue> marked = new ArrayList<>();
188-
marked.add(new AgentResult.OutputValidationIssue(AgentResult.ERROR_CODE_REPAIR_CALL_FAILED, ROOT_PATH,
189-
"The bounded repair re-ask could not be performed or returned no assistant message"));
190-
marked.addAll(first.issues());
191-
return AgentResult.validationFailed(first.message(), first.usage(), List.copyOf(marked));
183+
RepairAttempt repair = attemptRepair(terminal, outputSchema, first.issues(), control);
184+
if (repair == null) {
185+
// The repair re-ask could not be performed at all: keep the original failure
186+
// and mark the distinction from a repair that ran but stayed invalid. The
187+
// call was issued yet no response proves what it consumed, so neither the
188+
// segment nor the delivery total may claim it spent nothing: both stay
189+
// unknown instead of the pre-repair usage standing in as an exact total.
190+
return AgentResult.validationFailed(first.message(), Usage.unknown(), Usage.unknown(),
191+
repairCallFailedIssues(first.issues()));
192+
}
193+
// The repair re-ask is part of the delivery: its reported usage folds into the
194+
// delivered total per field, and an unknown field on either side stays unknown
195+
// instead of being summed as zero — including a main usage that was never
196+
// reported at all.
197+
Usage deliveryUsage = (usage == null ? Usage.unknown() : usage).plus(repair.reportedUsage());
198+
if (repair.message() == null) {
199+
// A response arrived without an assistant message: the call happened and its
200+
// reported usage counts, but there is nothing to re-validate.
201+
return AgentResult.validationFailed(first.message(), deliveryUsage, repair.reportedUsage(),
202+
repairCallFailedIssues(first.issues()));
192203
}
193204
if (control != null) {
194205
// Deadline checks are synchronous: a deadline that expired during the
195206
// re-ask stops the delivery here instead of publishing a late output.
196207
control.assertActive();
197208
}
198-
AgentResult second = validateAgainst(repaired, schema, usage);
209+
AgentResult second = validateAgainst(repair.message(), schema, deliveryUsage, repair.reportedUsage());
199210
if (second.status() != AgentResult.Status.VALIDATION_FAILED) {
200211
return second;
201212
}
202213
List<AgentResult.OutputValidationIssue> marked = new ArrayList<>();
203214
marked.add(new AgentResult.OutputValidationIssue(AgentResult.ERROR_CODE_REPAIRED_STILL_INVALID, ROOT_PATH,
204215
"One bounded repair attempt did not produce a schema-valid output"));
205216
marked.addAll(second.issues());
206-
return AgentResult.validationFailed(repaired, usage, List.copyOf(marked));
217+
return AgentResult.validationFailed(repair.message(), deliveryUsage, repair.reportedUsage(),
218+
List.copyOf(marked));
207219
}
208220

209-
private AgentResult validateAgainst(ChatMessage terminal, Schema schema, Usage usage) {
221+
private AgentResult validateAgainst(ChatMessage terminal, Schema schema, Usage usage, Usage repairUsage) {
210222
JsonNode root;
211223
try {
212224
root = MAPPER.readTree(terminal.text());
213225
}
214226
catch (RuntimeException ex) {
215-
return AgentResult.validationFailed(terminal, usage,
227+
return AgentResult.validationFailed(terminal, usage, repairUsage,
216228
List.of(new AgentResult.OutputValidationIssue(AgentResult.ERROR_CODE_INVALID_JSON, ROOT_PATH,
217229
"Terminal output is not valid JSON")));
218230
}
@@ -224,17 +236,20 @@ private AgentResult validateAgainst(ChatMessage terminal, Schema schema, Usage u
224236
issues.add(new AgentResult.OutputValidationIssue(AgentResult.ERROR_CODE_SCHEMA_VIOLATION,
225237
error.getInstanceLocation().toString(), error.getMessage()));
226238
}
227-
return AgentResult.validationFailed(terminal, usage, List.copyOf(issues));
239+
return AgentResult.validationFailed(terminal, usage, repairUsage, List.copyOf(issues));
228240
}
229241

230-
return AgentResult.completed(terminal, usage, root);
242+
return AgentResult.completed(terminal, usage, repairUsage, root);
231243
}
232244

233245
/**
234-
* One bounded repair re-ask: single-turn, no tools, no graph. Returns the repaired
235-
* assistant message, or {@code null} when the repair client itself failed.
246+
* One bounded repair re-ask: single-turn, no tools, no graph. Returns what the re-ask
247+
* produced — the repaired assistant message (or {@code null} when the response carried
248+
* none) together with the usage that response reported, or {@code null} when no
249+
* response was received at all and therefore nothing about the call's consumption
250+
* is known.
236251
*/
237-
private ChatMessage attemptRepair(ChatMessage terminal, String outputSchema,
252+
private RepairAttempt attemptRepair(ChatMessage terminal, String outputSchema,
238253
List<AgentResult.OutputValidationIssue> issues, ExecutionGuard control) {
239254
try {
240255
String issueSummary = issues.stream()
@@ -251,10 +266,12 @@ private ChatMessage attemptRepair(ChatMessage terminal, String outputSchema,
251266
GenerationOptions.builder().toolChoice(ToolChoice.none()).build(), Map.of());
252267
ChatResponse response = control == null ? this.repairClient.callBlocking(request)
253268
: callUnderControl(request, control);
254-
return response.messages().stream()
269+
ChatMessage repaired = response.messages().stream()
255270
.filter(message -> message.role() == ChatRole.ASSISTANT)
256271
.findFirst()
257272
.orElse(null);
273+
Usage reportedUsage = response.usage() == null ? Usage.unknown() : response.usage();
274+
return new RepairAttempt(repaired, reportedUsage);
258275
}
259276
catch (CancellationException cancelled) {
260277
if (control == null) {
@@ -278,6 +295,25 @@ private ChatMessage attemptRepair(ChatMessage terminal, String outputSchema,
278295
}
279296
}
280297

298+
/** The outcome of a repair re-ask that received a response. */
299+
private record RepairAttempt(ChatMessage message, Usage reportedUsage) {
300+
301+
}
302+
303+
/**
304+
* Marks a repair re-ask that produced no re-validatable message — either no response
305+
* at all or a response without an assistant message — in front of the original
306+
* validation failure.
307+
*/
308+
private static List<AgentResult.OutputValidationIssue> repairCallFailedIssues(
309+
List<AgentResult.OutputValidationIssue> original) {
310+
List<AgentResult.OutputValidationIssue> marked = new ArrayList<>();
311+
marked.add(new AgentResult.OutputValidationIssue(AgentResult.ERROR_CODE_REPAIR_CALL_FAILED, ROOT_PATH,
312+
"The bounded repair re-ask could not be performed or returned no assistant message"));
313+
marked.addAll(original);
314+
return List.copyOf(marked);
315+
}
316+
281317
/**
282318
* Performs the repair re-ask bound to the run's guard: a stop raised while the call
283319
* is in flight cancels the pending call, and the cancellation observed while waiting

0 commit comments

Comments
 (0)