Skip to content

Commit d701250

Browse files
committed
feat(ai-proxy): validate generated content against operator schemas with bounded opt-in repair (#8257)
1 parent 6cb63d0 commit d701250

22 files changed

Lines changed: 1444 additions & 115 deletions

File tree

‎shenyu-common/src/main/java/org/apache/shenyu/common/dto/convert/rule/AiProxyHandle.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ public class AiProxyHandle {
8888

8989
private Boolean requestBodyForceOverride;
9090

91+
/**
92+
* Opt-in structured output policy (issue #8257); absent or schema-less means
93+
* fully disabled and responses pass through byte for byte.
94+
*/
95+
private AiStructuredOutputConfig structuredOutput;
96+
9197
// flat fallback fields (to fit dashboard flat handle capability)
9298
private String fallbackEnabled;
9399

@@ -323,6 +329,24 @@ public void setRequestBodyForceOverride(final Boolean requestBodyForceOverride)
323329
this.requestBodyForceOverride = requestBodyForceOverride;
324330
}
325331

332+
/**
333+
* Get the opt-in structured output policy.
334+
*
335+
* @return structured output config, or {@code null} when disabled
336+
*/
337+
public AiStructuredOutputConfig getStructuredOutput() {
338+
return structuredOutput;
339+
}
340+
341+
/**
342+
* Set the opt-in structured output policy.
343+
*
344+
* @param structuredOutput structured output config
345+
*/
346+
public void setStructuredOutput(final AiStructuredOutputConfig structuredOutput) {
347+
this.structuredOutput = structuredOutput;
348+
}
349+
326350
/**
327351
* Normalize this handle by assembling nested FallbackConfig from flat fields when enabled.
328352
*
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.shenyu.common.dto.convert.rule;
19+
20+
import java.io.Serializable;
21+
22+
/**
23+
* Operator-managed structured output policy for one AI selector (issue #8257).
24+
*
25+
* <p>Both switches are opt-in: without a non-blank {@code schema} nothing is
26+
* validated and responses pass through byte for byte; without {@code repairEnabled}
27+
* a failed validation rejects the response instead of regenerating it. The schema
28+
* is validated against a bounded Draft 2020-12 dialect at use time — unsupported
29+
* or oversized schemas fail fast, never silently degrade.</p>
30+
*/
31+
public class AiStructuredOutputConfig implements Serializable {
32+
33+
/**
34+
* Maximum regeneration attempts after a failed validation; bounded from above
35+
* so a broken schema cannot multiply upstream spend.
36+
*/
37+
public static final int MAX_REPAIR_ATTEMPTS_LIMIT = 3;
38+
39+
private static final long serialVersionUID = -4216685160670534827L;
40+
41+
/**
42+
* Operator-managed JSON Schema (Draft 2020-12, bounded dialect).
43+
*/
44+
private String schema;
45+
46+
private Boolean repairEnabled = false;
47+
48+
private String repairPrompt;
49+
50+
private Integer repairMaxAttempts = 1;
51+
52+
private Integer repairMaxTokens;
53+
54+
public String getSchema() {
55+
return schema;
56+
}
57+
58+
public void setSchema(final String schema) {
59+
this.schema = schema;
60+
}
61+
62+
public Boolean getRepairEnabled() {
63+
return repairEnabled;
64+
}
65+
66+
public void setRepairEnabled(final Boolean repairEnabled) {
67+
this.repairEnabled = repairEnabled;
68+
}
69+
70+
public String getRepairPrompt() {
71+
return repairPrompt;
72+
}
73+
74+
public void setRepairPrompt(final String repairPrompt) {
75+
this.repairPrompt = repairPrompt;
76+
}
77+
78+
public Integer getRepairMaxAttempts() {
79+
return repairMaxAttempts;
80+
}
81+
82+
public void setRepairMaxAttempts(final Integer repairMaxAttempts) {
83+
this.repairMaxAttempts = repairMaxAttempts;
84+
}
85+
86+
public Integer getRepairMaxTokens() {
87+
return repairMaxTokens;
88+
}
89+
90+
public void setRepairMaxTokens(final Integer repairMaxTokens) {
91+
this.repairMaxTokens = repairMaxTokens;
92+
}
93+
}

‎shenyu-common/src/main/java/org/apache/shenyu/common/metrics/AiAttemptEvent.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,6 @@ public AiAttemptEvent(final AgentRequestContext context, final int attemptNumber
5858
* How this actual network attempt was selected.
5959
*/
6060
public enum Kind {
61-
PRIMARY, RETRY, FALLBACK
61+
PRIMARY, RETRY, FALLBACK, REPAIR
6262
}
6363
}

‎shenyu-plugin/shenyu-plugin-agent-gateway/pom.xml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,6 @@
2525
<modelVersion>4.0.0</modelVersion>
2626
<artifactId>shenyu-plugin-agent-gateway</artifactId>
2727

28-
<properties>
29-
<json-schema-validator.version>2.0.0</json-schema-validator.version>
30-
</properties>
31-
3228
<build>
3329
<resources>
3430
<resource>
@@ -54,11 +50,6 @@
5450
<version>${project.version}</version>
5551
<scope>test</scope>
5652
</dependency>
57-
<dependency>
58-
<groupId>com.networknt</groupId>
59-
<artifactId>json-schema-validator</artifactId>
60-
<version>${json-schema-validator.version}</version>
61-
</dependency>
6253
<dependency>
6354
<groupId>io.modelcontextprotocol.sdk</groupId>
6455
<artifactId>mcp-core</artifactId>

‎shenyu-plugin/shenyu-plugin-agent-gateway/src/main/java/org/apache/shenyu/plugin/agent/gateway/protocol/AgentMcpResultFilter.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.fasterxml.jackson.databind.JsonNode;
2121
import com.fasterxml.jackson.databind.node.ObjectNode;
2222
import org.apache.shenyu.common.dto.AgentGatewayResultFilterConfig;
23+
import org.apache.shenyu.plugin.base.schema.BoundedJsonSchemaValidator;
2324

2425
import java.util.Objects;
2526

@@ -30,15 +31,15 @@ final class AgentMcpResultFilter {
3031

3132
private final AgentMcpStructuredContentFilter filter;
3233

33-
private final AgentMcpOutputSchemaValidator validator;
34+
private final BoundedJsonSchemaValidator validator;
3435

3536
AgentMcpResultFilter(final AgentGatewayResultFilterConfig config, final String name, final JsonNode outputSchema) {
3637
if (config.getPlan(name).isEmpty()) {
3738
filter = null;
3839
validator = null;
3940
} else {
4041
filter = new AgentMcpStructuredContentFilter(config, name);
41-
validator = Objects.isNull(outputSchema) || outputSchema.isNull() ? null : new AgentMcpOutputSchemaValidator(outputSchema);
42+
validator = Objects.isNull(outputSchema) || outputSchema.isNull() ? null : new BoundedJsonSchemaValidator(outputSchema);
4243
}
4344
}
4445

@@ -48,7 +49,7 @@ ObjectNode apply(final ObjectNode result) {
4849
}
4950
ObjectNode filtered = filter.filter(result);
5051
if (Objects.nonNull(validator) && (filtered.has("structuredContent") || !filtered.path("isError").asBoolean(false))) {
51-
validator.validate(filtered.get("structuredContent"));
52+
validator.validateObject(filtered.get("structuredContent"));
5253
}
5354
new AgentMcpResultCopiesFilter().apply(result, filtered);
5455
return filtered;

shenyu-plugin/shenyu-plugin-agent-gateway/src/test/java/org/apache/shenyu/plugin/agent/gateway/protocol/AgentMcpOutputSchemaValidatorTest.java renamed to shenyu-plugin/shenyu-plugin-agent-gateway/src/test/java/org/apache/shenyu/plugin/agent/gateway/protocol/AgentMcpResultFilterTest.java

Lines changed: 5 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -17,28 +17,24 @@
1717

1818
package org.apache.shenyu.plugin.agent.gateway.protocol;
1919

20-
import com.fasterxml.jackson.databind.JsonNode;
2120
import com.fasterxml.jackson.databind.ObjectMapper;
2221
import com.fasterxml.jackson.databind.node.ObjectNode;
2322
import com.google.gson.JsonParser;
24-
import com.networknt.schema.InvalidSchemaException;
25-
import com.networknt.schema.SchemaLocation;
26-
import com.sun.net.httpserver.HttpServer;
2723
import org.apache.shenyu.common.dto.AgentGatewayResultFilterConfig;
2824
import org.junit.jupiter.api.Test;
29-
import org.junit.jupiter.params.ParameterizedTest;
30-
import org.junit.jupiter.params.provider.ValueSource;
3125

32-
import java.net.InetSocketAddress;
3326
import java.util.Set;
34-
import java.util.concurrent.atomic.AtomicInteger;
3527

3628
import static org.junit.jupiter.api.Assertions.assertEquals;
3729
import static org.junit.jupiter.api.Assertions.assertFalse;
3830
import static org.junit.jupiter.api.Assertions.assertThrows;
3931
import static org.junit.jupiter.api.Assertions.assertTrue;
4032

41-
class AgentMcpOutputSchemaValidatorTest {
33+
/**
34+
* Behavior tests of the MCP result filter against the shared bounded schema
35+
* validator: deletion keeps the schema unchanged and validation stays fail-closed.
36+
*/
37+
class AgentMcpResultFilterTest {
4238

4339
private final ObjectMapper mapper = new ObjectMapper();
4440

@@ -76,15 +72,6 @@ void defaultsCannotRestoreADeletedFieldOrSatisfyARequiredField() throws Exceptio
7672
assertThrows(RuntimeException.class, () -> required.apply(result));
7773
}
7874

79-
@Test
80-
void schemaSnapshotIsIndependentOfMutableDefinition() throws Exception {
81-
ObjectNode declaration = json("{\"type\":\"object\",\"required\":[\"keep\"]}");
82-
AgentMcpOutputSchemaValidator validator = new AgentMcpOutputSchemaValidator(declaration);
83-
declaration.removeAll();
84-
assertThrows(RuntimeException.class, () -> validator.validate(json("{\"secret\":\"private\"}")));
85-
validator.validate(json("{\"keep\":1}"));
86-
}
87-
8875
@Test
8976
void ignoresUnsupportedSchemasWhenNoPlanIsConfigured() throws Exception {
9077
ObjectNode source = json("{\"structuredContent\":{\"secret\":\"private\"}}");
@@ -93,76 +80,13 @@ void ignoresUnsupportedSchemasWhenNoPlanIsConfigured() throws Exception {
9380
assertTrue(source == filter.apply(source));
9481
}
9582

96-
@ParameterizedTest
97-
@ValueSource(strings = {"{\"$ref\":\"#\"}", "{\"$ref\":\"https://untrusted.invalid/schema\"}",
98-
"{\"$dynamicRef\":\"#anchor\"}", "{\"pattern\":\"(a+)+$\"}", "{\"patternProperties\":{}}",
99-
"{\"$schema\":\"https://untrusted.invalid/meta\"}", "{\"type\":\"invalid-type\"}",
100-
"{\"properties\":true}", "{\"allOf\":[true,true,true,true,true]}", "{\"unknownAssertion\":true}"})
101-
void rejectsUnsupportedOrMalformedSchemas(final String declaration) throws Exception {
102-
assertThrows(RuntimeException.class, () -> new AgentMcpOutputSchemaValidator(json(declaration)));
103-
}
104-
105-
@Test
106-
void annotationDataDoesNotBecomeAReference() throws Exception {
107-
AgentMcpOutputSchemaValidator validator = new AgentMcpOutputSchemaValidator(json(
108-
"{\"type\":\"object\",\"properties\":{\"keep\":{\"const\":{\"$ref\":\"not-a-control-plane\"}}}}"));
109-
validator.validate(json("{\"keep\":{\"$ref\":\"not-a-control-plane\"}}"));
110-
}
111-
112-
@Test
113-
void actualReferenceResolverCannotContactALocalHttpEndpoint() throws Exception {
114-
AtomicInteger requests = new AtomicInteger();
115-
HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
116-
server.createContext("/schema", exchange -> {
117-
requests.incrementAndGet();
118-
exchange.sendResponseHeaders(500, -1);
119-
exchange.close();
120-
});
121-
server.start();
122-
try {
123-
String uri = "http://127.0.0.1:" + server.getAddress().getPort() + "/schema";
124-
JsonNode declaration = json("{\"$ref\":\"" + uri + "\"}");
125-
InvalidSchemaException failure = assertThrows(InvalidSchemaException.class,
126-
() -> AgentMcpOutputSchemaValidator.localRegistry().getSchema(declaration).validate(json("{}")));
127-
assertTrue(failure.getMessage().contains("not allowed"));
128-
assertEquals(0, requests.get());
129-
} finally {
130-
server.stop(0);
131-
}
132-
}
133-
134-
@ParameterizedTest
135-
@ValueSource(strings = {"file:///does-not-exist/schema.json", "classpath:META-INF/shenyu/agent-gateway-version.properties",
136-
"https://untrusted.invalid/schema"})
137-
void actualLoaderRejectsNonBuiltinResources(final String uri) {
138-
InvalidSchemaException failure = assertThrows(InvalidSchemaException.class,
139-
() -> AgentMcpOutputSchemaValidator.localRegistry().getSchema(SchemaLocation.of(uri)));
140-
assertTrue(failure.getMessage().contains("not allowed"));
141-
}
142-
14383
@Test
14484
void activeCopyPolicyRequiresStructuredContentForBusinessErrorsToo() throws Exception {
14585
AgentMcpResultFilter filter = new AgentMcpResultFilter(config(), "crm.profile", json("{\"type\":\"object\"}"));
14686
assertThrows(RuntimeException.class, () -> filter.apply(json("{\"isError\":false}")));
14787
assertThrows(RuntimeException.class, () -> filter.apply(json("{\"isError\":true}")));
14888
}
14989

150-
@Test
151-
void limitsInstanceAndSchemaCostBeforeValidation() throws Exception {
152-
AgentMcpOutputSchemaValidator validator = new AgentMcpOutputSchemaValidator(json("{\"type\":\"object\"}"));
153-
ObjectNode large = json("{}");
154-
var array = large.putArray("values");
155-
for (int index = 0; index < 129; index++) {
156-
array.add(index);
157-
}
158-
assertThrows(RuntimeException.class, () -> validator.validate(large));
159-
ObjectNode longString = json("{}");
160-
longString.put("value", "x".repeat(4097));
161-
assertThrows(RuntimeException.class, () -> validator.validate(longString));
162-
assertThrows(RuntimeException.class, () -> new AgentMcpOutputSchemaValidator(json(
163-
"{\"allOf\":[{\"allOf\":[{\"allOf\":[true]}]}]}")));
164-
}
165-
16690
private AgentGatewayResultFilterConfig config() {
16791
return AgentGatewayResultFilterConfig.parse(
16892
JsonParser.parseString("{\"crm.profile\":[\"/secret\"]}"), Set.of("crm.profile"));

0 commit comments

Comments
 (0)