Skip to content

Commit 904da07

Browse files
56wjilayaperumalg
authored andcommitted
Cache tool results in BedrockProxyChatModel
Add an opt-in cache point after the final tool result message for conversation-history caching. Expose the option through Spring Boot configuration and document its behavior. Fixes #6938 Signed-off-by: 56wj <[email protected]>
1 parent 877233a commit 904da07

8 files changed

Lines changed: 301 additions & 7 deletions

File tree

‎auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/converse/autoconfigure/BedrockCacheProperties.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ public class BedrockCacheProperties {
2828

2929
private @Nullable BedrockCacheTtl ttl;
3030

31+
private boolean cacheToolResults;
32+
3133
public @Nullable BedrockCacheStrategy getStrategy() {
3234
return this.strategy;
3335
}
@@ -44,8 +46,20 @@ public void setTtl(@Nullable BedrockCacheTtl ttl) {
4446
this.ttl = ttl;
4547
}
4648

49+
public boolean isCacheToolResults() {
50+
return this.cacheToolResults;
51+
}
52+
53+
public void setCacheToolResults(boolean cacheToolResults) {
54+
this.cacheToolResults = cacheToolResults;
55+
}
56+
4757
public BedrockCacheOptions toOptions() {
48-
return BedrockCacheOptions.builder().strategy(this.strategy).ttl(this.ttl).build();
58+
return BedrockCacheOptions.builder()
59+
.strategy(this.strategy)
60+
.ttl(this.ttl)
61+
.cacheToolResults(this.cacheToolResults)
62+
.build();
4963
}
5064

5165
}

‎auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/test/java/org/springframework/ai/model/bedrock/converse/autoconfigure/BedrockConverseProxyChatPropertiesTests.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,17 +122,20 @@ public void requestParametersSupportsMixedFlatAndNestedValuesTest() {
122122
public void cacheOptionsTest() {
123123
new ApplicationContextRunner()
124124
.withPropertyValues("spring.ai.bedrock.converse.chat.cache-options.strategy=SYSTEM_ONLY",
125-
"spring.ai.bedrock.converse.chat.cache-options.ttl=ONE_HOUR")
125+
"spring.ai.bedrock.converse.chat.cache-options.ttl=ONE_HOUR",
126+
"spring.ai.bedrock.converse.chat.cache-options.cache-tool-results=true")
126127
.withConfiguration(AutoConfigurations.of(BedrockConverseProxyChatAutoConfiguration.class,
127128
ToolCallingAutoConfiguration.class))
128129
.run(context -> {
129130
var chatProperties = context.getBean(BedrockConverseProxyChatProperties.class);
130131
assertThat(chatProperties.getCacheOptions().getStrategy()).isEqualTo(BedrockCacheStrategy.SYSTEM_ONLY);
131132
assertThat(chatProperties.getCacheOptions().getTtl()).isEqualTo(BedrockCacheTtl.ONE_HOUR);
133+
assertThat(chatProperties.getCacheOptions().isCacheToolResults()).isTrue();
132134

133135
var options = chatProperties.toOptions();
134136
assertThat(options.getCacheOptions().getStrategy()).isEqualTo(BedrockCacheStrategy.SYSTEM_ONLY);
135137
assertThat(options.getCacheOptions().getTtl()).isEqualTo(BedrockCacheTtl.ONE_HOUR);
138+
assertThat(options.getCacheOptions().isCacheToolResults()).isTrue();
136139
});
137140
}
138141

‎models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,9 @@ ConverseRequest createRequest(Prompt prompt) {
253253
BedrockCacheOptions cacheOptions = options.getCacheOptions();
254254
boolean shouldCacheConversationHistory = cacheOptions != null
255255
&& cacheOptions.getStrategy() == BedrockCacheStrategy.CONVERSATION_HISTORY;
256+
boolean shouldCacheToolResults = cacheOptions != null
257+
&& cacheOptions.getStrategy() == BedrockCacheStrategy.CONVERSATION_HISTORY
258+
&& cacheOptions.isCacheToolResults();
256259

257260
// Get all non-system messages
258261
List<org.springframework.ai.chat.messages.Message> allNonSystemMessages = prompt.getInstructions()
@@ -275,14 +278,27 @@ ConverseRequest createRequest(Prompt prompt) {
275278
}
276279
}
277280

281+
// Find the final tool result message. Placing a cache point after its last
282+
// content block caches all prior tool outputs for subsequent tool-calling rounds.
283+
int lastToolMessageIndex = -1;
284+
if (shouldCacheToolResults) {
285+
for (int i = allNonSystemMessages.size() - 1; i >= 0; i--) {
286+
if (allNonSystemMessages.get(i).getMessageType() == MessageType.TOOL
287+
&& !((ToolResponseMessage) allNonSystemMessages.get(i)).getResponses().isEmpty()) {
288+
lastToolMessageIndex = i;
289+
break;
290+
}
291+
}
292+
}
293+
278294
// Build instruction messages with potential caching
279295
List<Message> instructionMessages = new ArrayList<>();
280296
for (int i = 0; i < allNonSystemMessages.size(); i++) {
281297
org.springframework.ai.chat.messages.Message message = allNonSystemMessages.get(i);
282298

283299
// Determine if this message should have a cache point
284300
// For CONVERSATION_HISTORY: cache point goes on the last user message
285-
boolean shouldApplyCachePoint = shouldCacheConversationHistory && i == lastUserMessageIndex;
301+
boolean shouldApplyUserCachePoint = shouldCacheConversationHistory && i == lastUserMessageIndex;
286302

287303
if (message.getMessageType() == MessageType.USER) {
288304
List<ContentBlock> contents = new ArrayList<>();
@@ -305,7 +321,7 @@ ConverseRequest createRequest(Prompt prompt) {
305321
}
306322

307323
// Apply cache point if this is the last user message
308-
if (shouldApplyCachePoint) {
324+
if (shouldApplyUserCachePoint) {
309325
contents.add(ContentBlock.fromCachePoint(buildCachePoint(cacheOptions)));
310326
logger.debug("Applied cache point on last user message (conversation history caching)");
311327
}
@@ -355,6 +371,14 @@ else if (message.getMessageType() == MessageType.TOOL) {
355371
return ContentBlock.fromToolResult(toolResultBlock);
356372
}).toList());
357373

374+
// ContentBlock is a union: toolResult and cachePoint must be separate
375+
// blocks. Avoid emitting a cache-point-only message for an empty
376+
// response.
377+
if (i == lastToolMessageIndex && !contentBlocks.isEmpty()) {
378+
contentBlocks.add(ContentBlock.fromCachePoint(buildCachePoint(cacheOptions)));
379+
logger.debug("Applied cache point after last tool result message");
380+
}
381+
358382
instructionMessages.add(Message.builder().content(contentBlocks).role(ConversationRole.USER).build());
359383
}
360384
else {

‎models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/api/BedrockCacheOptions.java‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,15 +59,23 @@ public class BedrockCacheOptions {
5959

6060
private final @Nullable BedrockCacheTtl ttl;
6161

62+
private final boolean cacheToolResults;
63+
6264
protected BedrockCacheOptions(@Nullable BedrockCacheStrategy strategy, boolean multiBlockSystemCaching) {
63-
this(strategy, multiBlockSystemCaching, null);
65+
this(strategy, multiBlockSystemCaching, null, false);
6466
}
6567

6668
protected BedrockCacheOptions(@Nullable BedrockCacheStrategy strategy, boolean multiBlockSystemCaching,
6769
@Nullable BedrockCacheTtl ttl) {
70+
this(strategy, multiBlockSystemCaching, ttl, false);
71+
}
72+
73+
protected BedrockCacheOptions(@Nullable BedrockCacheStrategy strategy, boolean multiBlockSystemCaching,
74+
@Nullable BedrockCacheTtl ttl, boolean cacheToolResults) {
6875
this.strategy = (strategy != null ? strategy : BedrockCacheStrategy.NONE);
6976
this.multiBlockSystemCaching = multiBlockSystemCaching;
7077
this.ttl = ttl;
78+
this.cacheToolResults = cacheToolResults;
7179
}
7280

7381
/**
@@ -112,6 +120,18 @@ public boolean isMultiBlockSystemCaching() {
112120
return this.ttl;
113121
}
114122

123+
/**
124+
* Returns whether prompt caching is enabled for tool result messages. When enabled
125+
* with {@link BedrockCacheStrategy#CONVERSATION_HISTORY}, a cache point is placed
126+
* after the last tool result message in a request so previous tool outputs can be
127+
* reused in subsequent tool-calling rounds. Disabled by default.
128+
* @return {@code true} if tool result caching is enabled; {@code false} otherwise
129+
* @since 2.1.0
130+
*/
131+
public boolean isCacheToolResults() {
132+
return this.cacheToolResults;
133+
}
134+
115135
/**
116136
* Builder for constructing BedrockCacheOptions instances.
117137
*/
@@ -123,6 +143,8 @@ public static class Builder {
123143

124144
private @Nullable BedrockCacheTtl ttl;
125145

146+
private boolean cacheToolResults = false;
147+
126148
/**
127149
* Sets the caching strategy.
128150
* @param strategy the BedrockCacheStrategy to use
@@ -160,12 +182,26 @@ public Builder ttl(@Nullable BedrockCacheTtl ttl) {
160182
return this;
161183
}
162184

185+
/**
186+
* Sets whether to cache tool result messages. This option takes effect only with
187+
* {@link BedrockCacheStrategy#CONVERSATION_HISTORY} and defaults to
188+
* {@code false}.
189+
* @param cacheToolResults whether to cache tool result messages
190+
* @return this Builder instance
191+
* @since 2.1.0
192+
*/
193+
public Builder cacheToolResults(boolean cacheToolResults) {
194+
this.cacheToolResults = cacheToolResults;
195+
return this;
196+
}
197+
163198
/**
164199
* Builds the BedrockCacheOptions instance.
165200
* @return the configured BedrockCacheOptions
166201
*/
167202
public BedrockCacheOptions build() {
168-
return new BedrockCacheOptions(this.strategy, this.multiBlockSystemCaching, this.ttl);
203+
return new BedrockCacheOptions(this.strategy, this.multiBlockSystemCaching, this.ttl,
204+
this.cacheToolResults);
169205
}
170206

171207
}

‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModelIT.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.springframework.ai.chat.messages.AssistantMessage;
4242
import org.springframework.ai.chat.messages.Message;
4343
import org.springframework.ai.chat.messages.SystemMessage;
44+
import org.springframework.ai.chat.messages.ToolResponseMessage;
4445
import org.springframework.ai.chat.messages.UserMessage;
4546
import org.springframework.ai.chat.model.ChatModel;
4647
import org.springframework.ai.chat.model.ChatResponse;
@@ -783,6 +784,59 @@ void testConversationHistoryPromptCachingWithClaude() {
783784
});
784785
}
785786

787+
@Test
788+
void testToolResultPromptCachingWithClaude() {
789+
// Keep the prefix before the tool result deliberately small. A cache read above
790+
// Claude Haiku 4.5's 4096-token minimum therefore demonstrates that the large
791+
// tool result is included in the cached prefix.
792+
String model = "us.anthropic.claude-haiku-4-5-20251001-v1:0";
793+
StringBuilder largeToolResult = new StringBuilder("Detailed hourly weather report for San Francisco.\n");
794+
for (int hour = 0; hour < 240; hour++) {
795+
largeToolResult.append("Hour ")
796+
.append(hour)
797+
.append(": temperature 18C, humidity 72%, wind 12 km/h from the west, visibility 10 km, ")
798+
.append("pressure 1013 hPa, no precipitation expected, UV index moderate.\n");
799+
}
800+
801+
String runMarker = "Session " + System.currentTimeMillis();
802+
BedrockCacheOptions cacheOptions = BedrockCacheOptions.builder()
803+
.strategy(BedrockCacheStrategy.CONVERSATION_HISTORY)
804+
.cacheToolResults(true)
805+
.build();
806+
BedrockChatOptions chatOptions = BedrockChatOptions.builder()
807+
.model(model)
808+
.cacheOptions(cacheOptions)
809+
.toolCallbacks(List.of(FunctionToolCallback.builder("getCurrentWeather", new MockWeatherService())
810+
.description("Get current weather for a location")
811+
.inputType(MockWeatherService.Request.class)
812+
.build()))
813+
.maxTokens(80)
814+
.build();
815+
816+
List<Message> conversation = List.of(new SystemMessage("You are a weather assistant. " + runMarker),
817+
new UserMessage("What's the detailed weather in San Francisco?"),
818+
AssistantMessage.builder()
819+
.content("")
820+
.toolCalls(List.of(new AssistantMessage.ToolCall("tool_sf_1", "function", "getCurrentWeather",
821+
"{\"location\":\"San Francisco, CA\",\"unit\":\"C\"}")))
822+
.build(),
823+
ToolResponseMessage.builder()
824+
.responses(List.of(new ToolResponseMessage.ToolResponse("tool_sf_1", "getCurrentWeather",
825+
largeToolResult.toString())))
826+
.build());
827+
Prompt prompt = new Prompt(conversation, chatOptions);
828+
829+
// Cross-region inference can route the first calls to different regions. Retry
830+
// the identical prompt until it reaches a region where the prefix was written.
831+
Awaitility.await().atMost(Duration.ofMinutes(2)).pollInterval(Duration.ofSeconds(3)).untilAsserted(() -> {
832+
ChatResponse response = this.chatModel.call(prompt);
833+
assertThat(response.getResults()).hasSize(1);
834+
assertThat(response.getResult().getOutput().getText()).isNotEmpty();
835+
Integer cacheRead = response.getMetadata().get("cacheReadInputTokens");
836+
assertThat(cacheRead).as("Should read the cached tool result prefix").isNotNull().isGreaterThan(4096);
837+
});
838+
}
839+
786840
/**
787841
* Helper method to create multiple tool callbacks with descriptions large enough to
788842
* exceed the 4096 token minimum required by Claude Haiku 4.5 for prompt caching.

0 commit comments

Comments
 (0)