|
41 | 41 | import org.springframework.ai.chat.messages.AssistantMessage; |
42 | 42 | import org.springframework.ai.chat.messages.Message; |
43 | 43 | import org.springframework.ai.chat.messages.SystemMessage; |
| 44 | +import org.springframework.ai.chat.messages.ToolResponseMessage; |
44 | 45 | import org.springframework.ai.chat.messages.UserMessage; |
45 | 46 | import org.springframework.ai.chat.model.ChatModel; |
46 | 47 | import org.springframework.ai.chat.model.ChatResponse; |
@@ -783,6 +784,59 @@ void testConversationHistoryPromptCachingWithClaude() { |
783 | 784 | }); |
784 | 785 | } |
785 | 786 |
|
| 787 | + @Test |
| 788 | + void testToolResultPromptCachingWithClaude() { |
| 789 | + // Keep the prefix before the tool result deliberately small. A cache read above |
| 790 | + // Claude Haiku 4.5's 4096-token minimum therefore demonstrates that the large |
| 791 | + // tool result is included in the cached prefix. |
| 792 | + String model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"; |
| 793 | + StringBuilder largeToolResult = new StringBuilder("Detailed hourly weather report for San Francisco.\n"); |
| 794 | + for (int hour = 0; hour < 240; hour++) { |
| 795 | + largeToolResult.append("Hour ") |
| 796 | + .append(hour) |
| 797 | + .append(": temperature 18C, humidity 72%, wind 12 km/h from the west, visibility 10 km, ") |
| 798 | + .append("pressure 1013 hPa, no precipitation expected, UV index moderate.\n"); |
| 799 | + } |
| 800 | + |
| 801 | + String runMarker = "Session " + System.currentTimeMillis(); |
| 802 | + BedrockCacheOptions cacheOptions = BedrockCacheOptions.builder() |
| 803 | + .strategy(BedrockCacheStrategy.CONVERSATION_HISTORY) |
| 804 | + .cacheToolResults(true) |
| 805 | + .build(); |
| 806 | + BedrockChatOptions chatOptions = BedrockChatOptions.builder() |
| 807 | + .model(model) |
| 808 | + .cacheOptions(cacheOptions) |
| 809 | + .toolCallbacks(List.of(FunctionToolCallback.builder("getCurrentWeather", new MockWeatherService()) |
| 810 | + .description("Get current weather for a location") |
| 811 | + .inputType(MockWeatherService.Request.class) |
| 812 | + .build())) |
| 813 | + .maxTokens(80) |
| 814 | + .build(); |
| 815 | + |
| 816 | + List<Message> conversation = List.of(new SystemMessage("You are a weather assistant. " + runMarker), |
| 817 | + new UserMessage("What's the detailed weather in San Francisco?"), |
| 818 | + AssistantMessage.builder() |
| 819 | + .content("") |
| 820 | + .toolCalls(List.of(new AssistantMessage.ToolCall("tool_sf_1", "function", "getCurrentWeather", |
| 821 | + "{\"location\":\"San Francisco, CA\",\"unit\":\"C\"}"))) |
| 822 | + .build(), |
| 823 | + ToolResponseMessage.builder() |
| 824 | + .responses(List.of(new ToolResponseMessage.ToolResponse("tool_sf_1", "getCurrentWeather", |
| 825 | + largeToolResult.toString()))) |
| 826 | + .build()); |
| 827 | + Prompt prompt = new Prompt(conversation, chatOptions); |
| 828 | + |
| 829 | + // Cross-region inference can route the first calls to different regions. Retry |
| 830 | + // the identical prompt until it reaches a region where the prefix was written. |
| 831 | + Awaitility.await().atMost(Duration.ofMinutes(2)).pollInterval(Duration.ofSeconds(3)).untilAsserted(() -> { |
| 832 | + ChatResponse response = this.chatModel.call(prompt); |
| 833 | + assertThat(response.getResults()).hasSize(1); |
| 834 | + assertThat(response.getResult().getOutput().getText()).isNotEmpty(); |
| 835 | + Integer cacheRead = response.getMetadata().get("cacheReadInputTokens"); |
| 836 | + assertThat(cacheRead).as("Should read the cached tool result prefix").isNotNull().isGreaterThan(4096); |
| 837 | + }); |
| 838 | + } |
| 839 | + |
786 | 840 | /** |
787 | 841 | * Helper method to create multiple tool callbacks with descriptions large enough to |
788 | 842 | * exceed the 4096 token minimum required by Claude Haiku 4.5 for prompt caching. |
|
0 commit comments