Skip to content

Commit 5084b26

Browse files
elliottlawsonelliottlawson
committed
fix: prevent tool result cache accumulation across multiple tool calls
Apply tool_result_cache_type only to the last tool result message across all messages instead of applying it to each tool result during creation. This prevents cache accumulation when there are multiple tool call rounds.
1 parent a1a0cc8 commit 5084b26

File tree

4 files changed

+270
-12
lines changed

4 files changed

+270
-12
lines changed

‎src/Providers/Anthropic/Handlers/Stream.php‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -617,12 +617,6 @@ protected function addMessagesToRequest(Request $request, array $toolResults, ?a
617617

618618
$message = new ToolResultMessage($toolResults);
619619

620-
// Apply tool result caching if configured
621-
$tool_result_cache_type = $request->providerOptions('tool_result_cache_type');
622-
if ($tool_result_cache_type) {
623-
$message->withProviderOptions(['cacheType' => $tool_result_cache_type]);
624-
}
625-
626620
$request->addMessage($message);
627621
}
628622

‎src/Providers/Anthropic/Handlers/Text.php‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,6 @@ protected function handleToolCalls(): Response
104104
$toolResults = $this->callTools($this->request->tools(), $this->tempResponse->toolCalls);
105105
$message = new ToolResultMessage($toolResults);
106106

107-
// Apply tool result caching if configured
108-
if ($tool_result_cache_type = $this->request->providerOptions('tool_result_cache_type')) {
109-
$message->withProviderOptions(['cacheType' => $tool_result_cache_type]);
110-
}
111-
112107
$this->request->addMessage($message);
113108

114109
$this->addStep($toolResults);

‎src/Providers/Anthropic/Maps/MessageMap.php‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,32 @@ public static function map(array $messages, array $requestProviderOptions = []):
3030
throw new PrismException('Anthropic does not support SystemMessages in the messages array. Use withSystemPrompt or withSystemPrompts instead.');
3131
}
3232

33-
return array_map(
33+
$mappedMessages = array_map(
3434
fn (Message $message): array => self::mapMessage($message, $requestProviderOptions),
3535
$messages
3636
);
37+
38+
if (isset($requestProviderOptions['tool_result_cache_type'])) {
39+
$lastToolResultIndex = null;
40+
41+
for ($i = count($mappedMessages) - 1; $i >= 0; $i--) {
42+
if ($mappedMessages[$i]['role'] === 'user' &&
43+
isset($mappedMessages[$i]['content'][0]['type']) &&
44+
$mappedMessages[$i]['content'][0]['type'] === 'tool_result') {
45+
$lastToolResultIndex = $i;
46+
break;
47+
}
48+
}
49+
50+
if ($lastToolResultIndex !== null) {
51+
$lastContent = &$mappedMessages[$lastToolResultIndex]['content'];
52+
$lastContent[count($lastContent) - 1]['cache_control'] = [
53+
'type' => $requestProviderOptions['tool_result_cache_type'],
54+
];
55+
}
56+
}
57+
58+
return $mappedMessages;
3759
}
3860

3961
/**

‎tests/Providers/Anthropic/ToolResultCachingTest.php‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Tests\Providers\Anthropic;
6+
7+
use Prism\Prism\Prism;
8+
use Prism\Prism\Providers\Anthropic\Handlers\Text;
9+
use Prism\Prism\Providers\Anthropic\Maps\MessageMap;
10+
use Prism\Prism\ValueObjects\Messages\AssistantMessage;
11+
use Prism\Prism\ValueObjects\Messages\ToolResultMessage;
12+
use Prism\Prism\ValueObjects\Messages\UserMessage;
13+
use Prism\Prism\ValueObjects\ToolCall;
14+
use Prism\Prism\ValueObjects\ToolResult;
15+
16+
beforeEach(function (): void {
17+
config()->set('prism.providers.anthropic.api_key', env('ANTHROPIC_API_KEY', 'sk-1234'));
18+
});
19+
20+
it('applies tool_result_cache_type only to the last tool result message across all messages', function (): void {
21+
// Create test messages simulating multiple tool call rounds
22+
$messages = [
23+
new UserMessage('What time is the tigers game today and should I wear a coat?'),
24+
new AssistantMessage('', toolCalls: [
25+
new ToolCall(
26+
id: 'call_1',
27+
name: 'search',
28+
arguments: ['query' => 'Detroit Tigers baseball game time today']
29+
),
30+
]),
31+
new ToolResultMessage([
32+
new ToolResult(
33+
toolCallId: 'call_1',
34+
toolName: 'search',
35+
args: ['query' => 'Detroit Tigers baseball game time today'],
36+
result: 'The tigers game is at 3pm in detroit'
37+
),
38+
]),
39+
new AssistantMessage('', toolCalls: [
40+
new ToolCall(
41+
id: 'call_2',
42+
name: 'weather',
43+
arguments: ['city' => 'Detroit']
44+
),
45+
]),
46+
new ToolResultMessage([
47+
new ToolResult(
48+
toolCallId: 'call_2',
49+
toolName: 'weather',
50+
args: ['city' => 'Detroit'],
51+
result: 'The weather will be 75° and sunny'
52+
),
53+
]),
54+
new AssistantMessage('The Tigers game is at 3pm today. The weather will be 75° and sunny, so you won\'t need a coat!'),
55+
];
56+
57+
// Map the messages with provider options
58+
$mappedMessages = MessageMap::map(
59+
$messages,
60+
['tool_result_cache_type' => 'ephemeral']
61+
);
62+
63+
// Verify that only the last tool result message has cache_control
64+
$toolResultMessages = array_filter($mappedMessages, fn ($message): bool => $message['role'] === 'user' &&
65+
isset($message['content'][0]['type']) &&
66+
$message['content'][0]['type'] === 'tool_result');
67+
68+
expect(count($toolResultMessages))->toBe(2);
69+
70+
// Get the tool result messages by their indices
71+
$toolResultIndices = array_keys($toolResultMessages);
72+
$firstToolResultIndex = $toolResultIndices[0];
73+
$lastToolResultIndex = $toolResultIndices[1];
74+
75+
// First tool result should NOT have cache_control
76+
$firstToolResult = $mappedMessages[$firstToolResultIndex];
77+
expect($firstToolResult['content'][0])->not->toHaveKey('cache_control');
78+
79+
// Last tool result SHOULD have cache_control
80+
$lastToolResult = $mappedMessages[$lastToolResultIndex];
81+
expect($lastToolResult['content'][0])->toHaveKey('cache_control');
82+
expect($lastToolResult['content'][0]['cache_control'])->toBe(['type' => 'ephemeral']);
83+
});
84+
85+
it('handles single tool result message with cache_control', function (): void {
86+
$messages = [
87+
new UserMessage('What is the weather?'),
88+
new AssistantMessage('', toolCalls: [
89+
new ToolCall(
90+
id: 'call_1',
91+
name: 'weather',
92+
arguments: ['city' => 'Detroit']
93+
),
94+
]),
95+
new ToolResultMessage([
96+
new ToolResult(
97+
toolCallId: 'call_1',
98+
toolName: 'weather',
99+
args: ['city' => 'Detroit'],
100+
result: 'The weather will be 75° and sunny'
101+
),
102+
]),
103+
];
104+
105+
// Map the messages with provider options
106+
$mappedMessages = MessageMap::map(
107+
$messages,
108+
['tool_result_cache_type' => 'ephemeral']
109+
);
110+
111+
// Find the tool result message
112+
$toolResultMessage = null;
113+
foreach ($mappedMessages as $message) {
114+
if ($message['role'] === 'user' &&
115+
isset($message['content'][0]['type']) &&
116+
$message['content'][0]['type'] === 'tool_result') {
117+
$toolResultMessage = $message;
118+
break;
119+
}
120+
}
121+
122+
// The single tool result should have cache_control
123+
expect($toolResultMessage)->not->toBeNull();
124+
expect($toolResultMessage['content'][0])->toHaveKey('cache_control');
125+
expect($toolResultMessage['content'][0]['cache_control'])->toBe(['type' => 'ephemeral']);
126+
});
127+
128+
it('does not apply cache_control when tool_result_cache_type is not set', function (): void {
129+
$messages = [
130+
new UserMessage('What is the weather?'),
131+
new AssistantMessage('', toolCalls: [
132+
new ToolCall(
133+
id: 'call_1',
134+
name: 'weather',
135+
arguments: ['city' => 'Detroit']
136+
),
137+
]),
138+
new ToolResultMessage([
139+
new ToolResult(
140+
toolCallId: 'call_1',
141+
toolName: 'weather',
142+
args: ['city' => 'Detroit'],
143+
result: 'The weather will be 75° and sunny'
144+
),
145+
]),
146+
];
147+
148+
// Map the messages without provider options
149+
$mappedMessages = MessageMap::map($messages);
150+
151+
// Find the tool result message
152+
$toolResultMessage = null;
153+
foreach ($mappedMessages as $message) {
154+
if ($message['role'] === 'user' &&
155+
isset($message['content'][0]['type']) &&
156+
$message['content'][0]['type'] === 'tool_result') {
157+
$toolResultMessage = $message;
158+
break;
159+
}
160+
}
161+
162+
// Should not have cache_control
163+
expect($toolResultMessage)->not->toBeNull();
164+
expect($toolResultMessage['content'][0])->not->toHaveKey('cache_control');
165+
});
166+
167+
it('sends only one cache block when request has multiple tool results in full lifecycle', function (): void {
168+
Prism::fake();
169+
170+
// Simulate a request that already has multiple tool call rounds in history
171+
$request = Prism::text()
172+
->using('anthropic', 'claude-3-5-sonnet-latest')
173+
->withMessages([
174+
new UserMessage('What time is the game and weather?'),
175+
new AssistantMessage('', toolCalls: [
176+
new ToolCall(
177+
id: 'call_1',
178+
name: 'search',
179+
arguments: ['query' => 'game time']
180+
),
181+
]),
182+
new ToolResultMessage([
183+
new ToolResult(
184+
toolCallId: 'call_1',
185+
toolName: 'search',
186+
args: ['query' => 'game time'],
187+
result: '3pm'
188+
),
189+
]),
190+
new AssistantMessage('', toolCalls: [
191+
new ToolCall(
192+
id: 'call_2',
193+
name: 'weather',
194+
arguments: ['city' => 'Detroit']
195+
),
196+
]),
197+
new ToolResultMessage([
198+
new ToolResult(
199+
toolCallId: 'call_2',
200+
toolName: 'weather',
201+
args: ['city' => 'Detroit'],
202+
result: 'sunny'
203+
),
204+
]),
205+
])
206+
->withProviderOptions(['tool_result_cache_type' => 'ephemeral']);
207+
208+
// Get the actual payload that would be sent
209+
$payload = Text::buildHttpRequestPayload($request->toRequest());
210+
211+
// Count cache blocks in the payload
212+
$cacheBlocks = 0;
213+
foreach ($payload['messages'] as $message) {
214+
foreach ($message['content'] as $content) {
215+
if (isset($content['cache_control'])) {
216+
$cacheBlocks++;
217+
}
218+
}
219+
}
220+
221+
expect($cacheBlocks)->toBe(1);
222+
223+
// Find the last tool result message
224+
$lastToolResultIndex = null;
225+
for ($i = count($payload['messages']) - 1; $i >= 0; $i--) {
226+
if ($payload['messages'][$i]['role'] === 'user' &&
227+
isset($payload['messages'][$i]['content'][0]['type']) &&
228+
$payload['messages'][$i]['content'][0]['type'] === 'tool_result') {
229+
$lastToolResultIndex = $i;
230+
break;
231+
}
232+
}
233+
234+
// Verify the cache is on the last tool result
235+
expect($lastToolResultIndex)->not->toBeNull();
236+
expect($payload['messages'][$lastToolResultIndex]['content'][0])->toHaveKey('cache_control');
237+
expect($payload['messages'][$lastToolResultIndex]['content'][0]['cache_control'])->toBe(['type' => 'ephemeral']);
238+
239+
// Verify earlier tool results don't have cache
240+
for ($i = 0; $i < $lastToolResultIndex; $i++) {
241+
if ($payload['messages'][$i]['role'] === 'user' &&
242+
isset($payload['messages'][$i]['content'][0]['type']) &&
243+
$payload['messages'][$i]['content'][0]['type'] === 'tool_result') {
244+
expect($payload['messages'][$i]['content'][0])->not->toHaveKey('cache_control');
245+
}
246+
}
247+
});

0 commit comments

Comments
 (0)