Skip to content

Commit cd5814e

Browse files
AtulJoshi1206copybara-github
authored andcommitted
fix: inject artifact text, not the Part repr, into instructions
Merge #7391 `{artifact.<name>}` and Jinja2 `artifact("<name>")` in instructions now convert `types.Part` artifacts via `load_artifacts_tool.as_safe_part_for_llm` instead of rendering `str(Part)`. PiperOrigin-RevId: 993763912
1 parent e9464ca commit cd5814e

3 files changed

Lines changed: 85 additions & 17 deletions

File tree

‎docs/guides/utils/instructions_utils/index.md‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,14 @@ the syntax does not make obvious.
9696
extra thought.
9797
* **`{artifact.filename}` loads a file instead of a state value.** The engine
9898
asks the artifact service for that artifact in the current session and
99-
substitutes `str()` of what comes back. A missing artifact raises
99+
converts a `types.Part` with `load_artifacts_tool.as_safe_part_for_llm`:
100+
`Part.text` and text-like `inline_data` (`text/*`, JSON, XML, CSV, or DOCX)
101+
are substituted as their text content, other binary blobs render as a
102+
`[Binary artifact: ...]` placeholder, and Gemini-native inline media (such
103+
as `image/*` or `application/pdf`) falls back to `str()`, which is the
104+
Pydantic field dump rather than prompt text. A missing artifact raises
100105
`KeyError`, and `{artifact.filename?}` substitutes an empty string in the
101-
same way the optional state form does. Check what that `str()` produces
102-
before relying on the form: an artifact fetched from an artifact service is
103-
a `types.Part`, and its `str()` is the whole Pydantic field dump rather than
104-
the text you had in mind.
106+
same way the optional state form does.
105107

106108
A state key may carry one of the state prefixes, so `{app:theme}`,
107109
`{user:locale}`, and `{temp:draft}` all work, and read from the corresponding
@@ -266,16 +268,17 @@ function to instructions. Anywhere you hold a `ReadonlyContext`, and a
266268
missing, so it raises. `{customer name}` is not a valid name, so it is left
267269
in the prompt untouched and the model sees the braces. The two typos fail in
268270
completely different ways.
269-
* **Artifacts are stringified, and that is almost never what you want.** Both
270-
engines insert `str(artifact)`, and an artifact loaded from an artifact
271-
service is a `types.Part`, whose `str()` is the Pydantic field dump. A
272-
plain-text artifact reading "hello world" reaches the prompt as
273-
`media_resolution=None code_execution_result=None ... text='hello world'
274-
thought=None ...`, which is every field of the Part when only one of them
275-
was what you wanted. Binary data is worse still. There is no option to
276-
extract `.text`, so an
277-
instruction that needs an artifact's contents should load it in Python and
278-
interpolate the text itself rather than using `{artifact.name}`.
271+
* **Binary and media artifacts cannot be inlined into an instruction.** Both
272+
engines pass a loaded `types.Part` through
273+
`load_artifacts_tool.as_safe_part_for_llm`, because an instruction is a
274+
plain string: `Part.text` and text-like `inline_data` (`text/*`, JSON, XML,
275+
CSV, or DOCX) unwrap to their text, other binary blobs render as a
276+
`[Binary artifact: name, type, size]` placeholder, and Gemini-native media
277+
(`image/*`, `audio/*`, `video/*`, `application/pdf`) or a `file_data`
278+
reference has no `.text` and falls back to `str(artifact)`, which inserts
279+
the Pydantic field dump into the prompt. Use `{artifact.name}` and
280+
`artifact('name')` only for text artifacts, and pass binary or media
281+
artifacts to the model as content parts instead.
279282
* **A callable instruction gets no injection.** The omission is deliberate,
280283
on the assumption that a function with access to the context can interpolate
281284
for itself, and it is the reason to call this function yourself.

‎src/google/adk/flows/llm_flows/prompt/_instructions_utils.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
from typing import Callable
2323
from typing import Union
2424

25+
from google.genai import types
2526
from typing_extensions import TypeAlias
2627

2728
from ....agents.readonly_context import ReadonlyContext
2829
from ....sessions.state import State
30+
from ....tools import load_artifacts_tool
2931

3032
__all__ = [
3133
'InstructionProvider',
@@ -43,6 +45,15 @@
4345
_TEMPLATE_VAR_PATTERN = re.compile(r'(?<![\$\{\\]){+[^{}]*}+')
4446

4547

48+
def _artifact_to_text(artifact: object, artifact_name: str) -> str:
49+
"""Renders a loaded artifact as instruction text."""
50+
if isinstance(artifact, types.Part):
51+
safe = load_artifacts_tool.as_safe_part_for_llm(artifact, artifact_name)
52+
if safe.text is not None:
53+
return safe.text
54+
return str(artifact)
55+
56+
4657
async def inject_session_state(
4758
template: str,
4859
readonly_context: ReadonlyContext,
@@ -164,7 +175,7 @@ async def _replace_match(match: re.Match[str]) -> str:
164175
f"Artifact '{var_name}' not found in agent"
165176
f" '{readonly_context.agent_name}'."
166177
)
167-
return str(artifact)
178+
return _artifact_to_text(artifact, var_name)
168179
else:
169180
if not _is_valid_state_name(var_name):
170181
return str(match.group())
@@ -238,7 +249,7 @@ async def _load_artifact(filename: str) -> str:
238249
f"Artifact '{filename}' not found in agent"
239250
f" '{readonly_context.agent_name}'."
240251
)
241-
return str(artifact)
252+
return _artifact_to_text(artifact, filename)
242253

243254
env = SandboxedEnvironment(
244255
enable_async=True,

‎tests/unittests/flows/llm_flows/prompt/test_instructions_utils.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@
1919
from google.adk.agents.llm_agent import Agent
2020
from google.adk.agents.llm_agent import InstructionProvider as LlmAgentInstructionProvider
2121
from google.adk.agents.readonly_context import ReadonlyContext
22+
from google.adk.artifacts import InMemoryArtifactService
2223
from google.adk.flows.llm_flows.prompt import _instructions_utils as instructions_utils
2324
from google.adk.flows.llm_flows.prompt._instructions_utils import _is_valid_state_name
2425
from google.adk.flows.llm_flows.prompt._instructions_utils import InstructionProvider
2526
from google.adk.sessions.session import Session
27+
from google.genai import types
2628
import pytest
2729

2830
from .... import testing_utils
@@ -566,3 +568,55 @@ async def test_inject_session_state_jinja2_state_mapping_is_read_only():
566568
use_jinja2=True,
567569
)
568570
assert invocation_context.session.state == {"user:name": "Foo", "count": 1}
571+
572+
573+
@pytest.mark.asyncio
574+
@pytest.mark.parametrize(
575+
"use_jinja2,template",
576+
[
577+
(False, "Notes: {artifact.notes}"),
578+
(True, "Notes: {{ artifact('notes') }}"),
579+
],
580+
ids=["regex", "jinja2"],
581+
)
582+
@pytest.mark.parametrize(
583+
"artifact,expected",
584+
[
585+
(types.Part(text="Buy milk."), "Notes: Buy milk."),
586+
(
587+
types.Part.from_bytes(data=b"Buy milk.", mime_type="text/plain"),
588+
"Notes: Buy milk.",
589+
),
590+
(
591+
types.Part.from_bytes(
592+
data=b"\x00\x01\x02\x03", mime_type="application/octet-stream"
593+
),
594+
(
595+
"Notes: [Binary artifact: notes, type:"
596+
" application/octet-stream, size: 0.0 KB. Content cannot be"
597+
" displayed inline.]"
598+
),
599+
),
600+
],
601+
ids=["text_part", "text_inline_data", "binary_inline_data"],
602+
)
603+
async def test_inject_session_state_artifact_part_renders_as_text(
604+
use_jinja2, template, artifact, expected
605+
):
606+
artifact_service = InMemoryArtifactService()
607+
await artifact_service.save_artifact(
608+
app_name="test_app",
609+
user_id="test_user",
610+
session_id="test_session_id",
611+
filename="notes",
612+
artifact=artifact,
613+
)
614+
invocation_context = await _create_test_readonly_context(
615+
artifact_service=artifact_service
616+
)
617+
618+
populated_instruction = await instructions_utils.inject_session_state(
619+
template, invocation_context, use_jinja2=use_jinja2
620+
)
621+
622+
assert populated_instruction == expected

0 commit comments

Comments
 (0)