Skip to content

Commit 68ff1b7

Browse files
committed
Validate the base64 payload of data URIs in the foundry_hosting image result
1 parent 57bb65d commit 68ff1b7

2 files changed

Lines changed: 46 additions & 8 deletions

File tree

‎python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3540,22 +3540,28 @@ def _json_safe_to_str(value: Any | None) -> str:
35403540
def _image_generation_result(outputs: Any) -> str | None:
35413541
"""Return the base64 image of an image generation tool result, as the Responses API expects.
35423542
3543-
Only a base64 string or the payload of a base64 data URI qualifies. Other URIs (https, file, blob) are not
3544-
base64, so they give None.
3543+
Only a non-empty, valid base64 string or the valid base64 payload of a base64 data URI qualifies. Other URIs
3544+
(https, file, blob), data URIs without the base64 marker and malformed payloads are not base64, so they give
3545+
None.
35453546
"""
35463547
if isinstance(outputs, str):
35473548
return outputs if _is_base64(outputs) else None
35483549
items: Sequence[Any] = [outputs]
35493550
if isinstance(outputs, Sequence) and not isinstance(outputs, (bytes, bytearray)):
35503551
items = cast(Sequence[Any], outputs)
35513552
for item in items:
3552-
if isinstance(item, Content) and item.uri and item.uri.startswith("data:") and ";base64," in item.uri:
3553-
return item.uri.split(";base64,", 1)[1]
3553+
if isinstance(item, Content) and item.uri and item.uri.startswith("data:"):
3554+
# RFC 2397: only the metadata before the first comma can carry the marker, as its last parameter.
3555+
metadata, _, payload = item.uri.partition(",")
3556+
if metadata.endswith(";base64") and _is_base64(payload):
3557+
return payload
35543558
return None
35553559

35563560

35573561
def _is_base64(value: str) -> bool:
3558-
"""Check whether a string is valid base64."""
3562+
"""Check whether a string is non-empty, valid base64."""
3563+
if not value:
3564+
return False
35593565
try:
35603566
base64.b64decode(value, validate=True)
35613567
except ValueError:

‎python/packages/foundry_hosting/tests/test_responses.py‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -627,10 +627,27 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text()
627627
([Content.from_text("caption"), Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="),
628628
(["caption", Content.from_uri("data:image/png;base64,iVBORw0KGgo=")], "iVBORw0KGgo="),
629629
("iVBORw0KGgo=", "iVBORw0KGgo="),
630+
(Content.from_uri("data:image/png;name=cat.png;base64,iVBORw0KGgo="), "iVBORw0KGgo="),
631+
(
632+
[
633+
Content.from_uri("data:image/png;base64,not-base64!"),
634+
Content.from_uri("data:image/png;base64,iVBORw0KGgo="),
635+
],
636+
"iVBORw0KGgo=",
637+
),
630638
(Content.from_uri("https://example.com/cat.png", media_type="image/png"), ""),
631639
(Content.from_uri("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"), ""),
640+
(Content.from_uri("data:image/png;base64,not-base64!"), ""),
641+
(Content.from_uri("data:image/png;base64,iVBORw0KGgo"), ""),
642+
(Content.from_uri("data:image/png;base64,iVBOR\nw0KGgo="), ""),
643+
(Content.from_uri("data:image/png;base64,"), ""),
644+
(Content.from_uri("data:text/plain,x;base64,iVBORw0KGgo="), ""),
632645
(Content.from_hosted_file("file_1"), ""),
633646
("https://example.com/cat.png", ""),
647+
("not-base64!", ""),
648+
("", ""),
649+
(b"\x89PNG\r\n\x1a\n", ""),
650+
({"result": "iVBORw0KGgo="}, ""),
634651
],
635652
ids=[
636653
"data-uri-content",
@@ -639,27 +656,42 @@ async def test_output_item_tracker_emits_native_refusal_events_for_marked_text()
639656
"image-after-text",
640657
"image-after-string",
641658
"base64-string",
659+
"data-uri-with-parameter",
660+
"image-after-malformed-payload",
642661
"https-uri-is-not-base64",
643662
"data-uri-without-base64",
663+
"data-uri-malformed-payload",
664+
"data-uri-missing-padding",
665+
"data-uri-whitespace-in-payload",
666+
"data-uri-empty-payload",
667+
"base64-marker-inside-payload",
644668
"hosted-file",
645669
"url-string-is-not-base64",
670+
"malformed-string",
671+
"empty-string",
672+
"bytes",
673+
"mapping",
646674
],
647675
)
648-
async def test_output_item_tracker_emits_image_generation_result_as_base64(outputs: Any, expected: str) -> None:
676+
async def test_output_item_tracker_emits_image_generation_result_as_base64(
677+
outputs: Any, expected: str, caplog: pytest.LogCaptureFixture
678+
) -> None:
649679
stream = ResponseEventStream(response_id="resp_image")
650680
stream.emit_created()
651681
stream.emit_in_progress()
652682
tracker = _OutputItemTracker(stream)
653683
events: list[Any] = []
654684

655-
async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)):
656-
events.append(event)
685+
with caplog.at_level(logging.WARNING):
686+
async for event in tracker.handle(Content.from_image_generation_tool_result(image_id="ig_1", outputs=outputs)):
687+
events.append(event)
657688
events.extend(tracker.close())
658689

659690
done = [e for e in events if (e.get("type") if isinstance(e, Mapping) else e.type) == "response.output_item.done"]
660691
assert len(done) == 1
661692
assert done[0]["item"]["type"] == "image_generation_call"
662693
assert done[0]["item"]["result"] == expected
694+
assert ("has no base64 image" in caplog.text) is (expected == "")
663695

664696

665697
async def test_output_item_tracker_keeps_mixed_text_and_refusal_in_one_message() -> None:

0 commit comments

Comments
 (0)