Skip to content

Commit 2704b4c

Browse files
fix: tell the model when an animation becomes a frame montage (#10119)
* fix: tell the model when an animation becomes a frame montage Animated inputs (GIF/APNG/animated WebP) are tiled into a 3x3 frame grid before being sent to the model, but nothing described that transform, so a contact sheet read as a single picture (reported as "nine identical characters" for a three-frame GIF). prepare_model_image now returns PreparedModelImage, which carries the tiled frame count next to the working path, and the request preparation appends a transient notice to the model content describing the grid, the reading order, the number of tiled frames, and the blank cells. The notice is only built for montages, marked as not persisted, and still images are untouched. * fix: simplify animation montage notices * fix: clarify single-image animation montage notice * fix: shorten animation notice and hide frame layout details * fix: describe converted GIF input directly --------- Co-authored-by: Pleiades1726 <[email protected]> Co-authored-by: Soulter <[email protected]>
1 parent e0aa8d3 commit 2704b4c

4 files changed

Lines changed: 146 additions & 69 deletions

File tree

‎astrbot/core/pipeline/process_stage/method/agent_sub_stages/image_input.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from pathlib import Path
44

55
from astrbot.core import logger
6-
from astrbot.core.agent.message import ImageURLPart
6+
from astrbot.core.agent.message import ImageURLPart, TextPart
77
from astrbot.core.platform.astr_message_event import AstrMessageEvent
88
from astrbot.core.provider.entities import ProviderRequest
99
from astrbot.core.utils.media_utils import (
@@ -49,19 +49,22 @@ async def prepare_request_images(
4949
if quote_image_ref:
5050
refs.append(quote_image_ref)
5151
failed = False
52+
has_montage = False
5253
for ref in dict.fromkeys(refs):
5354
if ref not in prepared:
5455
path = None
5556
if enabled:
56-
path = await prepare_model_image(
57+
image = await prepare_model_image(
5758
ref,
5859
max_size=max_size,
5960
output_dir=output_dir,
6061
quality=quality,
6162
montage_max_size=montage_max_size,
6263
)
63-
if path:
64+
if image:
65+
path, is_montage = image
6466
event.track_temporary_local_file(path)
67+
has_montage |= is_montage
6568
else:
6669
try:
6770
async with MediaResolver(
@@ -118,3 +121,16 @@ async def prepare_request_images(
118121
for part in parts
119122
):
120123
req.prompt = "[Image unavailable]"
124+
125+
if has_montage:
126+
# Transient per-request hint; the montage file itself is not persisted.
127+
req.extra_user_content_parts = [
128+
*req.extra_user_content_parts,
129+
TextPart(
130+
text="<system_notice>\n"
131+
"The input includes a GIF converted into a single image with frames "
132+
"in reading order. Treat it as an animation; "
133+
"do not mention the conversion or frame layout.\n"
134+
"</system_notice>"
135+
).mark_as_temp(),
136+
]

‎astrbot/core/utils/media_utils.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,7 +1367,7 @@ async def prepare_model_image(
13671367
output_dir: Path,
13681368
quality: int = IMAGE_COMPRESS_DEFAULT_QUALITY,
13691369
montage_max_size: int | None = None,
1370-
) -> str | None:
1370+
) -> tuple[str, bool] | None:
13711371
"""Prepare a single local model-ready image for the caller to own until consumption.
13721372
13731373
Args:
@@ -1382,15 +1382,17 @@ async def prepare_model_image(
13821382
``max_size``.
13831383
13841384
Returns:
1385-
An existing JPEG or PNG path, or None for a recoverable input or write
1386-
failure.
1385+
Tuple of an existing JPEG or PNG path and whether it is an animation
1386+
frame montage (including cache hits), or None for a recoverable input
1387+
or write failure.
13871388
The caller owns this file; shared cache entries are never returned.
13881389
"""
13891390
try:
13901391
async with MediaResolver(image_ref, media_type="image").as_path() as source:
13911392
image_bytes = await asyncio.to_thread(source.read_bytes)
13921393
frame_count = await asyncio.to_thread(_inspect_image, image_bytes)
1393-
if frame_count > 1:
1394+
is_montage = frame_count > 1
1395+
if is_montage:
13941396
converted_bytes, _ = await asyncio.to_thread(
13951397
_extract_animation_montage_sync,
13961398
image_bytes,
@@ -1415,7 +1417,7 @@ async def prepare_model_image(
14151417
except BaseException:
14161418
output_path.unlink(missing_ok=True)
14171419
raise
1418-
return str(output_path)
1420+
return str(output_path), is_montage
14191421
except Exception as exc:
14201422
if not is_recoverable_image_error(exc):
14211423
raise

‎tests/test_model_image_preparation.py‎

Lines changed: 86 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ def isolated_cache(tmp_path, monkeypatch):
1818
monkeypatch.setattr(media, "get_astrbot_temp_path", lambda: str(tmp_path / "cache"))
1919

2020

21+
async def _prepare_image_path(*args, **kwargs):
22+
"""Path-only view of prepare_model_image for pixel-focused assertions."""
23+
prepared = await media.prepare_model_image(*args, **kwargs)
24+
return None if prepared is None else prepared[0]
25+
26+
2127
@pytest.mark.asyncio
2228
@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "BMP", "WEBP", "GIF"])
2329
async def test_oversized_opaque_stills_become_jpeg_without_mutating_source(
@@ -26,9 +32,7 @@ async def test_oversized_opaque_stills_become_jpeg_without_mutating_source(
2632
source = tmp_path / "misleading.jpg"
2733
Image.new("RGB", (200, 100), "red").save(source, fmt)
2834
original = source.read_bytes()
29-
path = await media.prepare_model_image(
30-
str(source), max_size=80, output_dir=tmp_path
31-
)
35+
path = await _prepare_image_path(str(source), max_size=80, output_dir=tmp_path)
3236
assert path and Path(path).is_file() and Path(path) != source
3337
assert path.endswith(".jpg")
3438
with Image.open(path) as image:
@@ -44,9 +48,7 @@ async def test_compliant_stills_pass_through_byte_identical(tmp_path, fmt):
4448
source = tmp_path / f"ok.{fmt.lower()}"
4549
Image.new("RGB", (200, 100), "red").save(source, fmt)
4650
original = source.read_bytes()
47-
path = await media.prepare_model_image(
48-
str(source), max_size=1280, output_dir=tmp_path
49-
)
51+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
5052
assert path and Path(path).read_bytes() == original
5153
assert source.read_bytes() == original
5254

@@ -58,9 +60,7 @@ async def test_rotated_jpeg_is_normalized_to_upright_jpeg(tmp_path):
5860
exif[274] = 6
5961
Image.new("RGB", (200, 100), "red").save(source, exif=exif)
6062
original = source.read_bytes()
61-
path = await media.prepare_model_image(
62-
str(source), max_size=1280, output_dir=tmp_path
63-
)
63+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
6464
assert path and Path(path).read_bytes() != original
6565
with Image.open(path) as image:
6666
assert image.format == "JPEG" and image.size == (100, 200)
@@ -73,9 +73,7 @@ async def test_oversized_alpha_png_flattens_to_jpeg(tmp_path, monkeypatch):
7373
monkeypatch.setattr(media, "MODEL_IMAGE_PNG_FALLBACK_MAX_BYTES", 1)
7474
source = tmp_path / "alpha.png"
7575
Image.new("RGBA", (2000, 100), (255, 0, 0, 128)).save(source)
76-
path = await media.prepare_model_image(
77-
str(source), max_size=1280, output_dir=tmp_path
78-
)
76+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
7977
assert path and path.endswith(".jpg")
8078
with Image.open(path) as image:
8179
assert image.format == "JPEG" and image.mode == "RGB"
@@ -91,9 +89,7 @@ async def test_16bit_jpeg_conversion_scales_instead_of_clipping(tmp_path):
9189
int(x * 65535 / (size[0] - 1)).to_bytes(2, "little") for x in range(size[0])
9290
)
9391
Image.frombytes("I;16", size, row * size[1]).save(source)
94-
path = await media.prepare_model_image(
95-
str(source), max_size=1280, output_dir=tmp_path
96-
)
92+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
9793
assert path and path.endswith(".jpg")
9894
with Image.open(path) as image:
9995
assert image.format == "JPEG" and image.mode == "L"
@@ -107,9 +103,7 @@ async def test_jpeg_transcode_preserves_icc_profile(tmp_path):
107103
icc = ImageCms.ImageCmsProfile(ImageCms.createProfile("sRGB")).tobytes()
108104
source = tmp_path / "photo.jpg"
109105
Image.new("RGB", (2000, 100), "red").save(source, icc_profile=icc)
110-
path = await media.prepare_model_image(
111-
str(source), max_size=1280, output_dir=tmp_path
112-
)
106+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
113107
assert path and path.endswith(".jpg")
114108
with Image.open(path) as result:
115109
assert result.format == "JPEG"
@@ -122,9 +116,7 @@ async def test_cmyk_icc_profile_is_not_attached_to_rgb_jpeg(tmp_path):
122116
icc = ImageCms.ImageCmsProfile(ImageCms.createProfile("sRGB")).tobytes()
123117
source = tmp_path / "cmyk.jpg"
124118
Image.new("CMYK", (2000, 100), (0, 255, 255, 0)).save(source, icc_profile=icc)
125-
path = await media.prepare_model_image(
126-
str(source), max_size=1280, output_dir=tmp_path
127-
)
119+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
128120
assert path and path.endswith(".jpg")
129121
with Image.open(path) as result:
130122
assert result.format == "JPEG" and result.mode == "RGB"
@@ -135,10 +127,10 @@ async def test_cmyk_icc_profile_is_not_attached_to_rgb_jpeg(tmp_path):
135127
async def test_jpeg_quality_is_honored_and_cached_separately(tmp_path):
136128
source = tmp_path / "photo.bmp"
137129
Image.new("RGB", (200, 100), "red").save(source)
138-
low = await media.prepare_model_image(
130+
low = await _prepare_image_path(
139131
str(source), max_size=1280, output_dir=tmp_path, quality=10
140132
)
141-
high = await media.prepare_model_image(
133+
high = await _prepare_image_path(
142134
str(source), max_size=1280, output_dir=tmp_path, quality=95
143135
)
144136
assert low and high and low != high
@@ -169,9 +161,7 @@ async def test_animation_sampling_cover_and_blank_cells(tmp_path, fmt, cover, co
169161
**({"default_image": cover} if fmt == "PNG" else {}),
170162
)
171163
original = source.read_bytes()
172-
path = await media.prepare_model_image(
173-
str(source), max_size=1280, output_dir=tmp_path
174-
)
164+
path = await _prepare_image_path(str(source), max_size=1280, output_dir=tmp_path)
175165
indices = [0, 1] if count == 2 else [0, 1, 3, 4, 6, 7, 8, 10, 11]
176166
with Image.open(path) as image:
177167
assert image.format == "JPEG" and getattr(image, "n_frames", 1) == 1
@@ -180,9 +170,7 @@ async def test_animation_sampling_cover_and_blank_cells(tmp_path, fmt, cover, co
180170
expected = colors[indices[cell]] if cell < len(indices) else (255, 255, 255)
181171
pixel = image.getpixel(((cell % 3) * 12 + 6, (cell // 3) * 8 + 4))
182172
assert all(abs(actual - want) <= 6 for actual, want in zip(pixel, expected))
183-
small = await media.prepare_model_image(
184-
str(source), max_size=18, output_dir=tmp_path
185-
)
173+
small = await _prepare_image_path(str(source), max_size=18, output_dir=tmp_path)
186174
with Image.open(small) as image:
187175
assert image.size == (18, 12)
188176
assert source.read_bytes() == original
@@ -196,9 +184,7 @@ async def test_exif_and_cmyk_are_preserved_for_display(tmp_path):
196184
image = Image.new("CMYK", (200, 100), (0, 255, 255, 0))
197185
image.save(source, exif=exif)
198186
original = source.read_bytes()
199-
path = await media.prepare_model_image(
200-
str(source), max_size=80, output_dir=tmp_path
201-
)
187+
path = await _prepare_image_path(str(source), max_size=80, output_dir=tmp_path)
202188
with Image.open(path) as result:
203189
assert (
204190
result.format == "JPEG" and result.size == (40, 80) and result.mode == "RGB"
@@ -220,9 +206,7 @@ async def test_alpha_and_color_key_survive_downscaling(tmp_path, mode, color):
220206
Image.new(mode, (200, 100), color).save(
221207
source, **({} if mode == "RGBA" else {"transparency": color})
222208
)
223-
path = await media.prepare_model_image(
224-
str(source), max_size=80, output_dir=tmp_path
225-
)
209+
path = await _prepare_image_path(str(source), max_size=80, output_dir=tmp_path)
226210
with Image.open(path) as result:
227211
assert result.size == (80, 40) and result.mode == "RGBA"
228212
assert result.getpixel((40, 20))[3] == (128 if mode == "RGBA" else 0)
@@ -240,9 +224,7 @@ async def test_palette_and_binary_thin_lines_use_lanczos(tmp_path, mode):
240224
image.save(source)
241225
expected = image.convert("RGB" if mode == "P" else "L")
242226
expected.thumbnail((16, 16), Image.Resampling.LANCZOS)
243-
path = await media.prepare_model_image(
244-
str(source), max_size=16, output_dir=tmp_path
245-
)
227+
path = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
246228
with Image.open(path) as result:
247229
actual = result.convert("L")
248230
assert actual.getextrema()[0] < 255
@@ -262,9 +244,7 @@ async def test_16bit_gradient_normalizes_to_8bit_jpeg(tmp_path, size, limit):
262244
)
263245
Image.frombytes("I;16", size, row * size[1]).save(source)
264246
original = source.read_bytes()
265-
path = await media.prepare_model_image(
266-
str(source), max_size=limit, output_dir=tmp_path
267-
)
247+
path = await _prepare_image_path(str(source), max_size=limit, output_dir=tmp_path)
268248
with Image.open(path) as result:
269249
assert result.format == "JPEG" and result.mode == "L"
270250
assert result.size == (limit, limit // 2)
@@ -299,9 +279,7 @@ async def test_bad_cache_rebuilt_and_work_files_independent(
299279
):
300280
source = tmp_path / "source.bmp"
301281
Image.new("RGB", (32, 16), "red").save(source)
302-
first = await media.prepare_model_image(
303-
str(source), max_size=16, output_dir=tmp_path
304-
)
282+
first = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
305283
cache = next((tmp_path / "cache" / media.CONVERT_CACHE_DIR_NAME).glob("*.img"))
306284
if bad_cache is None:
307285
cache.unlink()
@@ -314,17 +292,13 @@ async def test_bad_cache_rebuilt_and_work_files_independent(
314292
)
315293
else:
316294
cache.write_bytes(bad_cache)
317-
second = await media.prepare_model_image(
318-
str(source), max_size=16, output_dir=tmp_path
319-
)
295+
second = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
320296
assert first != second and Path(first).read_bytes() == Path(second).read_bytes()
321297
Path(first).unlink()
322298
assert cache.is_file() and Path(second).is_file()
323299
encode = Mock(side_effect=AssertionError("valid cache must skip encoding"))
324300
monkeypatch.setattr(media, "_encode_image_frame_bytes", encode)
325-
third = await media.prepare_model_image(
326-
str(source), max_size=16, output_dir=tmp_path
327-
)
301+
third = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
328302
assert Path(third).read_bytes() == Path(second).read_bytes()
329303

330304

@@ -333,13 +307,10 @@ async def test_unwritable_cache_bypassed_but_work_file_failure_skips(tmp_path):
333307
source = tmp_path / "source.bmp"
334308
Image.new("RGB", (32, 16), "red").save(source)
335309
(tmp_path / "cache").write_text("blocks directory creation")
336-
path = await media.prepare_model_image(
337-
str(source), max_size=16, output_dir=tmp_path
338-
)
310+
path = await _prepare_image_path(str(source), max_size=16, output_dir=tmp_path)
339311
assert Path(path).is_file()
340312
assert (
341-
await media.prepare_model_image(str(source), max_size=16, output_dir=source)
342-
is None
313+
await _prepare_image_path(str(source), max_size=16, output_dir=source) is None
343314
)
344315

345316

@@ -357,10 +328,7 @@ async def fail(*args, **kwargs):
357328
raise error
358329

359330
monkeypatch.setattr(media.MediaResolver, "_resolve_path", fail)
360-
assert (
361-
await media.prepare_model_image("ref", max_size=1280, output_dir=tmp_path)
362-
is None
363-
)
331+
assert await _prepare_image_path("ref", max_size=1280, output_dir=tmp_path) is None
364332

365333

366334
@pytest.mark.asyncio
@@ -380,7 +348,7 @@ async def fail(*args, **kwargs):
380348

381349
monkeypatch.setattr(media.MediaResolver, "_resolve_path", fail)
382350
with pytest.raises(type(error)):
383-
await media.prepare_model_image("ref", max_size=1280, output_dir=tmp_path)
351+
await _prepare_image_path("ref", max_size=1280, output_dir=tmp_path)
384352

385353

386354
def test_cache_publication_is_atomic_and_closes_file(tmp_path, monkeypatch):
@@ -408,10 +376,67 @@ async def download(url, target):
408376
_raise_for_download_status(SimpleNamespace(status=status), url)
409377

410378
monkeypatch.setattr(media, "download_file", download)
411-
result = await media.prepare_model_image(
379+
result = await _prepare_image_path(
412380
"https://example.com/image?token=secret-token",
413381
max_size=1280,
414382
output_dir=tmp_path,
415383
)
416384
assert result is None
417385
assert "secret-token" not in caplog.text
386+
387+
388+
@pytest.mark.asyncio
389+
@pytest.mark.parametrize("count", [2, 12])
390+
async def test_animation_reports_montage_including_cache_hits(tmp_path, count):
391+
"""Animated inputs report a montage even when its bytes are cached."""
392+
frames = [Image.new("RGB", (12, 8), (index * 20, 0, 0)) for index in range(count)]
393+
source = tmp_path / "animation.gif"
394+
frames[0].save(
395+
source, "GIF", save_all=True, append_images=frames[1:], duration=100, loop=0
396+
)
397+
398+
prepared = await media.prepare_model_image(
399+
str(source), max_size=1280, output_dir=tmp_path
400+
)
401+
assert prepared and prepared[1] is True
402+
assert Path(prepared[0]).is_file()
403+
404+
cached = await media.prepare_model_image(
405+
str(source), max_size=1280, output_dir=tmp_path
406+
)
407+
assert cached and cached[1] is True
408+
assert Path(cached[0]).read_bytes() == Path(prepared[0]).read_bytes()
409+
410+
411+
@pytest.mark.asyncio
412+
async def test_apng_with_cover_reports_montage(tmp_path):
413+
"""An APNG with a cover still reports an animation montage."""
414+
frames = [Image.new("RGB", (12, 8), (index * 20, 0, 0)) for index in range(3)]
415+
# An APNG cover frame precedes the animation frames it is not part of.
416+
frames.insert(0, Image.new("RGB", (12, 8), "blue"))
417+
source = tmp_path / "animation.png"
418+
frames[0].save(
419+
source,
420+
"PNG",
421+
save_all=True,
422+
append_images=frames[1:],
423+
default_image=True,
424+
duration=100,
425+
loop=0,
426+
)
427+
prepared = await media.prepare_model_image(
428+
str(source), max_size=1280, output_dir=tmp_path
429+
)
430+
assert prepared and prepared[1] is True
431+
432+
433+
@pytest.mark.asyncio
434+
async def test_still_image_reports_no_montage(tmp_path):
435+
"""Still images do not report a montage, so no notice is built."""
436+
source = tmp_path / "still.png"
437+
Image.new("RGB", (32, 16), "red").save(source)
438+
prepared = await media.prepare_model_image(
439+
str(source), max_size=1280, output_dir=tmp_path
440+
)
441+
assert prepared is not None
442+
assert prepared[1] is False

0 commit comments

Comments
 (0)