@@ -460,6 +460,98 @@ async def test_ignore_robots_txt_skips_the_check(web: FakeWeb) -> None:
460460 assert web .urls () == [PAGE ]
461461
462462
463+ OTHER_ROBOTS = "https://other.example.org/robots.txt"
464+ OTHER_PAGE = "https://other.example.org/secret"
465+
466+
467+ async def test_redirect_to_host_whose_robots_disallows_is_refused (
468+ web : FakeWeb ,
469+ ) -> None :
470+ # robots.txt belongs to one origin, so a redirect to another host is
471+ # checked against that host's robots.txt before it is requested.
472+ other_robots = "User-agent: *\n Disallow: /"
473+ web .add (ROBOTS , plain ("" , status = 404 ))
474+ web .add (PAGE , httpx .Response (302 , headers = {"location" : OTHER_PAGE }))
475+ web .add (OTHER_ROBOTS , plain (other_robots ))
476+ web .add (OTHER_PAGE , plain ("other-secret" ))
477+ async with connect () as (session , _ ):
478+ result = await call (session , {"url" : PAGE })
479+ assert wire (result )["isError" ] is True
480+ text = text_of (result )
481+ assert text .startswith (
482+ f"{ TOOL_ERROR } The sites robots.txt ({ OTHER_ROBOTS } ), specifies that "
483+ "autonomous fetching of this page is not allowed, "
484+ )
485+ assert f"<url>{ OTHER_PAGE } </url>" in text
486+ assert "other-secret" not in text
487+ # The redirect target itself is never requested.
488+ assert web .urls () == [ROBOTS , PAGE , OTHER_ROBOTS ]
489+
490+
491+ async def test_redirect_to_a_disallowed_path_on_the_same_host_is_refused (
492+ web : FakeWeb ,
493+ ) -> None :
494+ private = "https://example.com/private/report"
495+ web .add (ROBOTS , plain ("User-agent: *\n Disallow: /private" ))
496+ web .add (PAGE , httpx .Response (302 , headers = {"location" : "/private/report" }))
497+ web .add (private , plain ("private-report" ))
498+ async with connect () as (session , _ ):
499+ result = await call (session , {"url" : PAGE })
500+ assert wire (result )["isError" ] is True
501+ assert f"<url>{ private } </url>" in text_of (result )
502+ assert web .urls () == [ROBOTS , PAGE , ROBOTS ]
503+
504+
505+ async def test_redirect_to_host_answering_robots_403_is_refused (
506+ web : FakeWeb ,
507+ ) -> None :
508+ web .add (ROBOTS , plain ("" , status = 404 ))
509+ web .add (PAGE , httpx .Response (302 , headers = {"location" : OTHER_PAGE }))
510+ web .add (OTHER_ROBOTS , plain ("" , status = 403 ))
511+ web .add (OTHER_PAGE , plain ("other-secret" ))
512+ async with connect () as (session , _ ):
513+ result = await call (session , {"url" : PAGE })
514+ assert wire (result )["isError" ] is True
515+ assert text_of (result ).startswith (
516+ f"{ TOOL_ERROR } When fetching robots.txt ({ OTHER_ROBOTS } ), received status 403"
517+ )
518+ assert web .urls () == [ROBOTS , PAGE , OTHER_ROBOTS ]
519+
520+
521+ async def test_redirect_to_an_allowed_target_is_followed (web : FakeWeb ) -> None :
522+ web .add (ROBOTS , plain ("" , status = 404 ))
523+ web .add (PAGE , httpx .Response (302 , headers = {"location" : OTHER_PAGE }))
524+ web .add (OTHER_ROBOTS , plain ("User-agent: *\n Disallow: /private" ))
525+ web .add (OTHER_PAGE , plain ("other-page" ))
526+ async with connect () as (session , _ ):
527+ result = await call (session , {"url" : PAGE })
528+ assert wire (result )["isError" ] is False
529+ assert text_of (result ) == f"{ RAW_PREFIX_PLAIN } Contents of { PAGE } :\n other-page"
530+ assert web .urls () == [ROBOTS , PAGE , OTHER_ROBOTS , OTHER_PAGE ]
531+ for request in web .requests :
532+ assert request .headers ["user-agent" ] == DEFAULT_USER_AGENT_AUTONOMOUS
533+
534+
535+ async def test_redirects_skip_robots_with_ignore_robots_txt_and_on_the_prompt (
536+ web : FakeWeb ,
537+ ) -> None :
538+ # --ignore-robots-txt and the user-initiated prompt never consult
539+ # robots.txt, for the requested URL or for any redirect hop.
540+ web .add (ROBOTS , plain ("User-agent: *\n Disallow: /" ))
541+ web .add (PAGE , httpx .Response (302 , headers = {"location" : OTHER_PAGE }))
542+ web .add (OTHER_ROBOTS , plain ("User-agent: *\n Disallow: /" ))
543+ web .add (OTHER_PAGE , plain ("other-page" ))
544+ async with connect (ignore_robots_txt = True ) as (session , _ ):
545+ result = await call (session , {"url" : PAGE })
546+ assert wire (result )["isError" ] is False
547+ assert web .urls () == [PAGE , OTHER_PAGE ]
548+ web .requests .clear ()
549+ async with connect () as (session , _ ):
550+ prompt = await session .get_prompt ("fetch" , {"url" : PAGE })
551+ assert "other-page" in wire (prompt )["messages" ][0 ]["content" ]["text" ]
552+ assert web .urls () == [PAGE , OTHER_PAGE ]
553+
554+
463555# --------------------------------------------------------------------------
464556# tools/call: fetching and content handling
465557# --------------------------------------------------------------------------
@@ -855,8 +947,9 @@ async def test_allow_private_ips_follows_redirect_to_private_address(
855947 assert wire (result )["isError" ] is False
856948 assert text_of (result ) == f"{ RAW_PREFIX_PLAIN } Contents of { PAGE } :\n instance-secret"
857949 assert prompt .description == PROMPT_DESCRIPTION
858- # robots.txt is only consulted for the original host, never the target.
859- assert web .urls () == [ROBOTS , PAGE , metadata , metadata ]
950+ # The redirect target's own robots.txt (a 404 here) is checked before it.
951+ metadata_robots = "http://169.254.169.254/robots.txt"
952+ assert web .urls () == [ROBOTS , PAGE , metadata_robots , metadata , metadata ]
860953 # With the guard off, nothing is resolved by the server.
861954 assert web .lookups == []
862955
0 commit comments