2525import asyncio
2626import ipaddress
2727import os
28+ import re
2829import socket
2930import zlib
3031from collections .abc import AsyncIterator , Awaitable , Callable , Mapping , Sequence
3132from contextlib import asynccontextmanager
3233from importlib .metadata import version
3334from typing import Annotated , Any
34- from urllib .parse import urlparse , urlunparse
35+ from urllib .parse import urlparse , urlsplit , urlunparse , urlunsplit
3536
3637import httpx
3738import markdownify
@@ -104,8 +105,13 @@ async def _resolve_host(host: str) -> list[str]:
104105 "2001::/23" ,
105106 "2001:db8::/32" ,
106107 "2002::/16" ,
108+ # IPv6 documentation (RFC 9637), absent from older is_global tables.
109+ "3fff::/20" ,
107110 "fc00::/7" ,
108111 "fe80::/10" ,
112+ # Deprecated site-local (RFC 3879), which is_global reads as global.
113+ # Outside 2000::/3 as well, but listed so it is refused by name.
114+ "fec0::/10" ,
109115 )
110116)
111117
@@ -131,6 +137,17 @@ async def _resolve_host(host: str) -> list[str]:
131137# in it to the IPv4 held in its low 32 bits, so that IPv4 is the destination.
132138_NAT64_WELL_KNOWN = ipaddress .IPv6Network ("64:ff9b::/96" )
133139
140+ # IPv4-translated addresses (RFC 2765, RFC 6145), which a stateless translator
141+ # forwards to the IPv4 held in their low 32 bits, like NAT64.
142+ _IPV4_TRANSLATED = ipaddress .IPv6Network ("::ffff:0:0:0/96" )
143+
144+ # IPv6 global unicast (RFC 4291 2.4, IANA's IPv6 address space registry). The
145+ # rest of the IPv6 space is reserved by the IETF, unique or site-local,
146+ # link-local or multicast, none of it a public destination, while is_global
147+ # reads most of it as global. Deny-by-default there means a range missing from
148+ # the list above cannot pass as public.
149+ _IPV6_GLOBAL_UNICAST = ipaddress .IPv6Network ("2000::/3" )
150+
134151
135152def _is_public_address (address : str ) -> bool :
136153 """True when ``address`` is a globally routable unicast address.
@@ -139,18 +156,24 @@ def _is_public_address(address: str) -> bool:
139156 cloud metadata endpoints), shared address space (100.64.0.0/10, which holds
140157 Alibaba Cloud's 100.100.100.200), unique local IPv6 (AWS's fd00:ec2::254),
141158 multicast, unspecified, documentation and other special-purpose ranges.
142- An IPv4-mapped or well-known NAT64 address is judged by the IPv4 it wraps;
143- a deprecated IPv4-compatible address (``::/96``) is refused whatever it holds.
159+ An IPv4-mapped, IPv4-translated or well-known NAT64 address is judged by
160+ the IPv4 it wraps; a deprecated IPv4-compatible address (``::/96``) is
161+ refused whatever it holds. Any other IPv6 address outside global unicast
162+ (``2000::/3``) is refused.
144163 """
145164 ip = ipaddress .ip_address (address )
146165 if isinstance (ip , ipaddress .IPv6Address ) and ip .ipv4_mapped is not None :
147166 ip = ip .ipv4_mapped
148- elif isinstance (ip , ipaddress .IPv6Address ) and ip in _NAT64_WELL_KNOWN :
167+ elif isinstance (ip , ipaddress .IPv6Address ) and (
168+ ip in _NAT64_WELL_KNOWN or ip in _IPV4_TRANSLATED
169+ ):
149170 ip = ipaddress .IPv4Address (int (ip ) & 0xFFFFFFFF )
150171 if any (ip in network for network in _PUBLIC_EXCEPTIONS ):
151172 return True
152173 if any (ip in network for network in _NON_PUBLIC_NETWORKS ):
153174 return False
175+ if isinstance (ip , ipaddress .IPv6Address ) and ip not in _IPV6_GLOBAL_UNICAST :
176+ return False
154177 return ip .is_global and not ip .is_multicast
155178
156179
@@ -660,6 +683,48 @@ def handle_starttag(
660683 return decode_text (response , content )
661684
662685
686+ # Escapes of the path separator and of the dot, which an origin may decode
687+ # before it resolves the path. Other escapes (such as %20) are left alone.
688+ _ENCODED_SLASH = re .compile ("%2f" , re .IGNORECASE )
689+ _ENCODED_DOT = re .compile ("%2e" , re .IGNORECASE )
690+ _REPEATED_SLASHES = re .compile ("/{2,}" )
691+
692+
693+ def _remove_dot_segments (path : str ) -> str :
694+ """``path`` with its ``.`` and ``..`` segments resolved (RFC 3986 5.2.4).
695+
696+ The result is always absolute, as robots.txt rules are: an empty path
697+ becomes ``/``.
698+ """
699+ segments = path .split ("/" )
700+ resolved : list [str ] = []
701+ for segment in segments [1 :]:
702+ if segment == ".." :
703+ if resolved :
704+ resolved .pop ()
705+ elif segment != "." :
706+ resolved .append (segment )
707+ # A path ending in a dot segment names a directory: "/a/b/.." is "/a/".
708+ trailing = "/" if resolved and segments [- 1 ] in ("." , ".." ) else ""
709+ return "/" + "/" .join (resolved ) + trailing
710+
711+
712+ def _resolved_path_url (url : str ) -> str :
713+ """``url`` with its path spelled the way a typical origin resolves it.
714+
715+ robots.txt rules match the path a crawler requests, but an origin such as
716+ nginx or Apache serves ``//private``, ``/x/../private`` and ``/%2Fprivate``
717+ as ``/private``. So the path's separator and dot escapes are decoded,
718+ repeated slashes merged and dot segments removed before rules are matched
719+ against it, as well as against the path as spelled. The query is kept as
720+ it is, and matching stays case-sensitive (RFC 9309 2.2.2).
721+ """
722+ parts = urlsplit (url )
723+ path = _ENCODED_DOT .sub ("." , _ENCODED_SLASH .sub ("/" , parts .path ))
724+ path = _remove_dot_segments (_REPEATED_SLASHES .sub ("/" , path ))
725+ return urlunsplit (parts ._replace (path = path ))
726+
727+
663728async def check_may_autonomously_fetch_url (
664729 url : str ,
665730 user_agent : str ,
@@ -706,7 +771,12 @@ async def check_may_autonomously_fetch_url(
706771 line for line in robot_txt .splitlines () if not line .strip ().startswith ("#" )
707772 )
708773 robot_parser = Protego .parse (processed_robot_txt )
709- if not robot_parser .can_fetch (str (url ), user_agent ):
774+ # Refused when either spelling of the path is disallowed: resolving the
775+ # path only ever adds refusals.
776+ if not (
777+ robot_parser .can_fetch (url , user_agent )
778+ and robot_parser .can_fetch (_resolved_path_url (url ), user_agent )
779+ ):
710780 raise ToolError (
711781 f"The sites robots.txt ({ robot_txt_url } ), specifies that autonomous fetching of this page is not allowed, "
712782 f"<useragent>{ user_agent } </useragent>\n "
0 commit comments