@@ -68,13 +68,24 @@ class BundleReference(TypedDict):
6868 disposition : ArtifactDisposition
6969
7070
71+ @dataclass (frozen = True )
72+ class SecurityTextReconstruction :
73+ """One projected run reconstructed from a bounded raw source span."""
74+
75+ derived_start : int
76+ derived_end : int
77+ source_start : int
78+ source_end : int
79+
80+
7181@dataclass (frozen = True )
7282class SecurityTextView :
7383 """A bounded derived text view and mapping to raw character offsets."""
7484
7585 name : str
7686 text : str
7787 source_offsets : array [int ] | None = None
88+ reconstructions : tuple [SecurityTextReconstruction , ...] = ()
7889
7990 def source_offset (self , derived_offset : int ) -> int :
8091 """Map a derived character offset to the corresponding source offset."""
@@ -85,6 +96,29 @@ def source_offset(self, derived_offset: int) -> int:
8596 index = min (max (derived_offset , 0 ), len (self .source_offsets ) - 1 )
8697 return self .source_offsets [index ]
8798
99+ def reconstructed_source_spans (
100+ self ,
101+ derived_start : int ,
102+ derived_end : int ,
103+ ) -> tuple [tuple [int , int ], ...]:
104+ """Return exact removed source gaps reconstructed by a derived range."""
105+ if derived_end <= derived_start or self .source_offsets is None :
106+ return ()
107+ gaps : list [tuple [int , int ]] = []
108+ for item in self .reconstructions :
109+ overlap_start = max (derived_start , item .derived_start )
110+ overlap_end = min (derived_end , item .derived_end )
111+ for offset in range (max (overlap_start + 1 , item .derived_start + 1 ), overlap_end ):
112+ previous_source = self .source_offsets [offset - 1 ]
113+ source = self .source_offsets [offset ]
114+ if source > previous_source + 1 :
115+ gaps .append ((previous_source + 1 , source ))
116+ return tuple (gaps )
117+
118+ def range_has_reconstruction (self , derived_start : int , derived_end : int ) -> bool :
119+ """Return whether a derived range joins characters across a removed gap."""
120+ return bool (self .reconstructed_source_spans (derived_start , derived_end ))
121+
88122
89123@dataclass (frozen = True )
90124class _ObfuscatedInstructionMatch :
@@ -201,7 +235,10 @@ class _ObfuscatedIgnoreState:
201235 r"(?:[^\W\d_](?:[^\w]|_)+){5}[^\W\d_]" ,
202236 re .UNICODE ,
203237)
204- _PROMPT_SPACED_PAIR = re .compile (r"(?<![A-Za-z])\S \S(?![A-Za-z])" )
238+ _PROMPT_SPACING_CANDIDATE = re .compile (
239+ r"(?<![^\W\d_])\S(?:[^\w\r\n]|_)+\S(?![^\W\d_])" ,
240+ re .UNICODE ,
241+ )
205242_MIN_LETTER_SPACING_RUN_LETTERS = 6
206243# Unicode 15.1.0 DerivedCoreProperties.txt: Default_Ignorable_Code_Point.
207244_DEFAULT_IGNORABLE_RANGES = (
@@ -1707,19 +1744,19 @@ def prompt_injection_letter_spacing_view(
17071744 * ,
17081745 preserve_identifier_boundaries : bool = True ,
17091746) -> SecurityTextView :
1710- """Collapse ASCII-spaced tokens without inventing word boundaries.
1747+ """Collapse consistently separated tokens without inventing word boundaries.
17111748
17121749 The prompt-injection analyzer alone consumes this projection. Existing
1713- multi-space word boundaries are retained verbatim, while one-space token
1714- interiors are removed with exact raw offsets. A completely boundary-free
1715- run therefore remains one condensed token and cannot acquire a guessed P3
1716- or P4 segmentation. Identifier-adjacent runs are preserved for semantic
1717- classification; artifact-integrity may opt into their projection solely to
1718- produce an ambiguity finding.
1750+ word boundaries are retained verbatim, while consistent spaces, tabs, or
1751+ punctuation between single-character tokens are removed with exact raw-gap
1752+ provenance. A completely boundary-free run therefore remains one condensed
1753+ token and cannot acquire a guessed P3 or P4 segmentation. Identifier-adjacent
1754+ runs are preserved for semantic classification; artifact-integrity may opt
1755+ into their projection solely to produce an ambiguity finding.
17191756 """
17201757 if check_runtime is not None :
17211758 check_runtime ()
1722- if _PROMPT_SPACED_PAIR .search (text ) is None :
1759+ if _PROMPT_SPACING_CANDIDATE .search (text ) is None :
17231760 return SecurityTextView ("prompt-letter-spacing" , text )
17241761 processed_since_check = 0
17251762
@@ -1734,6 +1771,7 @@ def record_work(characters: int = 1) -> None:
17341771
17351772 output = StringIO ()
17361773 offsets = array ("I" )
1774+ reconstructions : list [SecurityTextReconstruction ] = []
17371775 transformed = False
17381776
17391777 def append_source (start : int , end : int ) -> None :
@@ -1746,54 +1784,135 @@ def append_source(start: int, end: int) -> None:
17461784 index = 0
17471785 while index < len (text ):
17481786 record_work ()
1749- if (
1750- text [index ].isspace ()
1751- or index + 2 >= len (text )
1752- or text [index + 1 ] != " "
1753- or text [index + 2 ].isspace ()
1754- ):
1787+ if _is_letter_spacing_separator (text [index ]):
17551788 index += 1
17561789 continue
17571790
17581791 run_start = index
1759- run_end = index + 1
1760- while run_end + 1 < len (text ) and text [run_end ] == " " and not text [run_end + 1 ].isspace ():
1761- run_end += 2
1762- record_work (2 )
1792+ unit_offsets = array ("I" , (run_start ,))
1793+ run_tail = text [run_start ].casefold ()[- 8 :]
1794+ url_mode = False
1795+ spacing_width : int | None = None
1796+ run_end = run_start + 1
1797+ while run_end < len (text ):
1798+ gap_start = run_end
1799+ if text [gap_start ].isspace () and text [gap_start ] not in _LOGICAL_LINE_BREAK_CHARACTERS :
1800+ while (
1801+ run_end < len (text )
1802+ and text [run_end ].isspace ()
1803+ and text [run_end ] not in _LOGICAL_LINE_BREAK_CHARACTERS
1804+ ):
1805+ record_work ()
1806+ run_end += 1
1807+ if run_end < len (text ) and _is_letter_spacing_separator (text [run_end ]):
1808+ punctuation = text [run_end ]
1809+ punctuation_is_unit = (
1810+ punctuation == ":"
1811+ and run_tail in {"http" , "https" }
1812+ or punctuation == "/"
1813+ and run_tail .endswith (("http:" , "https:" , "http:/" , "https:/" ))
1814+ or punctuation == "."
1815+ and url_mode
1816+ or punctuation == "'"
1817+ and run_tail .endswith ("user" )
1818+ )
1819+ if not punctuation_is_unit :
1820+ run_end += 1
1821+ record_work ()
1822+ while (
1823+ run_end < len (text )
1824+ and text [run_end ].isspace ()
1825+ and text [run_end ] not in _LOGICAL_LINE_BREAK_CHARACTERS
1826+ ):
1827+ record_work ()
1828+ run_end += 1
1829+ elif _is_letter_spacing_separator (text [gap_start ]):
1830+ marker = text [gap_start ]
1831+ while run_end < len (text ) and (
1832+ text [run_end ] == marker
1833+ or text [run_end ].isspace ()
1834+ and text [run_end ] not in _LOGICAL_LINE_BREAK_CHARACTERS
1835+ ):
1836+ record_work ()
1837+ run_end += 1
1838+ if gap_start == run_end :
1839+ break
1840+ if run_end >= len (text ):
1841+ run_end = gap_start
1842+ break
1843+
1844+ gap = text [gap_start :run_end ]
1845+ gap_signature = _letter_spacing_gap_signature (gap )
1846+ whitespace_gap = all (character .isspace () for character in gap )
1847+ if gap_signature is None and not whitespace_gap :
1848+ run_end = gap_start
1849+ break
1850+
1851+ # A separator before a multi-letter word is an observed token
1852+ # boundary, not another inter-character gap. In particular, this
1853+ # keeps the first character of "conversation" out of "s e n d".
1854+ if text [run_end ].isalpha () and run_end + 1 < len (text ) and text [run_end + 1 ].isalpha ():
1855+ run_end = gap_start
1856+ break
1857+
1858+ if whitespace_gap :
1859+ if spacing_width is not None and len (gap ) != spacing_width :
1860+ run_end = gap_start
1861+ break
1862+ spacing_width = len (gap )
1863+
1864+ unit_offsets .append (run_end )
1865+ run_tail = (run_tail + text [run_end ].casefold ())[- 8 :]
1866+ url_mode = url_mode or "://" in run_tail
1867+ run_end += 1
1868+
1869+ if len (unit_offsets ) == 1 :
1870+ index += 1
1871+ continue
17631872
17641873 # Do not rewrite a letter-spaced fragment embedded in an identifier.
17651874 # Advance past rejected runs too, so attacker-controlled near misses
17661875 # cannot force quadratic rescanning from every interior character.
1767- left_identifier = run_start > 0 and (
1768- text [run_start - 1 ].isascii ()
1769- and (text [run_start - 1 ].isalnum () or text [run_start - 1 ] == "_" )
1770- )
1771- right_identifier = run_end < len (text ) and (
1772- text [run_end ].isascii () and (text [run_end ].isalnum () or text [run_end ] == "_" )
1773- )
1774- left_letter = (
1775- run_start > 0 and text [run_start - 1 ].isascii () and text [run_start - 1 ].isalpha ()
1776- )
1777- right_letter = run_end < len (text ) and text [run_end ].isascii () and text [run_end ].isalpha ()
1876+ left_identifier = run_start > 0 and _is_word_character (text [run_start - 1 ])
1877+ right_identifier = run_end < len (text ) and _is_word_character (text [run_end ])
1878+ left_letter = run_start > 0 and text [run_start - 1 ].isalpha ()
1879+ right_letter = run_end < len (text ) and text [run_end ].isalpha ()
17781880 boundary_is_safe = (
17791881 not left_identifier and not right_identifier
17801882 if preserve_identifier_boundaries
17811883 else not left_letter and not right_letter
17821884 )
17831885 if boundary_is_safe :
17841886 append_source (cursor , run_start )
1785- for source_offset in range (run_start , run_end , 2 ):
1786- record_work (2 )
1887+ derived_start = len (offsets )
1888+ for source_offset in unit_offsets :
1889+ record_work ()
17871890 output .write (text [source_offset ])
17881891 offsets .append (source_offset )
1892+ reconstructions .append (
1893+ SecurityTextReconstruction (
1894+ derived_start = derived_start ,
1895+ derived_end = len (offsets ),
1896+ source_start = run_start ,
1897+ source_end = run_end ,
1898+ )
1899+ )
17891900 cursor = run_end
17901901 transformed = True
1791- index = run_end
1902+ # Retrying at the second unit after a left-side rejection prevents one
1903+ # malformed prefix from hiding a valid suffix. Only the first attempt
1904+ # can have an adjacent left identifier, so the scan remains linear.
1905+ index = unit_offsets [1 ] if not boundary_is_safe and left_identifier else run_end
17921906
17931907 if not transformed :
17941908 return SecurityTextView ("prompt-letter-spacing" , text )
17951909 append_source (cursor , len (text ))
1796- return SecurityTextView ("prompt-letter-spacing" , output .getvalue (), offsets )
1910+ return SecurityTextView (
1911+ "prompt-letter-spacing" ,
1912+ output .getvalue (),
1913+ offsets ,
1914+ tuple (reconstructions ),
1915+ )
17971916
17981917
17991918def _requires_normalized_security_view (text : str ) -> bool :
0 commit comments