@@ -68,13 +68,24 @@ class BundleReference(TypedDict):
6868 disposition : ArtifactDisposition
6969
7070
71+ @dataclass (frozen = True )
72+ class SecurityTextReconstruction :
73+ """One projected run reconstructed from a bounded raw source span."""
74+
75+ derived_start : int
76+ derived_end : int
77+ source_start : int
78+ source_end : int
79+
80+
7181@dataclass (frozen = True )
7282class SecurityTextView :
7383 """A bounded derived text view and mapping to raw character offsets."""
7484
7585 name : str
7686 text : str
7787 source_offsets : array [int ] | None = None
88+ reconstructions : tuple [SecurityTextReconstruction , ...] = ()
7889
7990 def source_offset (self , derived_offset : int ) -> int :
8091 """Map a derived character offset to the corresponding source offset."""
@@ -85,6 +96,29 @@ def source_offset(self, derived_offset: int) -> int:
8596 index = min (max (derived_offset , 0 ), len (self .source_offsets ) - 1 )
8697 return self .source_offsets [index ]
8798
99+ def reconstructed_source_spans (
100+ self ,
101+ derived_start : int ,
102+ derived_end : int ,
103+ ) -> tuple [tuple [int , int ], ...]:
104+ """Return exact removed source gaps reconstructed by a derived range."""
105+ if derived_end <= derived_start or self .source_offsets is None :
106+ return ()
107+ gaps : list [tuple [int , int ]] = []
108+ for item in self .reconstructions :
109+ overlap_start = max (derived_start , item .derived_start )
110+ overlap_end = min (derived_end , item .derived_end )
111+ for offset in range (max (overlap_start + 1 , item .derived_start + 1 ), overlap_end ):
112+ previous_source = self .source_offsets [offset - 1 ]
113+ source = self .source_offsets [offset ]
114+ if source > previous_source + 1 :
115+ gaps .append ((previous_source + 1 , source ))
116+ return tuple (gaps )
117+
118+ def range_has_reconstruction (self , derived_start : int , derived_end : int ) -> bool :
119+ """Return whether a derived range joins characters across a removed gap."""
120+ return bool (self .reconstructed_source_spans (derived_start , derived_end ))
121+
88122
89123@dataclass (frozen = True )
90124class _ObfuscatedInstructionMatch :
@@ -201,6 +235,10 @@ class _ObfuscatedIgnoreState:
201235 r"(?:[^\W\d_](?:[^\w]|_)+){5}[^\W\d_]" ,
202236 re .UNICODE ,
203237)
238+ _PROMPT_SPACING_CANDIDATE = re .compile (
239+ r"(?<![^\W\d_])\S(?:[^\w\r\n]|_)+\S(?![^\W\d_])" ,
240+ re .UNICODE ,
241+ )
204242_MIN_LETTER_SPACING_RUN_LETTERS = 6
205243# Unicode 15.1.0 DerivedCoreProperties.txt: Default_Ignorable_Code_Point.
206244_DEFAULT_IGNORABLE_RANGES = (
@@ -1700,6 +1738,183 @@ def compact_letter_view(text: str) -> SecurityTextView:
17001738 return SecurityTextView ("compact" , output .getvalue (), offsets )
17011739
17021740
1741+ def prompt_injection_letter_spacing_view (
1742+ text : str ,
1743+ check_runtime : Callable [[], None ] | None = None ,
1744+ * ,
1745+ preserve_identifier_boundaries : bool = True ,
1746+ ) -> SecurityTextView :
1747+ """Collapse consistently separated tokens without inventing word boundaries.
1748+
1749+ The prompt-injection analyzer alone consumes this projection. Existing
1750+ word boundaries are retained verbatim, while consistent spaces, tabs, or
1751+ punctuation between single-character tokens are removed with exact raw-gap
1752+ provenance. A completely boundary-free run therefore remains one condensed
1753+ token and cannot acquire a guessed P3 or P4 segmentation. Identifier-adjacent
1754+ runs are preserved for semantic classification; artifact-integrity may opt
1755+ into their projection solely to produce an ambiguity finding.
1756+ """
1757+ if check_runtime is not None :
1758+ check_runtime ()
1759+ if _PROMPT_SPACING_CANDIDATE .search (text ) is None :
1760+ return SecurityTextView ("prompt-letter-spacing" , text )
1761+ processed_since_check = 0
1762+
1763+ def record_work (characters : int = 1 ) -> None :
1764+ nonlocal processed_since_check
1765+ if check_runtime is None :
1766+ return
1767+ processed_since_check += characters
1768+ if processed_since_check >= 4096 :
1769+ check_runtime ()
1770+ processed_since_check %= 4096
1771+
1772+ output = StringIO ()
1773+ offsets = array ("I" )
1774+ reconstructions : list [SecurityTextReconstruction ] = []
1775+ transformed = False
1776+
1777+ def append_source (start : int , end : int ) -> None :
1778+ for source_offset in range (start , end ):
1779+ record_work ()
1780+ output .write (text [source_offset ])
1781+ offsets .append (source_offset )
1782+
1783+ cursor = 0
1784+ index = 0
1785+ while index < len (text ):
1786+ record_work ()
1787+ if _is_letter_spacing_separator (text [index ]):
1788+ index += 1
1789+ continue
1790+
1791+ run_start = index
1792+ unit_offsets = array ("I" , (run_start ,))
1793+ run_tail = text [run_start ].casefold ()[- 8 :]
1794+ url_mode = False
1795+ spacing_width : int | None = None
1796+ run_end = run_start + 1
1797+ while run_end < len (text ):
1798+ gap_start = run_end
1799+ if text [gap_start ].isspace () and text [gap_start ] not in _LOGICAL_LINE_BREAK_CHARACTERS :
1800+ while (
1801+ run_end < len (text )
1802+ and text [run_end ].isspace ()
1803+ and text [run_end ] not in _LOGICAL_LINE_BREAK_CHARACTERS
1804+ ):
1805+ record_work ()
1806+ run_end += 1
1807+ if run_end < len (text ) and _is_letter_spacing_separator (text [run_end ]):
1808+ punctuation = text [run_end ]
1809+ punctuation_is_unit = (
1810+ punctuation == ":"
1811+ and run_tail in {"http" , "https" }
1812+ or punctuation == "/"
1813+ and run_tail .endswith (("http:" , "https:" , "http:/" , "https:/" ))
1814+ or punctuation == "."
1815+ and url_mode
1816+ or punctuation == "'"
1817+ and run_tail .endswith ("user" )
1818+ )
1819+ if not punctuation_is_unit :
1820+ run_end += 1
1821+ record_work ()
1822+ while (
1823+ run_end < len (text )
1824+ and text [run_end ].isspace ()
1825+ and text [run_end ] not in _LOGICAL_LINE_BREAK_CHARACTERS
1826+ ):
1827+ record_work ()
1828+ run_end += 1
1829+ elif _is_letter_spacing_separator (text [gap_start ]):
1830+ marker = text [gap_start ]
1831+ while run_end < len (text ) and (
1832+ text [run_end ] == marker
1833+ or text [run_end ].isspace ()
1834+ and text [run_end ] not in _LOGICAL_LINE_BREAK_CHARACTERS
1835+ ):
1836+ record_work ()
1837+ run_end += 1
1838+ if gap_start == run_end :
1839+ break
1840+ if run_end >= len (text ):
1841+ run_end = gap_start
1842+ break
1843+
1844+ gap = text [gap_start :run_end ]
1845+ gap_signature = _letter_spacing_gap_signature (gap )
1846+ whitespace_gap = all (character .isspace () for character in gap )
1847+ if gap_signature is None and not whitespace_gap :
1848+ run_end = gap_start
1849+ break
1850+
1851+ # A separator before a multi-letter word is an observed token
1852+ # boundary, not another inter-character gap. In particular, this
1853+ # keeps the first character of "conversation" out of "s e n d".
1854+ if text [run_end ].isalpha () and run_end + 1 < len (text ) and text [run_end + 1 ].isalpha ():
1855+ run_end = gap_start
1856+ break
1857+
1858+ if whitespace_gap :
1859+ if spacing_width is not None and len (gap ) != spacing_width :
1860+ run_end = gap_start
1861+ break
1862+ spacing_width = len (gap )
1863+
1864+ unit_offsets .append (run_end )
1865+ run_tail = (run_tail + text [run_end ].casefold ())[- 8 :]
1866+ url_mode = url_mode or "://" in run_tail
1867+ run_end += 1
1868+
1869+ if len (unit_offsets ) == 1 :
1870+ index += 1
1871+ continue
1872+
1873+ # Do not rewrite a letter-spaced fragment embedded in an identifier.
1874+ # Advance past rejected runs too, so attacker-controlled near misses
1875+ # cannot force quadratic rescanning from every interior character.
1876+ left_identifier = run_start > 0 and _is_word_character (text [run_start - 1 ])
1877+ right_identifier = run_end < len (text ) and _is_word_character (text [run_end ])
1878+ left_letter = run_start > 0 and text [run_start - 1 ].isalpha ()
1879+ right_letter = run_end < len (text ) and text [run_end ].isalpha ()
1880+ boundary_is_safe = (
1881+ not left_identifier and not right_identifier
1882+ if preserve_identifier_boundaries
1883+ else not left_letter and not right_letter
1884+ )
1885+ if boundary_is_safe :
1886+ append_source (cursor , run_start )
1887+ derived_start = len (offsets )
1888+ for source_offset in unit_offsets :
1889+ record_work ()
1890+ output .write (text [source_offset ])
1891+ offsets .append (source_offset )
1892+ reconstructions .append (
1893+ SecurityTextReconstruction (
1894+ derived_start = derived_start ,
1895+ derived_end = len (offsets ),
1896+ source_start = run_start ,
1897+ source_end = run_end ,
1898+ )
1899+ )
1900+ cursor = run_end
1901+ transformed = True
1902+ # Retrying at the second unit after a left-side rejection prevents one
1903+ # malformed prefix from hiding a valid suffix. Only the first attempt
1904+ # can have an adjacent left identifier, so the scan remains linear.
1905+ index = unit_offsets [1 ] if not boundary_is_safe and left_identifier else run_end
1906+
1907+ if not transformed :
1908+ return SecurityTextView ("prompt-letter-spacing" , text )
1909+ append_source (cursor , len (text ))
1910+ return SecurityTextView (
1911+ "prompt-letter-spacing" ,
1912+ output .getvalue (),
1913+ offsets ,
1914+ tuple (reconstructions ),
1915+ )
1916+
1917+
17031918def _requires_normalized_security_view (text : str ) -> bool :
17041919 """Return whether normalization can produce a distinct security view."""
17051920 if _IGNORED_ASCII_CONTROL .search (text ) is not None :
0 commit comments