@@ -1008,23 +1008,22 @@ class _TP4Candidate:
10081008_TP4_CONTEXT_CLOSE = "End of document context.\n "
10091009
10101010
1011- def _tp4_preceding_block (preceding : Sequence [str ]) -> list [str ]:
1011+ def _tp4_preceding_block (lines : Sequence [str ], start : int , stop : int ) -> list [str ]:
10121012 """Collect the trailing prose block, walking back to a nearby heading.
10131013
10141014 Returns at most ``TP4_PRE_CONTEXT_LINES`` stripped lines in document
1015- order. A blank line ends the introducing block, unless the next
1016- non-blank line above it is a Markdown heading, in which case that one
1017- heading is included: headings routinely carry the safety framing (for
1018- example marking the fence as an example that must not be run). The walk
1019- never passes a non-blank, non-heading line, so prose from an earlier
1020- section cannot leak in.
1015+ order from ``lines[start:stop]`` . A blank line ends the introducing
1016+ block, unless the next non-blank line above it is a Markdown heading, in
1017+ which case that one heading is included: headings routinely carry the
1018+ safety framing (for example marking the fence as an example that must not
1019+ be run). The walk never passes a non-blank, non-heading line, so prose
1020+ from an earlier section cannot leak in.
10211021 """
10221022 collected : list [str ] = []
1023- lines = list (preceding )
1024- index = len (lines ) - 1
1025- while index >= 0 and not lines [index ].strip ():
1023+ index = stop - 1
1024+ while index >= start and not lines [index ].strip ():
10261025 index -= 1
1027- while index >= 0 and len (collected ) < TP4_PRE_CONTEXT_LINES :
1026+ while index >= start and len (collected ) < TP4_PRE_CONTEXT_LINES :
10281027 stripped = lines [index ].strip ()
10291028 if not stripped :
10301029 break
@@ -1037,9 +1036,9 @@ def _tp4_preceding_block(preceding: Sequence[str]) -> list[str]:
10371036 # collected block.
10381037 cursor = index
10391038 if collected and not _TP4_HEADING_RE .match (collected [- 1 ]):
1040- while cursor >= 0 and not lines [cursor ].strip ():
1039+ while cursor >= start and not lines [cursor ].strip ():
10411040 cursor -= 1
1042- if cursor >= 0 and _TP4_HEADING_RE .match (lines [cursor ].strip ()):
1041+ if cursor >= start and _TP4_HEADING_RE .match (lines [cursor ].strip ()):
10431042 if full :
10441043 collected .pop ()
10451044 collected .append (lines [cursor ].strip ())
@@ -1073,7 +1072,7 @@ def _truncate_preceding(text: str, max_chars: int = TP4_PRE_CONTEXT_CHARS) -> st
10731072 return text [- max_chars :]
10741073
10751074
1076- def _tp4_trailing_block (following : Sequence [str ]) -> list [str ]:
1075+ def _tp4_trailing_block (lines : Sequence [str ], start : int ) -> list [str ]:
10771076 """Collect the prose that follows a fence, stopping at structure.
10781077
10791078 Returns at most ``TP4_POST_CONTEXT_LINES`` stripped lines in document
@@ -1084,8 +1083,8 @@ def _tp4_trailing_block(following: Sequence[str]) -> list[str]:
10841083 candidate), or at the line cap.
10851084 """
10861085 collected : list [str ] = []
1087- for raw in following :
1088- stripped = raw .strip ()
1086+ for index in range ( start , len ( lines )) :
1087+ stripped = lines [ index ] .strip ()
10891088 if not stripped :
10901089 continue
10911090 if _TP4_HEADING_RE .match (stripped ):
@@ -1099,7 +1098,9 @@ def _tp4_trailing_block(following: Sequence[str]) -> list[str]:
10991098 return collected
11001099
11011100
1102- def _tp4_fence_context (preceding : Sequence [str ], following : Sequence [str ]) -> tuple [str , str ]:
1101+ def _tp4_fence_context (
1102+ lines : Sequence [str ], resume : int , opened : int , closed : int
1103+ ) -> tuple [str , str ]:
11031104 """Return the bounded prose around a fence as ``(before, after)``.
11041105
11051106 A fenced block is often introduced (or followed) by text that changes how
@@ -1111,8 +1112,10 @@ def _tp4_fence_context(preceding: Sequence[str], following: Sequence[str]) -> tu
11111112 two sides stay separate so budget pressure can shrink each one with
11121113 nearest-fence priority instead of cutting the trailing side off.
11131114 """
1114- before = _truncate_preceding ("\n " .join (_tp4_preceding_block (preceding )), TP4_PRE_CONTEXT_CHARS )
1115- after_lines = _tp4_trailing_block (following )
1115+ before = _truncate_preceding (
1116+ "\n " .join (_tp4_preceding_block (lines , resume , opened )), TP4_PRE_CONTEXT_CHARS
1117+ )
1118+ after_lines = _tp4_trailing_block (lines , closed + 1 )
11161119 after = "\n " .join (after_lines )
11171120 if len (after ) > TP4_POST_CONTEXT_CHARS :
11181121 after = after [:TP4_POST_CONTEXT_CHARS ]
@@ -1176,7 +1179,7 @@ def _iter_tp4_markdown_fences(
11761179 continue
11771180 body .append (line )
11781181 for language , body_text , start , end , resume , opened , closed in fences :
1179- before , after = _tp4_fence_context (lines [ resume : opened ], lines [ closed + 1 :] )
1182+ before , after = _tp4_fence_context (lines , resume , opened , closed )
11801183 yield language , body_text , start , end , before , after
11811184
11821185
@@ -1231,14 +1234,27 @@ def _bounded_utf8_prefix(text: str, max_bytes: int) -> tuple[str, int, bool]:
12311234 return candidate , len (encoded ), len (candidate ) < len (text )
12321235
12331236
1237+ def _tp4_line_tokens (line : str ) -> int :
1238+ """Return the token estimate for one line."""
1239+ return max (1 , (len (line ) + 3 ) // 4 )
1240+
1241+
1242+ def _tp4_max_line_tokens (content : str ) -> int :
1243+ """Return the largest single-line token estimate in *content*."""
1244+ return max (
1245+ (_tp4_line_tokens (line ) for line in content .splitlines (keepends = True )),
1246+ default = 1 ,
1247+ )
1248+
1249+
12341250def _tp4_line_chunks (content : str , max_tokens : int ) -> Iterator [_TP4CodeChunk ]:
12351251 """Yield bounded line-aligned chunks; oversized single lines fail closed."""
12361252 lines = content .splitlines (keepends = True )
12371253 current : list [str ] = []
12381254 current_tokens = 0
12391255 start_line = 1
12401256 for line_number , line in enumerate (lines , start = 1 ):
1241- line_tokens = max ( 1 , ( len ( line ) + 3 ) // 4 )
1257+ line_tokens = _tp4_line_tokens ( line )
12421258 if line_tokens > max_tokens :
12431259 if current :
12441260 yield _TP4CodeChunk ("" .join (current ), start_line , line_number - 1 )
@@ -1455,8 +1471,14 @@ def plan_candidate(candidate: _TP4Candidate) -> None:
14551471 # kept the preceding side whole and dropped the trailing
14561472 # instruction first. Shrink the context first and drop it before
14571473 # ever touching code.
1474+ header_tokens = estimate_tokens (header )
1475+ code_floor = TP4_MIN_CODE_TOKENS
1476+ if had_context :
1477+ max_line_tokens = _tp4_max_line_tokens (candidate .content )
1478+ if max_line_tokens <= code_token_budget - header_tokens :
1479+ code_floor = max (code_floor , max_line_tokens )
14581480 fixed = len (header ) + len (_TP4_CONTEXT_OPEN ) + 1 + len (_TP4_CONTEXT_CLOSE )
1459- room = (code_token_budget - TP4_MIN_CODE_TOKENS ) * 4 - fixed
1481+ room = (code_token_budget - code_floor ) * 4 - fixed
14601482 if had_context :
14611483 half = max (0 , room // 2 )
14621484 before_cap = half + max (0 , half - len (after ))
@@ -1482,11 +1504,11 @@ def plan_candidate(candidate: _TP4Candidate) -> None:
14821504 )
14831505 )
14841506 candidate_token_budget = code_token_budget - estimate_tokens (header + context_block )
1485- if candidate_token_budget < TP4_MIN_CODE_TOKENS and context_block :
1507+ if candidate_token_budget < code_floor and context_block :
14861508 # Safety net if the token estimate ever diverges from the
14871509 # character allowance above: drop the context, never the code.
14881510 context_block = ""
1489- candidate_token_budget = code_token_budget - estimate_tokens ( header )
1511+ candidate_token_budget = code_token_budget - header_tokens
14901512 if candidate_token_budget < TP4_MIN_CODE_TOKENS :
14911513 stop_planning (
14921514 LedgerReason .SIZE_LIMIT ,
0 commit comments