1717import tempfile
1818import threading
1919import uuid
20- from concurrent .futures import ThreadPoolExecutor
20+ from concurrent .futures import Future , ThreadPoolExecutor
2121from datetime import datetime , timezone
2222from pathlib import Path
2323from typing import Any
@@ -405,6 +405,18 @@ def record_step_result(self, step_id: str, data: dict[str, Any]) -> None:
405405 with self ._lock :
406406 self .step_results [step_id ] = data
407407
408+ def set_step_output (self , step_id : str , output : Any ) -> None :
409+ """Replace an already-recorded step's ``output`` under the run lock.
410+
411+ Fan-out updates its parent step's output after the items have run;
412+ routing that nested mutation through the lock keeps it from racing a
413+ ``save()`` serializing ``step_results`` — the same invariant
414+ ``record_step_result`` provides for the top-level assignment.
415+ """
416+ with self ._lock :
417+ if step_id in self .step_results :
418+ self .step_results [step_id ]["output" ] = output
419+
408420 def save (self ) -> None :
409421 """Persist current state to disk.
410422
@@ -927,7 +939,7 @@ def _execute_steps(
927939 fan_out_output = dict (result .output )
928940 fan_out_output ["results" ] = fan_out_results
929941 context .steps [step_id ]["output" ] = fan_out_output
930- state .step_results [ step_id ][ "output" ] = fan_out_output
942+ state .set_step_output ( step_id , fan_out_output )
931943 if state .status in (
932944 RunStatus .PAUSED ,
933945 RunStatus .FAILED ,
@@ -938,7 +950,7 @@ def _execute_steps(
938950 # Empty items or no template — normalize output
939951 result .output ["results" ] = []
940952 context .steps [step_id ]["output" ] = result .output
941- state .step_results [ step_id ][ "output" ] = result .output
953+ state .set_step_output ( step_id , result .output )
942954
943955 def _run_fan_out (
944956 self ,
@@ -952,25 +964,40 @@ def _run_fan_out(
952964 ) -> list [Any ]:
953965 """Run a fan-out template once per item; return per-item outputs in item order.
954966
955- ``max_concurrency`` <= 1 (the default) runs items sequentially, identical to
956- the historical fan-out behavior. ``max_concurrency`` > 1 runs up to that many
957- items concurrently on a bounded thread pool. In both modes results are returned
958- in item order (never completion order), and the first item to reach a halting
959- run status (PAUSED/FAILED/ABORTED) stops further dispatch; on halt the
960- contiguous prefix of items that ran is returned. A non-int / 0 / negative /
961- ``None`` ``max_concurrency`` coerces to 1 (sequential).
967+ ``max_concurrency`` <= 1 (the default) runs items sequentially, identical
968+ to the historical fan-out behavior. ``max_concurrency`` > 1 runs items on a
969+ bounded thread pool using a sliding submission window of that size: at most
970+ that many items are ever in flight, and no new item is launched once the run
971+ has reached a halting status, so a halt cannot keep starting queued work.
972+
973+ Results are always returned in item order (never completion order). On a
974+ halt (PAUSED/FAILED/ABORTED) the returned prefix is the items up to and
975+ including the first item *in item order* whose own execution halted the run
976+ — identical to the sequential path — and any later in-flight items are
977+ cancelled. Halt is attributed per item from that item's recorded result
978+ (not the shared run status, which a concurrently-running later item may have
979+ already flipped), so the prefix never drops the actual halting item.
980+
981+ ``max_concurrency`` is coerced with ``int()``; a value that cannot be
982+ coerced (``None``, a non-numeric string, …) or that coerces to <= 1 runs
983+ sequentially, while a numeric string like ``"4"`` or a float like ``4.0``
984+ is honored.
962985 """
963986 halting = (RunStatus .PAUSED , RunStatus .FAILED , RunStatus .ABORTED )
964987 try :
965988 workers = max (1 , int (max_concurrency ))
966989 except (TypeError , ValueError ):
967990 workers = 1
968991
992+ base_id = template .get ("id" , "item" )
993+
994+ def item_id (idx : int ) -> str :
995+ # Per-item ID grammar: parentId:templateId:index.
996+ return f"{ step_id } :{ base_id } :{ idx } "
997+
969998 def run_item (idx : int , item_ctx : StepContext ) -> Any :
970- # Per-item ID: parentId:templateId:index
971999 item_step = dict (template )
972- base_id = item_step .get ("id" , "item" )
973- item_step ["id" ] = f"{ step_id } :{ base_id } :{ idx } "
1000+ item_step ["id" ] = item_id (idx )
9741001 self ._execute_steps (
9751002 [item_step ], item_ctx , state , registry , step_offset = - 1 ,
9761003 )
@@ -986,7 +1013,7 @@ def run_item(idx: int, item_ctx: StepContext) -> Any:
9861013 break
9871014 return results
9881015
989- # Concurrent path — bounded thread pool ; results assembled in item order.
1016+ # Concurrent path — bounded sliding window ; results assembled in item order.
9901017 n = len (items )
9911018 slots : list [Any ] = [None ] * n
9921019
@@ -996,18 +1023,54 @@ def run_isolated(idx: int) -> Any:
9961023 # disjoint parentId:templateId:index key (GIL-safe on distinct keys).
9971024 return run_item (idx , dataclasses .replace (context , item = items [idx ]))
9981025
1026+ def item_halted (idx : int ) -> bool :
1027+ # Did THIS item's own execution halt the run? Decide from the item's
1028+ # own recorded result, not the shared run status, so a later item's
1029+ # concurrent halt is never misattributed here. Mirrors the sequential
1030+ # break condition: PAUSED halts; FAILED halts unless continue_on_error
1031+ # routes around it; an abort always halts.
1032+ rec = context .steps .get (item_id (idx ))
1033+ if rec is None :
1034+ # Ran but recorded nothing — only happens when the item failed
1035+ # before record_step_result (e.g. an unknown step type returns
1036+ # early). Every item runs the same template, so the run status is
1037+ # this item's own outcome; attribute the halt to it.
1038+ return state .status in halting
1039+ status = rec .get ("status" )
1040+ if status == StepStatus .PAUSED .value :
1041+ return True
1042+ if status == StepStatus .FAILED .value :
1043+ out = rec .get ("output" ) or {}
1044+ if out .get ("aborted" ) or template .get ("continue_on_error" ) is not True :
1045+ return True
1046+ return False
1047+
9991048 first_exc : Exception | None = None
1000- ran = 0
1049+ halted_at : int | None = None
1050+ collected = 0
10011051 with ThreadPoolExecutor (max_workers = workers ) as pool :
1002- futures : dict [Any , int ] = {}
1052+ futures : dict [int , Future ] = {}
1053+ next_submit = 0
10031054 for idx in range (n ):
1004- if state .status in halting :
1055+ # Refill the window: keep <= workers in flight, and stop launching
1056+ # new items once the run is halting so a halt cannot keep starting
1057+ # queued work. Already-submitted futures are still collected in
1058+ # item order below.
1059+ while (
1060+ next_submit < n
1061+ and len (futures ) < workers
1062+ and state .status not in halting
1063+ ):
1064+ futures [next_submit ] = pool .submit (run_isolated , next_submit )
1065+ next_submit += 1
1066+
1067+ fut = futures .pop (idx , None )
1068+ if fut is None :
1069+ # Safety net: the window submits indices in order and the loop
1070+ # breaks at the first halting item, so every collected index has
1071+ # an in-flight future. Stop cleanly rather than raise if a future
1072+ # change ever breaks that invariant.
10051073 break
1006- futures [pool .submit (run_isolated , idx )] = idx
1007- # Collect in submission (item) order: each .result() blocks on that
1008- # item, so slots fill 0,1,2,… and the collected set is a prefix.
1009- for fut in list (futures ):
1010- idx = futures [fut ]
10111074 try :
10121075 slots [idx ] = fut .result ()
10131076 except Exception as exc :
@@ -1016,18 +1079,23 @@ def run_isolated(idx: int) -> Any:
10161079 # outstanding work and re-raise so the engine marks the run
10171080 # failed instead of reporting a vacuous completion.
10181081 first_exc = exc
1019- for other in futures :
1082+ for other in futures . values () :
10201083 other .cancel ()
10211084 break
1022- ran = idx + 1
1023- if state .status in halting :
1024- for other in futures :
1085+ collected = idx + 1
1086+ if item_halted (idx ):
1087+ # First halting item in item order: include it (slots[idx] is
1088+ # already set), then cancel everything still pending.
1089+ halted_at = idx
1090+ for other in futures .values ():
10251091 other .cancel ()
10261092 break
10271093
10281094 if first_exc is not None :
10291095 raise first_exc
1030- return slots [:ran ] if state .status in halting else slots
1096+ if halted_at is not None :
1097+ return slots [: halted_at + 1 ]
1098+ return slots [:collected ]
10311099
10321100 def _resolve_inputs (
10331101 self ,
0 commit comments