Skip to content

Commit d4479ed

Browse files
committed
feat(workflows): address review — sliding-window fan-out, locked output, faithful halt
Address the reviewer feedback on the bounded fan-out concurrency: - Sliding submission window: keep at most `workers` items in flight and stop launching new items once the run is halting, instead of submitting all items up front (which let the pool keep starting queued work after a halt). - Faithful halt prefix: attribute a halt to the specific item whose own recorded result halted the run (replaying the sequential break condition, honoring continue_on_error/aborted), not the shared run status a later concurrent item may have flipped. The returned prefix now includes the actual halting item, matching the sequential path. An item that fails before recording a result (e.g. an unknown step type) is attributed too, since every item runs the same template. - Lock the parent fan-out output mutation: route the post-fan-out step_results[...]['output'] update through a new RunState.set_step_output() under the run lock, so it cannot race a concurrent save(). - Docstring: describe int() coercion accurately (numeric strings / floats are honored; only non-coercible or <= 1 runs sequentially). Tests: add concurrent halt-includes-halting-item, continue_on_error-does-not- truncate, and unknown-template-type-matches-sequential coverage; make the timing test use a monotonic clock with a looser threshold to avoid CI flakiness.
1 parent f69afd1 commit d4479ed

2 files changed

Lines changed: 155 additions & 31 deletions

File tree

‎src/specify_cli/workflows/engine.py‎

Lines changed: 95 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import tempfile
1818
import threading
1919
import uuid
20-
from concurrent.futures import ThreadPoolExecutor
20+
from concurrent.futures import Future, ThreadPoolExecutor
2121
from datetime import datetime, timezone
2222
from pathlib import Path
2323
from typing import Any
@@ -405,6 +405,18 @@ def record_step_result(self, step_id: str, data: dict[str, Any]) -> None:
405405
with self._lock:
406406
self.step_results[step_id] = data
407407

408+
def set_step_output(self, step_id: str, output: Any) -> None:
409+
"""Replace an already-recorded step's ``output`` under the run lock.
410+
411+
Fan-out updates its parent step's output after the items have run;
412+
routing that nested mutation through the lock keeps it from racing a
413+
``save()`` serializing ``step_results`` — the same invariant
414+
``record_step_result`` provides for the top-level assignment.
415+
"""
416+
with self._lock:
417+
if step_id in self.step_results:
418+
self.step_results[step_id]["output"] = output
419+
408420
def save(self) -> None:
409421
"""Persist current state to disk.
410422
@@ -927,7 +939,7 @@ def _execute_steps(
927939
fan_out_output = dict(result.output)
928940
fan_out_output["results"] = fan_out_results
929941
context.steps[step_id]["output"] = fan_out_output
930-
state.step_results[step_id]["output"] = fan_out_output
942+
state.set_step_output(step_id, fan_out_output)
931943
if state.status in (
932944
RunStatus.PAUSED,
933945
RunStatus.FAILED,
@@ -938,7 +950,7 @@ def _execute_steps(
938950
# Empty items or no template — normalize output
939951
result.output["results"] = []
940952
context.steps[step_id]["output"] = result.output
941-
state.step_results[step_id]["output"] = result.output
953+
state.set_step_output(step_id, result.output)
942954

943955
def _run_fan_out(
944956
self,
@@ -952,25 +964,40 @@ def _run_fan_out(
952964
) -> list[Any]:
953965
"""Run a fan-out template once per item; return per-item outputs in item order.
954966
955-
``max_concurrency`` <= 1 (the default) runs items sequentially, identical to
956-
the historical fan-out behavior. ``max_concurrency`` > 1 runs up to that many
957-
items concurrently on a bounded thread pool. In both modes results are returned
958-
in item order (never completion order), and the first item to reach a halting
959-
run status (PAUSED/FAILED/ABORTED) stops further dispatch; on halt the
960-
contiguous prefix of items that ran is returned. A non-int / 0 / negative /
961-
``None`` ``max_concurrency`` coerces to 1 (sequential).
967+
``max_concurrency`` <= 1 (the default) runs items sequentially, identical
968+
to the historical fan-out behavior. ``max_concurrency`` > 1 runs items on a
969+
bounded thread pool using a sliding submission window of that size: at most
970+
that many items are ever in flight, and no new item is launched once the run
971+
has reached a halting status, so a halt cannot keep starting queued work.
972+
973+
Results are always returned in item order (never completion order). On a
974+
halt (PAUSED/FAILED/ABORTED) the returned prefix is the items up to and
975+
including the first item *in item order* whose own execution halted the run
976+
— identical to the sequential path — and any later in-flight items are
977+
cancelled. Halt is attributed per item from that item's recorded result
978+
(not the shared run status, which a concurrently-running later item may have
979+
already flipped), so the prefix never drops the actual halting item.
980+
981+
``max_concurrency`` is coerced with ``int()``; a value that cannot be
982+
coerced (``None``, a non-numeric string, …) or that coerces to <= 1 runs
983+
sequentially, while a numeric string like ``"4"`` or a float like ``4.0``
984+
is honored.
962985
"""
963986
halting = (RunStatus.PAUSED, RunStatus.FAILED, RunStatus.ABORTED)
964987
try:
965988
workers = max(1, int(max_concurrency))
966989
except (TypeError, ValueError):
967990
workers = 1
968991

992+
base_id = template.get("id", "item")
993+
994+
def item_id(idx: int) -> str:
995+
# Per-item ID grammar: parentId:templateId:index.
996+
return f"{step_id}:{base_id}:{idx}"
997+
969998
def run_item(idx: int, item_ctx: StepContext) -> Any:
970-
# Per-item ID: parentId:templateId:index
971999
item_step = dict(template)
972-
base_id = item_step.get("id", "item")
973-
item_step["id"] = f"{step_id}:{base_id}:{idx}"
1000+
item_step["id"] = item_id(idx)
9741001
self._execute_steps(
9751002
[item_step], item_ctx, state, registry, step_offset=-1,
9761003
)
@@ -986,7 +1013,7 @@ def run_item(idx: int, item_ctx: StepContext) -> Any:
9861013
break
9871014
return results
9881015

989-
# Concurrent path — bounded thread pool; results assembled in item order.
1016+
# Concurrent path — bounded sliding window; results assembled in item order.
9901017
n = len(items)
9911018
slots: list[Any] = [None] * n
9921019

@@ -996,18 +1023,54 @@ def run_isolated(idx: int) -> Any:
9961023
# disjoint parentId:templateId:index key (GIL-safe on distinct keys).
9971024
return run_item(idx, dataclasses.replace(context, item=items[idx]))
9981025

1026+
def item_halted(idx: int) -> bool:
1027+
# Did THIS item's own execution halt the run? Decide from the item's
1028+
# own recorded result, not the shared run status, so a later item's
1029+
# concurrent halt is never misattributed here. Mirrors the sequential
1030+
# break condition: PAUSED halts; FAILED halts unless continue_on_error
1031+
# routes around it; an abort always halts.
1032+
rec = context.steps.get(item_id(idx))
1033+
if rec is None:
1034+
# Ran but recorded nothing — only happens when the item failed
1035+
# before record_step_result (e.g. an unknown step type returns
1036+
# early). Every item runs the same template, so the run status is
1037+
# this item's own outcome; attribute the halt to it.
1038+
return state.status in halting
1039+
status = rec.get("status")
1040+
if status == StepStatus.PAUSED.value:
1041+
return True
1042+
if status == StepStatus.FAILED.value:
1043+
out = rec.get("output") or {}
1044+
if out.get("aborted") or template.get("continue_on_error") is not True:
1045+
return True
1046+
return False
1047+
9991048
first_exc: Exception | None = None
1000-
ran = 0
1049+
halted_at: int | None = None
1050+
collected = 0
10011051
with ThreadPoolExecutor(max_workers=workers) as pool:
1002-
futures: dict[Any, int] = {}
1052+
futures: dict[int, Future] = {}
1053+
next_submit = 0
10031054
for idx in range(n):
1004-
if state.status in halting:
1055+
# Refill the window: keep <= workers in flight, and stop launching
1056+
# new items once the run is halting so a halt cannot keep starting
1057+
# queued work. Already-submitted futures are still collected in
1058+
# item order below.
1059+
while (
1060+
next_submit < n
1061+
and len(futures) < workers
1062+
and state.status not in halting
1063+
):
1064+
futures[next_submit] = pool.submit(run_isolated, next_submit)
1065+
next_submit += 1
1066+
1067+
fut = futures.pop(idx, None)
1068+
if fut is None:
1069+
# Safety net: the window submits indices in order and the loop
1070+
# breaks at the first halting item, so every collected index has
1071+
# an in-flight future. Stop cleanly rather than raise if a future
1072+
# change ever breaks that invariant.
10051073
break
1006-
futures[pool.submit(run_isolated, idx)] = idx
1007-
# Collect in submission (item) order: each .result() blocks on that
1008-
# item, so slots fill 0,1,2,… and the collected set is a prefix.
1009-
for fut in list(futures):
1010-
idx = futures[fut]
10111074
try:
10121075
slots[idx] = fut.result()
10131076
except Exception as exc:
@@ -1016,18 +1079,23 @@ def run_isolated(idx: int) -> Any:
10161079
# outstanding work and re-raise so the engine marks the run
10171080
# failed instead of reporting a vacuous completion.
10181081
first_exc = exc
1019-
for other in futures:
1082+
for other in futures.values():
10201083
other.cancel()
10211084
break
1022-
ran = idx + 1
1023-
if state.status in halting:
1024-
for other in futures:
1085+
collected = idx + 1
1086+
if item_halted(idx):
1087+
# First halting item in item order: include it (slots[idx] is
1088+
# already set), then cancel everything still pending.
1089+
halted_at = idx
1090+
for other in futures.values():
10251091
other.cancel()
10261092
break
10271093

10281094
if first_exc is not None:
10291095
raise first_exc
1030-
return slots[:ran] if state.status in halting else slots
1096+
if halted_at is not None:
1097+
return slots[: halted_at + 1]
1098+
return slots[:collected]
10311099

10321100
def _resolve_inputs(
10331101
self,

‎tests/test_workflows.py‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2057,13 +2057,21 @@ def on_item(item):
20572057
def test_concurrency_is_real(self, tmp_path):
20582058
import time
20592059

2060+
n = 4
2061+
delay = 0.2
2062+
20602063
def on_item(item):
2061-
time.sleep(0.3)
2064+
time.sleep(delay)
20622065
return None
20632066

2064-
t0 = time.time()
2065-
self._run(tmp_path, list(range(4)), 4, on_item)
2066-
assert time.time() - t0 < 0.6 # serialized would be >= 1.2s
2067+
# Monotonic clock (immune to wall-clock adjustments). All n items run
2068+
# concurrently, so elapsed is ~one delay; a generous bound well under the
2069+
# serialized baseline keeps a clear gap while tolerating slow/loaded CI.
2070+
serialized = n * delay
2071+
t0 = time.monotonic()
2072+
self._run(tmp_path, list(range(n)), n, on_item)
2073+
elapsed = time.monotonic() - t0
2074+
assert elapsed < serialized * 0.6 # serialized would be >= n*delay
20672075

20682076
@pytest.mark.parametrize("bad", [0, -1, None, "abc", 1.0])
20692077
def test_invalid_max_concurrency_coerces_to_sequential(self, tmp_path, bad):
@@ -2094,6 +2102,54 @@ def on_item(item):
20942102
assert results[2] == {"seen": 2}
20952103
assert state.status == RunStatus.FAILED
20962104

2105+
def test_halt_on_failure_concurrent_includes_halting_item(self, tmp_path):
2106+
# The concurrent prefix must match the sequential one: items up to and
2107+
# INCLUDING the failing item (2), never a short prefix that drops it just
2108+
# because a later in-flight item flipped the shared run status first.
2109+
from specify_cli.workflows.base import RunStatus, StepStatus
2110+
2111+
def on_item(item):
2112+
return StepStatus.FAILED if item == 2 else None
2113+
2114+
results, state = self._run(tmp_path, list(range(6)), 4, on_item)
2115+
assert results == [{"seen": 0}, {"seen": 1}, {"seen": 2}]
2116+
assert state.status == RunStatus.FAILED
2117+
2118+
def test_continue_on_error_item_does_not_halt_concurrent(self, tmp_path):
2119+
# A failing item whose template sets continue_on_error must NOT truncate
2120+
# the fan-out: every item still runs and is returned in order.
2121+
from specify_cli.workflows.base import StepStatus
2122+
2123+
def on_item(item):
2124+
return StepStatus.FAILED if item == 2 else None
2125+
2126+
engine, context, state, registry, template = self._build(tmp_path, on_item)
2127+
template["continue_on_error"] = True
2128+
results = engine._run_fan_out(
2129+
list(range(5)), template, "fan", context, state, registry, 4
2130+
)
2131+
assert results == [{"seen": i} for i in range(5)]
2132+
2133+
def test_unknown_template_type_halts_concurrent_like_sequential(self, tmp_path):
2134+
# A template whose type isn't registered fails fast and records no result;
2135+
# the concurrent path must still attribute the halt to the first item and
2136+
# return the same prefix as sequential — never run on as if completed.
2137+
from specify_cli.workflows.base import RunStatus, StepContext
2138+
from specify_cli.workflows.engine import RunState, WorkflowEngine
2139+
2140+
def fresh():
2141+
state = RunState(run_id="r", workflow_id="w", project_root=tmp_path)
2142+
state.status = RunStatus.RUNNING
2143+
return WorkflowEngine(project_root=tmp_path), StepContext(), state
2144+
2145+
template = {"id": "impl", "type": "does-not-exist"}
2146+
e1, c1, s1 = fresh()
2147+
seq = e1._run_fan_out(list(range(5)), template, "fan", c1, s1, {}, 1)
2148+
e2, c2, s2 = fresh()
2149+
con = e2._run_fan_out(list(range(5)), template, "fan", c2, s2, {}, 4)
2150+
assert seq == con == [{}] # halted at the first item; rest never returned
2151+
assert s1.status == s2.status == RunStatus.FAILED
2152+
20972153
def test_first_exception_cancels_and_reraises(self, tmp_path):
20982154
def on_item(item):
20992155
if item == 0:

0 commit comments

Comments
 (0)