Skip to content

Commit ce352a3

Browse files
committed
feat(workflows): address second review pass — concurrency hardening
- append_log: serialize the log_entries append + log.jsonl write under a dedicated RunState._log_lock so concurrent fan-out workers can't interleave or corrupt log lines (kept separate from the state lock; never nested). - _run_fan_out.run_item: read the item output back through the item_ctx it executed against rather than the outer context closure — clearer and robust if StepContext ever stops sharing the steps dict by reference. - StepBase: document the thread-safety contract — STEP_REGISTRY holds one shared instance per type, so concurrent fan-out invokes execute() on the same object; implementations must be stateless/thread-safe (the built-ins already are). - test_concurrency_is_real: prove parallelism deterministically with a threading.Barrier (sequential execution can't clear it) instead of a wall-clock timing assertion.
1 parent d4479ed commit ce352a3

3 files changed

Lines changed: 34 additions & 17 deletions

File tree

‎src/specify_cli/workflows/base.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,13 @@ class StepBase(ABC):
9797
9898
Every step type — built-in or extension-provided — implements this
9999
interface and registers in ``STEP_REGISTRY``.
100+
101+
Thread-safety: ``STEP_REGISTRY`` holds a single shared instance per type, so
102+
a concurrent ``fan-out`` (``max_concurrency > 1``) can invoke ``execute`` on
103+
the same instance from several threads at once. Implementations must be
104+
stateless / thread-safe — derive all per-run state from the ``config`` and
105+
``context`` arguments and never mutate ``self`` in ``execute``. The built-in
106+
steps follow this rule.
100107
"""
101108

102109
#: Matches the ``type:`` value in workflow YAML.

‎src/specify_cli/workflows/engine.py‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,11 @@ def __init__(
386386
# mutate the dict while save() is serializing it (which would raise
387387
# "dictionary changed size during iteration").
388388
self._lock = threading.Lock()
389+
# Serializes append_log's list append + log.jsonl write so concurrent
390+
# fan-out workers cannot interleave or corrupt log lines. Kept separate
391+
# from _lock so frequent logging never contends with state saves; since
392+
# append_log is never called while _lock is held, the two never nest.
393+
self._log_lock = threading.Lock()
389394
self.inputs: dict[str, Any] = {}
390395
self.created_at = datetime.now(timezone.utc).isoformat()
391396
self.updated_at = self.created_at
@@ -506,14 +511,18 @@ def load(cls, run_id: str, project_root: Path) -> RunState:
506511
return state
507512

508513
def append_log(self, entry: dict[str, Any]) -> None:
509-
"""Append a log entry to the run log."""
510-
entry["timestamp"] = datetime.now(timezone.utc).isoformat()
511-
self.log_entries.append(entry)
514+
"""Append a log entry to the run log.
512515
516+
Held under ``_log_lock`` so concurrent fan-out workers serialize their
517+
list append and ``log.jsonl`` write rather than interleaving lines.
518+
"""
519+
entry["timestamp"] = datetime.now(timezone.utc).isoformat()
513520
runs_dir = self.runs_dir
514521
runs_dir.mkdir(parents=True, exist_ok=True)
515-
with open(runs_dir / "log.jsonl", "a", encoding="utf-8") as f:
516-
f.write(json.dumps(entry) + "\n")
522+
with self._log_lock:
523+
self.log_entries.append(entry)
524+
with open(runs_dir / "log.jsonl", "a", encoding="utf-8") as f:
525+
f.write(json.dumps(entry) + "\n")
517526

518527

519528
# -- Workflow Engine ------------------------------------------------------
@@ -1001,7 +1010,10 @@ def run_item(idx: int, item_ctx: StepContext) -> Any:
10011010
self._execute_steps(
10021011
[item_step], item_ctx, state, registry, step_offset=-1,
10031012
)
1004-
return context.steps.get(item_step["id"], {}).get("output", {})
1013+
# Read back through the context that was actually executed against,
1014+
# not the outer closure — clearer and robust if StepContext copying
1015+
# ever stops sharing the steps dict by reference.
1016+
return item_ctx.steps.get(item_step["id"], {}).get("output", {})
10051017

10061018
# Sequential path — identical to the historical behavior.
10071019
if workers <= 1:

‎tests/test_workflows.py‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2055,23 +2055,21 @@ def on_item(item):
20552055
assert completion == list(reversed(range(n)))
20562056

20572057
def test_concurrency_is_real(self, tmp_path):
2058-
import time
2058+
import threading
20592059

2060+
# Deterministic proof of real parallelism (no wall-clock threshold to
2061+
# tune or flake): every item must reach the barrier before any may pass.
2062+
# Sequential execution would block the first item forever — the barrier
2063+
# times out, raises BrokenBarrierError, and fails the test.
20602064
n = 4
2061-
delay = 0.2
2065+
barrier = threading.Barrier(n, timeout=5)
20622066

20632067
def on_item(item):
2064-
time.sleep(delay)
2068+
barrier.wait()
20652069
return None
20662070

2067-
# Monotonic clock (immune to wall-clock adjustments). All n items run
2068-
# concurrently, so elapsed is ~one delay; a generous bound well under the
2069-
# serialized baseline keeps a clear gap while tolerating slow/loaded CI.
2070-
serialized = n * delay
2071-
t0 = time.monotonic()
2072-
self._run(tmp_path, list(range(n)), n, on_item)
2073-
elapsed = time.monotonic() - t0
2074-
assert elapsed < serialized * 0.6 # serialized would be >= n*delay
2071+
results, _ = self._run(tmp_path, list(range(n)), n, on_item)
2072+
assert results == [{"seen": i} for i in range(n)]
20752073

20762074
@pytest.mark.parametrize("bad", [0, -1, None, "abc", 1.0])
20772075
def test_invalid_max_concurrency_coerces_to_sequential(self, tmp_path, bad):

0 commit comments

Comments
 (0)