Skip to content

Commit d1b1f74

Browse files
anmolxlightAnmol Mishra
andauthored
fix(openlineage): self-heal ProcessPoolExecutor on BrokenProcessPool (#67400)
* fix(openlineage): self-heal ProcessPoolExecutor on BrokenProcessPool When a child process in the OpenLineage listener's ProcessPoolExecutor terminates abruptly, concurrent.futures marks the pool as permanently broken. Every subsequent submission raises BrokenProcessPool and lineage data stops flowing until the scheduler is restarted. This adds self-healing: submit_callable now catches BrokenProcessPool, shuts down the broken executor, creates a fresh one, and retries the submission so lineage reporting recovers automatically. Closes #67283 * fix(openlineage): sort imports to fix ruff I001 static check ruff I001: `from concurrent.futures.process import BrokenProcessPool` must follow `from concurrent.futures import ProcessPoolExecutor` * fix(openlineage): collapse log.warning to single line for ruff-format The warning message fits within the line-length limit so it should not be split across three lines. --------- Co-authored-by: Anmol Mishra <[email protected]>
1 parent 99816d8 commit d1b1f74

2 files changed

Lines changed: 40 additions & 1 deletion

File tree

‎providers/openlineage/src/airflow/providers/openlineage/plugins/listener.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import os
2121
import sys
2222
from concurrent.futures import ProcessPoolExecutor
23+
from concurrent.futures.process import BrokenProcessPool
2324
from datetime import datetime
2425
from functools import cache
2526
from typing import TYPE_CHECKING
@@ -1049,7 +1050,13 @@ def on_dag_run_failed(self, dag_run: DagRun, msg: str) -> None:
10491050
self.log.warning("OpenLineage received exception in method on_dag_run_failed", exc_info=e)
10501051

10511052
def submit_callable(self, callable, *args, **kwargs):
1052-
fut = self.executor.submit(callable, *args, **kwargs)
1053+
try:
1054+
fut = self.executor.submit(callable, *args, **kwargs)
1055+
except BrokenProcessPool:
1056+
self.log.warning("ProcessPoolExecutor is broken; recreating and retrying submission.")
1057+
self._executor.shutdown(wait=False)
1058+
self._executor = None
1059+
fut = self.executor.submit(callable, *args, **kwargs)
10531060
fut.add_done_callback(self.log_submit_error)
10541061
return fut
10551062

‎providers/openlineage/tests/unit/openlineage/plugins/test_listener.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2128,6 +2128,38 @@ def set_result(*args, **kwargs):
21282128
listener.log.debug.assert_not_called()
21292129
listener.log.warning.assert_called_once()
21302130

2131+
def test_submit_callable_recreates_executor_on_broken_pool(self):
2132+
"""When a child process dies and BrokenProcessPool is raised, the
2133+
listener should shut down the broken executor, create a fresh one, and
2134+
retry the submission."""
2135+
from concurrent.futures.process import BrokenProcessPool
2136+
2137+
listener = OpenLineageListener()
2138+
broken_executor = MagicMock()
2139+
broken_executor.submit.side_effect = BrokenProcessPool()
2140+
new_executor = MagicMock()
2141+
new_future = MagicMock()
2142+
new_executor.submit.return_value = new_future
2143+
2144+
listener._executor = broken_executor
2145+
listener.log = MagicMock()
2146+
2147+
def dummy_callable():
2148+
pass
2149+
2150+
with mock.patch(
2151+
"airflow.providers.openlineage.plugins.listener.ProcessPoolExecutor",
2152+
return_value=new_executor,
2153+
):
2154+
fut = listener.submit_callable(dummy_callable, "arg1", kwarg1="val1")
2155+
2156+
broken_executor.shutdown.assert_called_once_with(wait=False)
2157+
new_executor.submit.assert_called_once_with(dummy_callable, "arg1", kwarg1="val1")
2158+
new_future.add_done_callback.assert_called_once_with(listener.log_submit_error)
2159+
assert fut is new_future
2160+
listener.log.warning.assert_called_once()
2161+
assert "recreating" in listener.log.warning.call_args[0][0]
2162+
21312163

21322164
@pytest.mark.skipif(AIRFLOW_V_3_0_PLUS, reason="Airflow 2 tests")
21332165
class TestOpenLineageSelectiveEnableAirflow2:

0 commit comments

Comments
 (0)