Skip to content

Commit 364a7d0

Browse files
committed
Thread bundle_path through sync_bag_to_db and skip listener args when no listener is registered
Address review feedback: - sync_bag_to_db() now forwards dagbag.bundle_path to update_dag_parsing_results_in_db(), so the CLI/dag.test() path no longer falls back to the bundle-instantiating full_file_path() inside the parsing transaction. - Restore a cheap has_listeners early exit so the fallback (select + full_file_path) never runs when no listener plugin is registered, regardless of whether bundle_path was threaded down. - Use the per-file bundle_name_ consistently when building the listener filename in the new-error branch. - Add regression tests: through sync_bag_to_db() end-to-end (fails if a production caller stops forwarding bundle_path), for the existing-error listener branch, and for the no-listener fallback.
1 parent 86ba347 commit 364a7d0

4 files changed

Lines changed: 131 additions & 11 deletions

File tree

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
Fix DAG import errors being silently dropped (never shown in the *Import Errors* view). While recording an import error, ``_update_import_errors`` called ``ParseImportError.full_file_path()`` to build the import-error listener argument. ``full_file_path()`` re-instantiates the DAG bundle and, for some bundle types (e.g. the Git bundle), resolves an Airflow Connection; doing so inside the DAG-parsing transaction could roll back the not-yet-flushed import error row, leaving the DAG marked as broken with nothing shown in the UI. The already-known bundle path is now threaded down to ``_update_import_errors`` and used to build the listener filename directly, so the bundle is no longer re-instantiated inside the transaction.
1+
Fix DAG import errors being silently dropped (never shown in the *Import Errors* view). While recording an import error, ``_update_import_errors`` called ``ParseImportError.full_file_path()`` to build the import-error listener argument. ``full_file_path()`` re-instantiates the DAG bundle and, for some bundle types (e.g. the Git bundle), resolves an Airflow Connection; doing so inside the DAG-parsing transaction could roll back the not-yet-flushed import error row, leaving the DAG marked as broken with nothing shown in the UI. The already-known bundle path is now threaded down to ``_update_import_errors`` (from both the DAG processor and ``sync_bag_to_db``) and used to build the listener filename directly, so the bundle is no longer re-instantiated inside the transaction. Additionally, when no listener plugin is registered, the listener arguments are no longer computed at all.

‎airflow-core/src/airflow/dag_processing/collection.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,13 @@ def _update_import_errors(
403403
):
404404
from airflow.listeners.listener import get_listener_manager
405405

406+
# When no listener plugin is registered at all, skip building the listener
407+
# arguments entirely: for callers that do not thread ``bundle_path`` down, the
408+
# fallback below would otherwise query ``ParseImportError`` and call
409+
# ``full_file_path()`` -- re-instantiating the DAG bundle inside this parsing
410+
# transaction -- just to feed a hook nobody listens to.
411+
notify_listeners = get_listener_manager().has_listeners
412+
406413
def _listener_filename(
407414
relative_fileloc: str, bundle_name_: str, import_error: ParseImportError | None = None
408415
) -> str | None:
@@ -464,11 +471,12 @@ def _listener_filename(
464471
# sending notification when an existing dag import error occurs
465472
try:
466473
# todo: make listener accept bundle_name and relative_filename
467-
filename = _listener_filename(relative_fileloc, bundle_name_)
468-
if filename is not None:
469-
get_listener_manager().hook.on_existing_dag_import_error(
470-
filename=filename, stacktrace=stacktrace
471-
)
474+
if notify_listeners:
475+
filename = _listener_filename(relative_fileloc, bundle_name_)
476+
if filename is not None:
477+
get_listener_manager().hook.on_existing_dag_import_error(
478+
filename=filename, stacktrace=stacktrace
479+
)
472480
except Exception:
473481
log.exception("error calling listener")
474482
else:
@@ -481,11 +489,12 @@ def _listener_filename(
481489
session.add(import_error)
482490
# sending notification when a new dag import error occurs
483491
try:
484-
filename = _listener_filename(relative_fileloc, bundle_name, import_error)
485-
if filename is not None:
486-
get_listener_manager().hook.on_new_dag_import_error(
487-
filename=filename, stacktrace=stacktrace
488-
)
492+
if notify_listeners:
493+
filename = _listener_filename(relative_fileloc, bundle_name_, import_error)
494+
if filename is not None:
495+
get_listener_manager().hook.on_new_dag_import_error(
496+
filename=filename, stacktrace=stacktrace
497+
)
489498
except Exception:
490499
log.exception("error calling listener")
491500
session.execute(

‎airflow-core/src/airflow/dag_processing/dagbag.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,4 +579,5 @@ def sync_bag_to_db(
579579
session=session,
580580
version_data=version_data,
581581
files_parsed=files_parsed,
582+
bundle_path=dagbag.bundle_path,
582583
)

‎airflow-core/tests/unit/dag_processing/test_collection.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -946,6 +946,116 @@ def test_import_error_listener_uses_bundle_path_without_full_file_path(
946946
# The listener still received the correct filename, built from bundle_path.
947947
assert dag_import_error_listener.new[str(bundle_path / "abc.py")] == "AnImportError"
948948

949+
@patch.object(ParseImportError, "full_file_path")
950+
@pytest.mark.usefixtures("clean_db")
951+
def test_existing_import_error_listener_uses_bundle_path_without_full_file_path(
952+
self, mock_full_path, session, dag_import_error_listener, testing_dag_bundle
953+
):
954+
"""
955+
The existing-error branch also builds the listener filename from ``bundle_path``.
956+
957+
When an import error row already exists for the file, ``on_existing_dag_import_error``
958+
is notified instead of ``on_new_dag_import_error``; that branch previously ran an extra
959+
``select(ParseImportError)`` and called ``full_file_path()`` just to build the listener
960+
argument. With ``bundle_path`` threaded down, neither must happen.
961+
"""
962+
mock_full_path.side_effect = AssertionError(
963+
"full_file_path() must not be called when bundle_path is provided"
964+
)
965+
bundle_path = Path("/opt/airflow/bundles/testing")
966+
session.add(
967+
ParseImportError(
968+
filename="abc.py", bundle_name="testing", timestamp=tz.utcnow(), stacktrace="OldError"
969+
)
970+
)
971+
session.flush()
972+
973+
_update_import_errors(
974+
files_parsed={("testing", "abc.py")},
975+
bundle_name="testing",
976+
import_errors={("testing", "abc.py"): "NewImportError"},
977+
session=session,
978+
bundle_path=bundle_path,
979+
)
980+
session.flush()
981+
982+
# The existing row is updated in place.
983+
persisted = session.scalars(select(ParseImportError)).all()
984+
assert len(persisted) == 1
985+
assert persisted[0].stacktrace == "NewImportError"
986+
987+
mock_full_path.assert_not_called()
988+
989+
# The existing-error listener received the bundle_path-based filename.
990+
assert dag_import_error_listener.new == {}
991+
assert dag_import_error_listener.existing[str(bundle_path / "abc.py")] == "NewImportError"
992+
993+
@patch.object(ParseImportError, "full_file_path")
994+
@pytest.mark.usefixtures("clean_db")
995+
def test_sync_bag_to_db_does_not_call_full_file_path(
996+
self, mock_full_path, tmp_path, session, dag_import_error_listener, testing_dag_bundle
997+
):
998+
"""
999+
``sync_bag_to_db`` threads ``dagbag.bundle_path`` down to ``_update_import_errors``.
1000+
1001+
This exercises a production caller end-to-end (instead of the private helper), so it
1002+
fails if any caller in the chain stops forwarding ``bundle_path`` and the code falls
1003+
back to the bundle-instantiating ``full_file_path()`` inside the parsing transaction.
1004+
"""
1005+
from airflow.dag_processing.dagbag import DagBag, sync_bag_to_db
1006+
1007+
mock_full_path.side_effect = AssertionError(
1008+
"full_file_path() must not be called when bundle_path is provided"
1009+
)
1010+
bundle_path = tmp_path / "bundle"
1011+
bundle_path.mkdir()
1012+
dag_file = bundle_path / "broken.py"
1013+
dag_file.write_text("raise ImportError('broken dag')")
1014+
1015+
dagbag = DagBag(dag_folder=str(dag_file), bundle_path=bundle_path, bundle_name="testing")
1016+
assert dagbag.import_errors
1017+
1018+
sync_bag_to_db(dagbag, "testing", None, session=session)
1019+
session.flush()
1020+
1021+
persisted = session.scalars(select(ParseImportError)).all()
1022+
assert len(persisted) == 1
1023+
assert persisted[0].bundle_name == "testing"
1024+
assert persisted[0].filename == "broken.py"
1025+
1026+
mock_full_path.assert_not_called()
1027+
assert dag_import_error_listener.new[str(bundle_path / "broken.py")] == persisted[0].stacktrace
1028+
1029+
@patch.object(ParseImportError, "full_file_path")
1030+
@pytest.mark.usefixtures("clean_db")
1031+
def test_import_error_persisted_without_listener_or_bundle_path(
1032+
self, mock_full_path, session, testing_dag_bundle
1033+
):
1034+
"""
1035+
With no listener registered, the ``full_file_path()`` fallback must not run at all.
1036+
1037+
Callers that do not thread ``bundle_path`` down would otherwise still hit the
1038+
bundle-instantiating fallback inside the parsing transaction just to build an argument
1039+
for a hook nobody listens to -- the original silent-drop bug.
1040+
"""
1041+
mock_full_path.side_effect = AssertionError(
1042+
"full_file_path() must not be called when no listener is registered"
1043+
)
1044+
_update_import_errors(
1045+
files_parsed={("testing", "abc.py")},
1046+
bundle_name="testing",
1047+
import_errors={("testing", "abc.py"): "AnImportError"},
1048+
session=session,
1049+
)
1050+
session.flush()
1051+
1052+
persisted = session.scalars(select(ParseImportError)).all()
1053+
assert len(persisted) == 1
1054+
assert persisted[0].filename == "abc.py"
1055+
assert persisted[0].stacktrace == "AnImportError"
1056+
1057+
mock_full_path.assert_not_called()
1058+
9491059
@patch.object(ParseImportError, "full_file_path")
9501060
@mark_fab_auth_manager_test
9511061
@conf_vars({("core", "min_serialized_dag_update_interval"): "5"})

0 commit comments

Comments
 (0)