Skip to content

Commit bc8ef9a

Browse files
Refactor deferrable execution in DbtCloudRunJobOperator to align with PR #64051 patterns. Simplify trigger polling/control flow and propagate execution_timeout via defer. Add tests for best-effort cancellation semantics in execute_complete and on_kill. (#66449)
1 parent 724ac5f commit bc8ef9a

5 files changed

Lines changed: 273 additions & 253 deletions

File tree

‎providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from pathlib import Path
2424
from typing import TYPE_CHECKING, Any
2525

26-
from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, BaseOperatorLink, XCom, conf
26+
from airflow.providers.common.compat.sdk import BaseOperator, BaseOperatorLink, XCom, conf
2727
from airflow.providers.dbt.cloud.hooks.dbt import (
2828
DbtCloudHook,
2929
DbtCloudJobRunException,
@@ -250,16 +250,16 @@ def execute(self, context: Context):
250250
# execution_timeout is a hard task-level limit (cancels the job),
251251
# while timeout only limits how long we wait for the job to finish.
252252
# If both are set, the earliest deadline wins.
253-
end_time = time.time() + self.timeout
253+
end_time = time.monotonic() + self.timeout
254254
execution_deadline = None
255-
if self.execution_timeout:
256-
execution_deadline = time.time() + self.execution_timeout.total_seconds()
255+
if self.execution_timeout is not None:
256+
execution_deadline = time.monotonic() + self.execution_timeout.total_seconds()
257257

258258
job_run_info = JobRunInfo(account_id=self.account_id, run_id=self.run_id)
259259
job_run_status = self.hook.get_job_run_status(**job_run_info)
260260
if not DbtCloudJobRunStatus.is_terminal(job_run_status):
261261
self.defer(
262-
timeout=None,
262+
timeout=self.execution_timeout,
263263
trigger=DbtCloudRunJobTrigger(
264264
conn_id=self.dbt_cloud_conn_id,
265265
run_id=self.run_id,
@@ -293,8 +293,22 @@ def execute_complete(self, context: Context, event: dict[str, Any]) -> int:
293293

294294
# Enforce execution_timeout semantics in deferrable mode by cancelling the job.
295295
if event["status"] == "timeout":
296-
self.hook.cancel_job_run(account_id=self.account_id, run_id=self.run_id)
297-
raise AirflowException(f"Job run {self.run_id} has timed out.")
296+
if self.run_id is not None:
297+
self.log.info("Cancelling DBT job run %s due to execution timeout", self.run_id)
298+
299+
# Attempt best-effort job run cancellation.
300+
try:
301+
self.hook.cancel_job_run(account_id=self.account_id, run_id=self.run_id)
302+
except Exception:
303+
self.log.warning(
304+
"Failed to cancel DBT job run %s after timeout",
305+
self.run_id,
306+
exc_info=True,
307+
)
308+
else:
309+
self.log.warning("No run_id found; skipping cancellation")
310+
311+
raise DbtCloudJobRunException(f"Job run {self.run_id} has timed out.")
298312

299313
self.log.info(event["message"])
300314
return int(event["run_id"])
@@ -303,7 +317,15 @@ def on_kill(self) -> None:
303317
if not self.run_id:
304318
return
305319

306-
self.hook.cancel_job_run(account_id=self.account_id, run_id=self.run_id)
320+
# Attempt best-effort job run cancellation.
321+
try:
322+
self.hook.cancel_job_run(account_id=self.account_id, run_id=self.run_id)
323+
except Exception:
324+
self.log.warning(
325+
"Failed to cancel DBT job run %s during on_kill",
326+
self.run_id,
327+
exc_info=True,
328+
)
307329

308330
# Attempt best-effort confirmation of cancellation.
309331
try:

‎providers/dbt/cloud/src/airflow/providers/dbt/cloud/triggers/dbt.py‎

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,43 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
7979
"""Make async connection to Dbt, polls for the pipeline run status."""
8080
hook = DbtCloudHook(self.conn_id, **self.hook_params)
8181
try:
82-
while await self.is_still_running(hook):
82+
while True:
83+
now = time.monotonic()
84+
85+
job_run_status = await hook.get_job_status(self.run_id, self.account_id)
86+
87+
if job_run_status == DbtCloudJobRunStatus.SUCCESS.value:
88+
yield TriggerEvent(
89+
{
90+
"status": "success",
91+
"message": f"Job run {self.run_id} has completed successfully.",
92+
"run_id": self.run_id,
93+
}
94+
)
95+
return
96+
97+
elif job_run_status == DbtCloudJobRunStatus.CANCELLED.value:
98+
yield TriggerEvent(
99+
{
100+
"status": "cancelled",
101+
"message": f"Job run {self.run_id} has been cancelled.",
102+
"run_id": self.run_id,
103+
}
104+
)
105+
return
106+
107+
elif job_run_status == DbtCloudJobRunStatus.ERROR.value:
108+
yield TriggerEvent(
109+
{
110+
"status": "error",
111+
"message": f"Job run {self.run_id} has failed.",
112+
"run_id": self.run_id,
113+
}
114+
)
115+
return
116+
83117
if self.execution_deadline is not None:
84-
if self.execution_deadline < time.time():
118+
if self.execution_deadline <= now:
85119
yield TriggerEvent(
86120
{
87121
"status": "timeout",
@@ -91,11 +125,7 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
91125
)
92126
return
93127

94-
if self.end_time < time.time():
95-
# Perform a final status check before declaring timeout, in case the
96-
# job completed between the last poll and the timeout expiry.
97-
if not await self.is_still_running(hook):
98-
break
128+
if self.end_time <= now:
99129
yield TriggerEvent(
100130
{
101131
"status": "error",
@@ -105,34 +135,12 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
105135
}
106136
)
107137
return
138+
108139
await asyncio.sleep(self.poll_interval)
109-
job_run_status = await hook.get_job_status(self.run_id, self.account_id)
110-
if job_run_status == DbtCloudJobRunStatus.SUCCESS.value:
111-
yield TriggerEvent(
112-
{
113-
"status": "success",
114-
"message": f"Job run {self.run_id} has completed successfully.",
115-
"run_id": self.run_id,
116-
}
117-
)
118-
elif job_run_status == DbtCloudJobRunStatus.CANCELLED.value:
119-
yield TriggerEvent(
120-
{
121-
"status": "cancelled",
122-
"message": f"Job run {self.run_id} has been cancelled.",
123-
"run_id": self.run_id,
124-
}
125-
)
126-
else:
127-
yield TriggerEvent(
128-
{
129-
"status": "error",
130-
"message": f"Job run {self.run_id} has failed.",
131-
"run_id": self.run_id,
132-
}
133-
)
140+
134141
except Exception as e:
135142
yield TriggerEvent({"status": "error", "message": str(e), "run_id": self.run_id})
143+
return
136144

137145
async def is_still_running(self, hook: DbtCloudHook) -> bool:
138146
"""Check whether the submitted job is running."""

‎providers/dbt/cloud/tests/unit/dbt/cloud/operators/test_dbt.py‎

Lines changed: 130 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import pytest
2424

2525
from airflow.models import DAG, Connection
26-
from airflow.providers.common.compat.sdk import AirflowException, TaskDeferred, timezone
26+
from airflow.providers.common.compat.sdk import TaskDeferred, timezone
2727
from airflow.providers.dbt.cloud.hooks.dbt import DbtCloudHook, DbtCloudJobRunException, DbtCloudJobRunStatus
2828
from airflow.providers.dbt.cloud.operators.dbt import (
2929
DbtCloudGetJobRunArtifactOperator,
@@ -199,7 +199,7 @@ def test_execute_failed_before_getting_deferred(
199199
def test_dbt_run_job_op_async(self, mock_trigger_job_run, mock_dbt_hook, mock_job_run_status, status):
200200
"""
201201
Asserts that a task is deferred and an DbtCloudRunJobTrigger will be fired
202-
when the DbtCloudRunJobOperator has deferrable param set to True
202+
when the DbtCloudRunJobOperator has deferrable param set to True.
203203
"""
204204
mock_job_run_status.return_value = status
205205
dbt_op = DbtCloudRunJobOperator(
@@ -215,6 +215,40 @@ def test_dbt_run_job_op_async(self, mock_trigger_job_run, mock_dbt_hook, mock_jo
215215
dbt_op.execute(MagicMock())
216216
assert isinstance(exc.value.trigger, DbtCloudRunJobTrigger), "Trigger is not a DbtCloudRunJobTrigger"
217217

218+
def test_execute_complete_timeout_without_run_id(self):
219+
"""
220+
Verify that when a deferrable dbt job emits a timeout event with no run_id,
221+
the operator cancels the job and fails.
222+
"""
223+
224+
operator = DbtCloudRunJobOperator(
225+
task_id=TASK_ID,
226+
dbt_cloud_conn_id=ACCOUNT_ID_CONN,
227+
job_id=JOB_ID,
228+
dag=self.dag,
229+
deferrable=True,
230+
)
231+
232+
# Pretend the job was already triggered.
233+
operator.run_id = None
234+
235+
# Mock the hook so we can assert cancellation.
236+
operator.hook = MagicMock()
237+
238+
timeout_event = {
239+
"status": "timeout",
240+
"run_id": None,
241+
"message": "Job run timed out.",
242+
}
243+
244+
with pytest.raises(DbtCloudJobRunException):
245+
operator.execute_complete(
246+
context=self.mock_context,
247+
event=timeout_event,
248+
)
249+
250+
operator.hook.cancel_job_run.assert_not_called()
251+
218252
def test_execute_complete_timeout_cancels_job(self):
219253
"""
220254
Verify that when a deferrable dbt job emits a timeout event,
@@ -240,7 +274,45 @@ def test_execute_complete_timeout_cancels_job(self):
240274
"message": "Job run timed out.",
241275
}
242276

243-
with pytest.raises(AirflowException, match="has timed out"):
277+
with pytest.raises(DbtCloudJobRunException, match="has timed out"):
278+
operator.execute_complete(
279+
context=self.mock_context,
280+
event=timeout_event,
281+
)
282+
283+
operator.hook.cancel_job_run.assert_called_once_with(
284+
account_id=operator.account_id,
285+
run_id=RUN_ID,
286+
)
287+
288+
def test_execute_complete_timeout_cancel_job_does_not_mask_original_error(self):
289+
"""
290+
Verify that when a deferrable dbt job is cancelled after a timeout event is received,
291+
the original error is not masked.
292+
"""
293+
operator = DbtCloudRunJobOperator(
294+
task_id=TASK_ID,
295+
dbt_cloud_conn_id=ACCOUNT_ID_CONN,
296+
job_id=JOB_ID,
297+
dag=self.dag,
298+
deferrable=True,
299+
)
300+
301+
# Pretend the job was already triggered.
302+
operator.run_id = RUN_ID
303+
304+
# Mock the hook so we can assert cancellation.
305+
operator.hook = MagicMock()
306+
307+
operator.hook.cancel_job_run.side_effect = Exception("Cancellation failed")
308+
309+
timeout_event = {
310+
"status": "timeout",
311+
"run_id": RUN_ID,
312+
"message": "Job run timed out.",
313+
}
314+
315+
with pytest.raises(DbtCloudJobRunException, match="has timed out"):
244316
operator.execute_complete(
245317
context=self.mock_context,
246318
event=timeout_event,
@@ -690,6 +762,61 @@ def test_custom_trigger_reason(self, mock_run_job, conn_id, account_id):
690762
additional_run_config=self.config["additional_run_config"],
691763
)
692764

765+
def test_on_kill_cancels_job_and_confirms_success(self):
766+
operator = DbtCloudRunJobOperator(
767+
task_id=TASK_ID,
768+
dbt_cloud_conn_id=ACCOUNT_ID_CONN,
769+
job_id=JOB_ID,
770+
dag=self.dag,
771+
)
772+
773+
operator.run_id = RUN_ID
774+
operator.hook = MagicMock()
775+
776+
# Simulate successful cancellation confirmation.
777+
operator.hook.wait_for_job_run_status.return_value = True
778+
779+
operator.on_kill()
780+
781+
operator.hook.cancel_job_run.assert_called_once_with(
782+
account_id=operator.account_id,
783+
run_id=RUN_ID,
784+
)
785+
786+
operator.hook.wait_for_job_run_status.assert_called_once_with(
787+
run_id=RUN_ID,
788+
account_id=operator.account_id,
789+
expected_statuses=DbtCloudJobRunStatus.CANCELLED.value,
790+
check_interval=operator.check_interval,
791+
timeout=operator.timeout,
792+
)
793+
794+
def test_on_kill_best_effort_cancellation_does_not_raise(self):
795+
operator = DbtCloudRunJobOperator(
796+
task_id=TASK_ID,
797+
dbt_cloud_conn_id=ACCOUNT_ID_CONN,
798+
job_id=JOB_ID,
799+
dag=self.dag,
800+
)
801+
802+
operator.run_id = RUN_ID
803+
operator.hook = MagicMock()
804+
805+
# Simulate cancellation failure.
806+
operator.hook.cancel_job_run.side_effect = Exception("Cancellation failed")
807+
808+
# Simulate confirmation also failing (normal path).
809+
operator.hook.wait_for_job_run_status.side_effect = DbtCloudJobRunException("Still running")
810+
811+
operator.on_kill()
812+
813+
operator.hook.cancel_job_run.assert_called_once_with(
814+
account_id=operator.account_id,
815+
run_id=RUN_ID,
816+
)
817+
818+
operator.hook.wait_for_job_run_status.assert_called_once()
819+
693820
@pytest.mark.parametrize(
694821
("conn_id", "account_id"),
695822
[(ACCOUNT_ID_CONN, None), (NO_ACCOUNT_ID_CONN, ACCOUNT_ID)],

0 commit comments

Comments
 (0)