Skip to content

Commit 9dcce2f

Browse files
Fixing DatabricksNotebookOperator invalid dependency graph issue (#48492)
1 parent 0ecc99f commit 9dcce2f

4 files changed

Lines changed: 53 additions & 30 deletions

File tree

‎providers/databricks/src/airflow/providers/databricks/operators/databricks.py‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1261,17 +1261,31 @@ def _get_hook(self, caller: str) -> DatabricksHook:
12611261
def databricks_task_key(self) -> str:
12621262
return self._generate_databricks_task_key()
12631263

1264-
def _generate_databricks_task_key(self, task_id: str | None = None) -> str:
1264+
def _generate_databricks_task_key(
1265+
self, task_id: str | None = None, task_dict: dict[str, BaseOperator] | None = None
1266+
) -> str:
12651267
"""Create a databricks task key using the hash of dag_id and task_id."""
1266-
if not self._databricks_task_key or len(self._databricks_task_key) > 100:
1267-
self.log.info(
1268-
"databricks_task_key has not be provided or the provided one exceeds 100 characters and will be truncated by the Databricks API. This will cause failure when trying to monitor the task. A task_key will be generated using the hash value of dag_id+task_id"
1269-
)
1270-
task_id = task_id or self.task_id
1271-
task_key = f"{self.dag_id}__{task_id}".encode()
1272-
self._databricks_task_key = hashlib.md5(task_key).hexdigest()
1273-
self.log.info("Generated databricks task_key: %s", self._databricks_task_key)
1274-
return self._databricks_task_key
1268+
if task_id:
1269+
if not task_dict:
1270+
raise ValueError(
1271+
"Must pass task_dict if task_id is provided in _generate_databricks_task_key."
1272+
)
1273+
_task = task_dict.get(task_id)
1274+
if _task and hasattr(_task, "databricks_task_key"):
1275+
_databricks_task_key = _task.databricks_task_key
1276+
else:
1277+
task_key = f"{self.dag_id}__{task_id}".encode()
1278+
_databricks_task_key = hashlib.md5(task_key).hexdigest()
1279+
return _databricks_task_key
1280+
else:
1281+
if not self._databricks_task_key or len(self._databricks_task_key) > 100:
1282+
self.log.info(
1283+
"databricks_task_key has not be provided or the provided one exceeds 100 characters and will be truncated by the Databricks API. This will cause failure when trying to monitor the task. A task_key will be generated using the hash value of dag_id+task_id"
1284+
)
1285+
task_key = f"{self.dag_id}__{self.task_id}".encode()
1286+
self._databricks_task_key = hashlib.md5(task_key).hexdigest()
1287+
self.log.info("Generated databricks task_key: %s", self._databricks_task_key)
1288+
return self._databricks_task_key
12751289

12761290
@property
12771291
def _databricks_workflow_task_group(self) -> DatabricksWorkflowTaskGroup | None:
@@ -1354,14 +1368,17 @@ def _get_current_databricks_task(self) -> dict[str, Any]:
13541368
return {task["task_key"]: task for task in sorted_task_runs}[self.databricks_task_key]
13551369

13561370
def _convert_to_databricks_workflow_task(
1357-
self, relevant_upstreams: list[BaseOperator], context: Context | None = None
1371+
self,
1372+
relevant_upstreams: list[BaseOperator],
1373+
task_dict: dict[str, BaseOperator],
1374+
context: Context | None = None,
13581375
) -> dict[str, object]:
13591376
"""Convert the operator to a Databricks workflow task that can be a task in a workflow."""
13601377
base_task_json = self._get_task_base_json()
13611378
result = {
13621379
"task_key": self.databricks_task_key,
13631380
"depends_on": [
1364-
{"task_key": self._generate_databricks_task_key(task_id)}
1381+
{"task_key": self._generate_databricks_task_key(task_id, task_dict)}
13651382
for task_id in self.upstream_task_ids
13661383
if task_id in relevant_upstreams
13671384
],
@@ -1571,7 +1588,10 @@ def _extend_workflow_notebook_packages(
15711588
self.notebook_packages.append(task_group_package)
15721589

15731590
def _convert_to_databricks_workflow_task(
1574-
self, relevant_upstreams: list[BaseOperator], context: Context | None = None
1591+
self,
1592+
relevant_upstreams: list[BaseOperator],
1593+
task_dict: dict[str, BaseOperator],
1594+
context: Context | None = None,
15751595
) -> dict[str, object]:
15761596
"""Convert the operator to a Databricks workflow task that can be a task in a workflow."""
15771597
databricks_workflow_task_group = self._databricks_workflow_task_group
@@ -1589,7 +1609,7 @@ def _convert_to_databricks_workflow_task(
15891609
**databricks_workflow_task_group.notebook_params,
15901610
}
15911611

1592-
return super()._convert_to_databricks_workflow_task(relevant_upstreams, context=context)
1612+
return super()._convert_to_databricks_workflow_task(relevant_upstreams, task_dict, context=context)
15931613

15941614

15951615
class DatabricksTaskOperator(DatabricksTaskBaseOperator):

‎providers/databricks/src/airflow/providers/databricks/operators/databricks_workflow.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ class _CreateDatabricksWorkflowOperator(BaseOperator):
8888
:param max_concurrent_runs: The maximum number of concurrent runs for the workflow.
8989
:param notebook_params: A dictionary of notebook parameters to pass to the workflow. These parameters
9090
will be passed to all notebooks in the workflow.
91-
:param tasks_to_convert: A list of tasks to convert to a Databricks workflow. This list can also be
91+
:param tasks_to_convert: A dict of tasks to convert to a Databricks workflow. This list can also be
9292
populated after instantiation using the `add_task` method.
9393
"""
9494

@@ -105,7 +105,7 @@ def __init__(
105105
job_clusters: list[dict[str, object]] | None = None,
106106
max_concurrent_runs: int = 1,
107107
notebook_params: dict | None = None,
108-
tasks_to_convert: list[BaseOperator] | None = None,
108+
tasks_to_convert: dict[str, BaseOperator] | None = None,
109109
**kwargs,
110110
):
111111
self.databricks_conn_id = databricks_conn_id
@@ -114,7 +114,7 @@ def __init__(
114114
self.job_clusters = job_clusters or []
115115
self.max_concurrent_runs = max_concurrent_runs
116116
self.notebook_params = notebook_params or {}
117-
self.tasks_to_convert = tasks_to_convert or []
117+
self.tasks_to_convert = tasks_to_convert or {}
118118
self.relevant_upstreams = [task_id]
119119
self.workflow_run_metadata: WorkflowRunMetadata | None = None
120120
super().__init__(task_id=task_id, **kwargs)
@@ -129,9 +129,9 @@ def _get_hook(self, caller: str) -> DatabricksHook:
129129
def _hook(self) -> DatabricksHook:
130130
return self._get_hook(caller=self.caller)
131131

132-
def add_task(self, task: BaseOperator) -> None:
133-
"""Add a task to the list of tasks to convert to a Databricks workflow."""
134-
self.tasks_to_convert.append(task)
132+
def add_task(self, task_id, task: BaseOperator) -> None:
133+
"""Add a task to the dict of tasks to convert to a Databricks workflow."""
134+
self.tasks_to_convert[task_id] = task
135135

136136
@property
137137
def job_name(self) -> str:
@@ -143,9 +143,9 @@ def create_workflow_json(self, context: Context | None = None) -> dict[str, obje
143143
"""Create a workflow json to be used in the Databricks API."""
144144
task_json = [
145145
task._convert_to_databricks_workflow_task( # type: ignore[attr-defined]
146-
relevant_upstreams=self.relevant_upstreams, context=context
146+
relevant_upstreams=self.relevant_upstreams, task_dict=self.tasks_to_convert, context=context
147147
)
148-
for task in self.tasks_to_convert
148+
for task_id, task in self.tasks_to_convert.items()
149149
]
150150

151151
default_json = {
@@ -334,7 +334,7 @@ def __exit__(
334334

335335
task.workflow_run_metadata = create_databricks_workflow_task.output
336336
create_databricks_workflow_task.relevant_upstreams.append(task.task_id)
337-
create_databricks_workflow_task.add_task(task)
337+
create_databricks_workflow_task.add_task(task.task_id, task)
338338

339339
for root_task in roots:
340340
root_task.set_upstream(create_databricks_workflow_task)

‎providers/databricks/tests/unit/databricks/operators/test_databricks.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2385,8 +2385,9 @@ def test_convert_to_databricks_workflow_task(self):
23852385
operator.task_id = "test_task"
23862386
operator.upstream_task_ids = ["upstream_task"]
23872387
relevant_upstreams = [MagicMock(task_id="upstream_task")]
2388+
task_dict = {"upstream_task": MagicMock(task_id="upstream_task")}
23882389

2389-
task_json = operator._convert_to_databricks_workflow_task(relevant_upstreams)
2390+
task_json = operator._convert_to_databricks_workflow_task(relevant_upstreams, task_dict)
23902391

23912392
task_key = hashlib.md5(b"example_dag__test_task").hexdigest()
23922393
expected_json = {
@@ -2423,12 +2424,13 @@ def test_convert_to_databricks_workflow_task_no_task_group(self):
24232424
)
24242425
operator.task_group = None
24252426
relevant_upstreams = [MagicMock(task_id="upstream_task")]
2427+
task_dict = {"upstream_task": MagicMock(task_id="upstream_task")}
24262428

24272429
with pytest.raises(
24282430
AirflowException,
24292431
match="Calling `_convert_to_databricks_workflow_task` without a parent TaskGroup.",
24302432
):
2431-
operator._convert_to_databricks_workflow_task(relevant_upstreams)
2433+
operator._convert_to_databricks_workflow_task(relevant_upstreams, task_dict)
24322434

24332435
def test_convert_to_databricks_workflow_task_cluster_conflict(self):
24342436
"""Test that an error is raised if both `existing_cluster_id` and `job_cluster_key` are set."""
@@ -2446,12 +2448,13 @@ def test_convert_to_databricks_workflow_task_cluster_conflict(self):
24462448
operator.job_cluster_key = "job-cluster-key"
24472449
operator.task_group = databricks_workflow_task_group
24482450
relevant_upstreams = [MagicMock(task_id="upstream_task")]
2451+
task_dict = {"upstream_task": MagicMock(task_id="upstream_task")}
24492452

24502453
with pytest.raises(
24512454
ValueError,
24522455
match="Both existing_cluster_id and job_cluster_key are set. Only one can be set per task.",
24532456
):
2454-
operator._convert_to_databricks_workflow_task(relevant_upstreams)
2457+
operator._convert_to_databricks_workflow_task(relevant_upstreams, task_dict)
24552458

24562459

24572460
class TestDatabricksTaskOperator:

‎providers/databricks/tests/unit/databricks/operators/test_databricks_workflow.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,9 @@ def test_create_workflow_json(mock_databricks_hook, context, mock_task_group):
8383
)
8484
operator.task_group = mock_task_group
8585

86-
task = MagicMock(spec=BaseOperator)
86+
task = MagicMock(spec=BaseOperator, task_id="task_1")
8787
task._convert_to_databricks_workflow_task = MagicMock(return_value={})
88-
operator.add_task(task)
88+
operator.add_task(task.task_id, task)
8989

9090
workflow_json = operator.create_workflow_json(context=context)
9191

@@ -150,9 +150,9 @@ def test_execute(mock_databricks_hook, context, mock_task_group):
150150
life_cycle_state=RunLifeCycleState.RUNNING.value
151151
)
152152

153-
task = MagicMock(spec=BaseOperator)
153+
task = MagicMock(spec=BaseOperator, task_id="task_1")
154154
task._convert_to_databricks_workflow_task = MagicMock(return_value={})
155-
operator.add_task(task)
155+
operator.add_task(task.task_id, task)
156156

157157
result = operator.execute(context)
158158

0 commit comments

Comments
 (0)