@@ -1261,17 +1261,31 @@ def _get_hook(self, caller: str) -> DatabricksHook:
12611261 def databricks_task_key (self ) -> str :
12621262 return self ._generate_databricks_task_key ()
12631263
1264- def _generate_databricks_task_key (self , task_id : str | None = None ) -> str :
1264+ def _generate_databricks_task_key (
1265+ self , task_id : str | None = None , task_dict : dict [str , BaseOperator ] | None = None
1266+ ) -> str :
12651267 """Create a databricks task key using the hash of dag_id and task_id."""
1266- if not self ._databricks_task_key or len (self ._databricks_task_key ) > 100 :
1267- self .log .info (
1268- "databricks_task_key has not be provided or the provided one exceeds 100 characters and will be truncated by the Databricks API. This will cause failure when trying to monitor the task. A task_key will be generated using the hash value of dag_id+task_id"
1269- )
1270- task_id = task_id or self .task_id
1271- task_key = f"{ self .dag_id } __{ task_id } " .encode ()
1272- self ._databricks_task_key = hashlib .md5 (task_key ).hexdigest ()
1273- self .log .info ("Generated databricks task_key: %s" , self ._databricks_task_key )
1274- return self ._databricks_task_key
1268+ if task_id :
1269+ if not task_dict :
1270+ raise ValueError (
1271+ "Must pass task_dict if task_id is provided in _generate_databricks_task_key."
1272+ )
1273+ _task = task_dict .get (task_id )
1274+ if _task and hasattr (_task , "databricks_task_key" ):
1275+ _databricks_task_key = _task .databricks_task_key
1276+ else :
1277+ task_key = f"{ self .dag_id } __{ task_id } " .encode ()
1278+ _databricks_task_key = hashlib .md5 (task_key ).hexdigest ()
1279+ return _databricks_task_key
1280+ else :
1281+ if not self ._databricks_task_key or len (self ._databricks_task_key ) > 100 :
1282+ self .log .info (
1283+ "databricks_task_key has not be provided or the provided one exceeds 100 characters and will be truncated by the Databricks API. This will cause failure when trying to monitor the task. A task_key will be generated using the hash value of dag_id+task_id"
1284+ )
1285+ task_key = f"{ self .dag_id } __{ self .task_id } " .encode ()
1286+ self ._databricks_task_key = hashlib .md5 (task_key ).hexdigest ()
1287+ self .log .info ("Generated databricks task_key: %s" , self ._databricks_task_key )
1288+ return self ._databricks_task_key
12751289
12761290 @property
12771291 def _databricks_workflow_task_group (self ) -> DatabricksWorkflowTaskGroup | None :
@@ -1354,14 +1368,17 @@ def _get_current_databricks_task(self) -> dict[str, Any]:
13541368 return {task ["task_key" ]: task for task in sorted_task_runs }[self .databricks_task_key ]
13551369
13561370 def _convert_to_databricks_workflow_task (
1357- self , relevant_upstreams : list [BaseOperator ], context : Context | None = None
1371+ self ,
1372+ relevant_upstreams : list [BaseOperator ],
1373+ task_dict : dict [str , BaseOperator ],
1374+ context : Context | None = None ,
13581375 ) -> dict [str , object ]:
13591376 """Convert the operator to a Databricks workflow task that can be a task in a workflow."""
13601377 base_task_json = self ._get_task_base_json ()
13611378 result = {
13621379 "task_key" : self .databricks_task_key ,
13631380 "depends_on" : [
1364- {"task_key" : self ._generate_databricks_task_key (task_id )}
1381+ {"task_key" : self ._generate_databricks_task_key (task_id , task_dict )}
13651382 for task_id in self .upstream_task_ids
13661383 if task_id in relevant_upstreams
13671384 ],
@@ -1571,7 +1588,10 @@ def _extend_workflow_notebook_packages(
15711588 self .notebook_packages .append (task_group_package )
15721589
15731590 def _convert_to_databricks_workflow_task (
1574- self , relevant_upstreams : list [BaseOperator ], context : Context | None = None
1591+ self ,
1592+ relevant_upstreams : list [BaseOperator ],
1593+ task_dict : dict [str , BaseOperator ],
1594+ context : Context | None = None ,
15751595 ) -> dict [str , object ]:
15761596 """Convert the operator to a Databricks workflow task that can be a task in a workflow."""
15771597 databricks_workflow_task_group = self ._databricks_workflow_task_group
@@ -1589,7 +1609,7 @@ def _convert_to_databricks_workflow_task(
15891609 ** databricks_workflow_task_group .notebook_params ,
15901610 }
15911611
1592- return super ()._convert_to_databricks_workflow_task (relevant_upstreams , context = context )
1612+ return super ()._convert_to_databricks_workflow_task (relevant_upstreams , task_dict , context = context )
15931613
15941614
15951615class DatabricksTaskOperator (DatabricksTaskBaseOperator ):
0 commit comments