4545)
4646from airflow .providers .common .compat .sdk import AirflowException , conf
4747
48- # ResumableJobMixin only exists on Airflow 3; this provider still targets >=2.11. Drop this
49- # fallback once the provider's minimum Airflow version is >=3.0 .
48+ # ResumableJobMixin only exists on Airflow 3.3+ ; this provider still targets >=2.11. Drop this
49+ # fallback once the provider's minimum Airflow version is >=3.3 .
5050try :
5151 from airflow .sdk import ResumableJobMixin
5252except ImportError :
@@ -129,11 +129,11 @@ class GlueJobOperator(ResumableJobMixin, AwsBaseOperator[GlueJobHook]):
129129 :param waiter_delay: Time in seconds to wait between status checks. (default: 60)
130130 :param waiter_max_attempts: Maximum number of attempts to check for job completion. (default: 20)
131131 :param resume_glue_job_on_retry: deprecated, use ``durable`` instead.
132- :param durable: When ``True`` (the default) , the Glue job run id is persisted to task state
133- before polling begins. A worker crash on retry reconnects to the existing run instead of
134- starting a duplicate. Set to ``False `` to always start a fresh run on retry.
135- Requires Airflow 3.3+ for the persisted lookup; on earlier versions the run is recovered by
136- searching job runs for the task's ``--airflow_task_uuid`` argument.
132+ :param durable: When ``True``, the Glue job run id is persisted to task state before polling
133+ begins. A worker crash on retry reconnects to the existing run instead of starting a
134+ duplicate. Defaults to ``True `` on Airflow 3.3+, which uses task state store for the
135+ persisted lookup; on earlier versions it defaults to ``False`` and, if set explicitly,
136+ recovers the run by searching job runs for the task's ``--airflow_task_uuid`` argument.
137137 :param aws_conn_id: The Airflow connection used for AWS credentials.
138138 If this is ``None`` or empty then the default boto3 behaviour is used. If
139139 running Airflow in a distributed manner and aws_conn_id is None or
@@ -195,6 +195,7 @@ def __init__(
195195 waiter_delay : int = 60 ,
196196 waiter_max_attempts : int = 75 ,
197197 resume_glue_job_on_retry : bool | None = None ,
198+ durable : bool | None = None ,
198199 openlineage_inject_parent_job_info : bool = conf .getboolean (
199200 "openlineage" , "spark_inject_parent_job_info" , fallback = False
200201 ),
@@ -205,14 +206,21 @@ def __init__(
205206 ):
206207 if resume_glue_job_on_retry is not None :
207208 # Kept as a real named parameter (not **kwargs) so `default_args` still applies correctly.
208- if AIRFLOW_V_3_3_PLUS :
209- warnings .warn (
210- "`resume_glue_job_on_retry` is deprecated and will be removed in a future release. "
211- "Use `durable` instead." ,
212- AirflowProviderDeprecationWarning ,
213- stacklevel = 2 ,
214- )
209+ warnings .warn (
210+ "`resume_glue_job_on_retry` is deprecated and will be removed in a future release. "
211+ "Use `durable` instead." ,
212+ AirflowProviderDeprecationWarning ,
213+ stacklevel = 2 ,
214+ )
215215 kwargs .setdefault ("durable" , resume_glue_job_on_retry )
216+ # durable is also named here (not left to **kwargs) so default_args={"durable": ...} reaches
217+ # it on every supported Airflow version, not just where the mixin's own _apply_defaults runs.
218+ if durable is not None :
219+ kwargs ["durable" ] = durable
220+ elif resume_glue_job_on_retry is None and not AIRFLOW_V_3_3_PLUS :
221+ # No task_state_store below 3.3 -- durable's default of True would otherwise turn on
222+ # the scan-based reattach for everyone who upgrades without touching their DAGs.
223+ kwargs ["durable" ] = False
216224 super ().__init__ (** kwargs )
217225 self .job_name = job_name
218226 self .job_desc = job_desc
@@ -304,6 +312,8 @@ def _prepare_script_args_with_task_uuid(
304312 return script_args , task_uuid
305313
306314 def _find_job_run_id_by_task_uuid (self , task_uuid : str ) -> tuple [str , str ] | None :
315+ # Unbounded walk with no page cap; a no-match run is the common retry shape and pays the
316+ # full scan. Tracked at https://github.com/apache/airflow/issues/71489.
307317 next_token : str | None = None
308318 while True :
309319 request = {"JobName" : self .job_name , "MaxResults" : 50 }
@@ -369,6 +379,7 @@ def _set_job_run_id(self, context: Context, job_run_id: str) -> None:
369379 if self ._job_run_id == job_run_id :
370380 return
371381 self ._job_run_id = job_run_id
382+ context ["ti" ].xcom_push (key = "glue_job_run_id" , value = job_run_id )
372383 GlueJobRunDetailsLink .persist (
373384 context = context ,
374385 operator = self ,
@@ -395,7 +406,10 @@ def _build_script_args(self, context: Context) -> dict:
395406 if self .openlineage_inject_transport_info :
396407 self .log .info ("Injecting OpenLineage transport information into Glue job arguments." )
397408 script_args = inject_transport_information_into_glue_arguments (script_args , context )
398- if self .durable :
409+ # Skipped for a synchronous run on 3.3+: task_state_store is the primary reconnect
410+ # mechanism there, so tagging every script's args is not worth it just for the narrow
411+ # crash-before-persist window this tag would otherwise cover.
412+ if self .durable and (self .deferrable or not AIRFLOW_V_3_3_PLUS ):
399413 script_args , _ = self ._prepare_script_args_with_task_uuid (context , base_args = script_args )
400414 return script_args
401415
@@ -404,8 +418,8 @@ def _find_previous_job_run(self, context: Context, task_uuid: str) -> str | None
404418 Look for a Glue job run this task instance already started.
405419
406420 Checks XCom for a cached run id first, then falls back to a task-UUID scan. The XCom tier
407- only works on Airflow 2.11-3.2; Airflow 3.3+ clears task XComs before every non-deferral
408- attempt, so it always misses there and the scan runs every time.
421+ only works on Airflow 2.x; every Airflow 3 release clears task XComs before each
422+ non-deferral attempt, so it always misses there and the scan runs every time.
409423 """
410424 ti = context ["ti" ]
411425 previous_job_run_id = ti .xcom_pull (key = "glue_job_run_id" , task_ids = ti .task_id )
@@ -414,7 +428,7 @@ def _find_previous_job_run(self, context: Context, task_uuid: str) -> str | None
414428 job_run = self .hook .conn .get_job_run (JobName = self .job_name , RunId = previous_job_run_id )
415429 state = job_run .get ("JobRun" , {}).get ("JobRunState" )
416430 self .log .info ("Previous Glue job_run_id: %s, state: %s" , previous_job_run_id , state )
417- if state in ( "RUNNING" , "STARTING" ):
431+ if self . is_job_active ( state ):
418432 return previous_job_run_id
419433 except Exception :
420434 self .log .warning ("Failed to get previous Glue job run state" , exc_info = True )
@@ -428,8 +442,7 @@ def _find_previous_job_run(self, context: Context, task_uuid: str) -> str | None
428442 existing_job_run_id ,
429443 existing_job_run_state ,
430444 )
431- if existing_job_run_state in ("RUNNING" , "STARTING" ):
432- ti .xcom_push (key = "glue_job_run_id" , value = existing_job_run_id )
445+ if self .is_job_active (existing_job_run_state ):
433446 return existing_job_run_id
434447 except Exception :
435448 self .log .warning ("Failed to find previous Glue job run by task UUID" , exc_info = True )
@@ -443,8 +456,17 @@ def submit_job(self, context: Context) -> str:
443456 """Start a Glue job run and return its run id, or reconnect to one this task already started."""
444457 script_args = self ._build_script_args (context )
445458 # Scan only when there's nothing else to go on: first attempt, no store, or a store that
446- # never recorded this key. A stored id (even terminal) means the caller already decided.
447- if self .durable and context ["ti" ].try_number > 1 and not self ._has_stored_external_id (context ):
459+ # never recorded this key. The store check itself is skipped on the deferrable path, where
460+ # nothing is ever written to it -- a store error there would otherwise block the scan.
461+ # Deferrable retries could reconnect via task_state_store too, avoiding this scan entirely,
462+ # if ResumableJobMixin exposed its reconnect decision apart from its polling loop;
463+ # tracked at https://github.com/apache/airflow/issues/71485.
464+ if (
465+ self .durable
466+ and (self .deferrable or not AIRFLOW_V_3_3_PLUS )
467+ and context ["ti" ].try_number > 1
468+ and (self .deferrable or not self ._has_stored_external_id (context ))
469+ ):
448470 existing_job_run_id = self ._find_previous_job_run (context , script_args [self .TASK_UUID_ARG ])
449471 if existing_job_run_id :
450472 self ._set_job_run_id (context , existing_job_run_id )
@@ -460,8 +482,6 @@ def submit_job(self, context: Context) -> str:
460482 glue_job_run = self .hook .initialize_job (script_args , self .run_job_kwargs )
461483 # Set before polling so on_kill can stop the run even if the worker dies immediately after.
462484 self ._set_job_run_id (context , glue_job_run ["JobRunId" ])
463- # Downstream tasks read this key directly; it's also what feeds the XCom tier above.
464- context ["ti" ].xcom_push (key = "glue_job_run_id" , value = glue_job_run ["JobRunId" ])
465485 return glue_job_run ["JobRunId" ]
466486
467487 def get_job_status (self , external_id : JsonValue , context : Context ) -> str :
0 commit comments