Skip to content

Commit 6c0652f

Browse files
Narrow exception handling in Glue job run fallback scan
Keep while True for the task-UUID pagination scan since the natural NextToken-based termination already handles exit. Narrowing except Exception to except ClientError prevents non-AWS errors from being silently swallowed. closes: apache#71489
1 parent de839fe commit 6c0652f

2 files changed

Lines changed: 12 additions & 8 deletions

File tree

  • providers/amazon

‎providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ def _prepare_script_args_with_task_uuid(
330330

331331
def _find_job_run_id_by_task_uuid(self, task_uuid: str) -> tuple[str, str] | None:
332332
next_token: str | None = None
333-
for _ in range(100):
333+
while True:
334334
request = {"JobName": self.job_name, "MaxResults": 50}
335335
if next_token:
336336
request["NextToken"] = next_token
@@ -345,7 +345,6 @@ def _find_job_run_id_by_task_uuid(self, task_uuid: str) -> tuple[str, str] | Non
345345
next_token = response.get("NextToken")
346346
if not next_token:
347347
return None
348-
return None
349348

350349
def execute(self, context: Context) -> str | None:
351350
"""

‎providers/amazon/tests/unit/amazon/aws/operators/test_glue.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1353,18 +1353,23 @@ def test_find_previous_job_run_fallback_catches_client_error(self, mock_log):
13531353

13541354
mock_log.exception.assert_called_with("Failed to find previous Glue job run by task UUID")
13551355

1356-
def test_find_job_run_id_by_task_uuid_bounds_pagination(self):
1356+
def test_find_job_run_id_by_task_uuid_exhausts_pages(self):
13571357
glue = self._build()
13581358
glue.hook.conn = mock.MagicMock()
1359-
glue.hook.conn.get_job_runs.return_value = {
1360-
"JobRuns": [{"JobRunId": f"jr_{i}", "Arguments": {}} for i in range(50)],
1361-
"NextToken": "continue"
1362-
}
1359+
glue.hook.conn.get_job_runs.side_effect = [
1360+
{
1361+
"JobRuns": [{"JobRunId": "jr_1", "Arguments": {}}],
1362+
"NextToken": "page2",
1363+
},
1364+
{
1365+
"JobRuns": [{"JobRunId": "jr_2", "Arguments": {}}],
1366+
},
1367+
]
13631368

13641369
result = glue._find_job_run_id_by_task_uuid("test_task_uuid")
13651370

13661371
assert result is None
1367-
assert glue.hook.conn.get_job_runs.call_count == 100
1372+
assert glue.hook.conn.get_job_runs.call_count == 2
13681373

13691374

13701375
class TestGlueDataQualityOperator:

0 commit comments

Comments
 (0)