Skip to content

Commit a4f2b33

Browse files
authored
Fix Google Cloud Data Fusion hook to handle pipeline start errors properly (#58698)
* Fix Google Cloud Data Fusion hook to handle pipeline start errors properly The start_pipeline method was using the multi-program start endpoint which returns HTTP 200 even when individual programs fail to start. This caused a KeyError when trying to access the runId from error responses. Changes: - Updated start_pipeline to use single-program start endpoint - Added validation to check if runId exists in response before accessing it - Improved error messages to provide context about failures - Updated tests to reflect new endpoint and added test for missing runId scenario Fixes #50387 * Add spec to MagicMock for better static type checking
1 parent 9ea7692 commit a4f2b33

2 files changed

Lines changed: 66 additions & 58 deletions

File tree

‎providers/google/src/airflow/providers/google/cloud/hooks/datafusion.py‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -469,31 +469,33 @@ def start_pipeline(
469469
is always default. If your pipeline belongs to an Enterprise edition instance, you
470470
can create a namespace.
471471
"""
472-
# TODO: This API endpoint starts multiple pipelines. There will eventually be a fix
473-
# return the run Id as part of the API request to run a single pipeline.
474-
# https://github.com/apache/airflow/pull/8954#discussion_r438223116
472+
# Use the single-program start endpoint for better error handling
473+
# https://cdap.atlassian.net/wiki/spaces/DOCS/pages/477560983/Lifecycle+Microservices#Start-a-Program
474+
program_type = self.cdap_program_type(pipeline_type=pipeline_type)
475+
program_id = self.cdap_program_id(pipeline_type=pipeline_type)
475476
url = os.path.join(
476-
instance_url,
477-
"v3",
478-
"namespaces",
479-
quote(namespace),
477+
self._base_url(instance_url, namespace),
478+
quote(pipeline_name),
479+
f"{program_type}s",
480+
program_id,
480481
"start",
481482
)
482483
runtime_args = runtime_args or {}
483-
body = [
484-
{
485-
"appId": pipeline_name,
486-
"runtimeargs": runtime_args,
487-
"programType": self.cdap_program_type(pipeline_type=pipeline_type),
488-
"programId": self.cdap_program_id(pipeline_type=pipeline_type),
489-
}
490-
]
491-
response = self._cdap_request(url=url, method="POST", body=body)
484+
response = self._cdap_request(url=url, method="POST", body=runtime_args)
492485
self._check_response_status_and_data(
493486
response, f"Starting a pipeline failed with code {response.status}"
494487
)
495488
response_json = json.loads(response.data)
496-
return response_json[0]["runId"]
489+
490+
# Extract and validate runId from response
491+
if "runId" not in response_json:
492+
error_message = response_json.get("error", "Unknown error")
493+
raise AirflowException(
494+
f"Failed to start pipeline '{pipeline_name}'. "
495+
f"The response does not contain a runId. Error: {error_message}"
496+
)
497+
498+
return str(response_json["runId"])
497499

498500
def stop_pipeline(self, pipeline_name: str, instance_url: str, namespace: str = "default") -> None:
499501
"""

‎providers/google/tests/unit/google/cloud/hooks/test_datafusion.py‎

Lines changed: 47 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from unittest import mock
2222

2323
import aiohttp
24+
import google.auth.transport
2425
import pytest
2526
from aiohttp.helpers import TimerNoop
2627
from yarl import URL
@@ -340,42 +341,38 @@ def test_list_pipelines_should_fail_if_status_not_200(self, mock_request, hook):
340341
@mock.patch(HOOK_STR.format("DataFusionHook._cdap_request"))
341342
def test_start_pipeline(self, mock_request, hook):
342343
run_id = 1234
343-
mock_request.return_value = mock.MagicMock(status=200, data=f'[{{"runId":{run_id}}}]')
344+
mock_request.return_value = mock.MagicMock(
345+
spec=google.auth.transport.Response, status=200, data=f'{{"runId":{run_id}}}'
346+
)
344347

345-
hook.start_pipeline(pipeline_name=PIPELINE_NAME, instance_url=INSTANCE_URL, runtime_args=RUNTIME_ARGS)
346-
body = [
347-
{
348-
"appId": PIPELINE_NAME,
349-
"programType": "workflow",
350-
"programId": "DataPipelineWorkflow",
351-
"runtimeargs": RUNTIME_ARGS,
352-
}
353-
]
348+
result = hook.start_pipeline(
349+
pipeline_name=PIPELINE_NAME, instance_url=INSTANCE_URL, runtime_args=RUNTIME_ARGS
350+
)
351+
assert result == str(run_id)
354352
mock_request.assert_called_once_with(
355-
url=f"{INSTANCE_URL}/v3/namespaces/default/start", method="POST", body=body
353+
url=f"{INSTANCE_URL}/v3/namespaces/default/apps/{PIPELINE_NAME}/workflows/DataPipelineWorkflow/start",
354+
method="POST",
355+
body=RUNTIME_ARGS,
356356
)
357357

358358
@mock.patch(HOOK_STR.format("DataFusionHook._cdap_request"))
359359
def test_start_pipeline_stream(self, mock_request, hook):
360-
run_id = 1234
361-
mock_request.return_value = mock.MagicMock(status=200, data=f'[{{"runId":{run_id}}}]')
360+
run_id = "test-run-123"
361+
mock_request.return_value = mock.MagicMock(
362+
spec=google.auth.transport.Response, status=200, data=f'{{"runId":"{run_id}"}}'
363+
)
362364

363-
hook.start_pipeline(
365+
result = hook.start_pipeline(
364366
pipeline_name=PIPELINE_NAME,
365367
instance_url=INSTANCE_URL,
366368
runtime_args=RUNTIME_ARGS,
367369
pipeline_type=DataFusionPipelineType.STREAM,
368370
)
369-
body = [
370-
{
371-
"appId": PIPELINE_NAME,
372-
"programType": "spark",
373-
"programId": "DataStreamsSparkStreaming",
374-
"runtimeargs": RUNTIME_ARGS,
375-
}
376-
]
371+
assert result == run_id
377372
mock_request.assert_called_once_with(
378-
url=f"{INSTANCE_URL}/v3/namespaces/default/start", method="POST", body=body
373+
url=f"{INSTANCE_URL}/v3/namespaces/default/apps/{PIPELINE_NAME}/sparks/DataStreamsSparkStreaming/start",
374+
method="POST",
375+
body=RUNTIME_ARGS,
379376
)
380377

381378
@mock.patch(HOOK_STR.format("DataFusionHook._cdap_request"))
@@ -390,16 +387,10 @@ def test_start_pipeline_should_fail_if_empty_data_response(self, mock_request, h
390387
hook.start_pipeline(
391388
pipeline_name=PIPELINE_NAME, instance_url=INSTANCE_URL, runtime_args=RUNTIME_ARGS
392389
)
393-
body = [
394-
{
395-
"appId": PIPELINE_NAME,
396-
"programType": "workflow",
397-
"programId": "DataPipelineWorkflow",
398-
"runtimeargs": RUNTIME_ARGS,
399-
}
400-
]
401390
mock_request.assert_called_once_with(
402-
url=f"{INSTANCE_URL}/v3/namespaces/default/start", method="POST", body=body
391+
url=f"{INSTANCE_URL}/v3/namespaces/default/apps/{PIPELINE_NAME}/workflows/DataPipelineWorkflow/start",
392+
method="POST",
393+
body=RUNTIME_ARGS,
403394
)
404395

405396
@mock.patch(HOOK_STR.format("DataFusionHook._cdap_request"))
@@ -409,16 +400,31 @@ def test_start_pipeline_should_fail_if_status_not_200(self, mock_request, hook):
409400
hook.start_pipeline(
410401
pipeline_name=PIPELINE_NAME, instance_url=INSTANCE_URL, runtime_args=RUNTIME_ARGS
411402
)
412-
body = [
413-
{
414-
"appId": PIPELINE_NAME,
415-
"programType": "workflow",
416-
"programId": "DataPipelineWorkflow",
417-
"runtimeargs": RUNTIME_ARGS,
418-
}
419-
]
420403
mock_request.assert_called_once_with(
421-
url=f"{INSTANCE_URL}/v3/namespaces/default/start", method="POST", body=body
404+
url=f"{INSTANCE_URL}/v3/namespaces/default/apps/{PIPELINE_NAME}/workflows/DataPipelineWorkflow/start",
405+
method="POST",
406+
body=RUNTIME_ARGS,
407+
)
408+
409+
@mock.patch(HOOK_STR.format("DataFusionHook._cdap_request"))
410+
def test_start_pipeline_should_fail_if_no_run_id(self, mock_request, hook):
411+
"""Test that start_pipeline fails gracefully when response doesn't contain runId."""
412+
error_response = '{"error": "Invalid runtime arguments"}'
413+
mock_request.return_value = mock.MagicMock(
414+
spec=google.auth.transport.Response, status=200, data=error_response
415+
)
416+
with pytest.raises(
417+
AirflowException,
418+
match=r"Failed to start pipeline 'shrubberyPipeline'. "
419+
r"The response does not contain a runId. Error: Invalid runtime arguments",
420+
):
421+
hook.start_pipeline(
422+
pipeline_name=PIPELINE_NAME, instance_url=INSTANCE_URL, runtime_args=RUNTIME_ARGS
423+
)
424+
mock_request.assert_called_once_with(
425+
url=f"{INSTANCE_URL}/v3/namespaces/default/apps/{PIPELINE_NAME}/workflows/DataPipelineWorkflow/start",
426+
method="POST",
427+
body=RUNTIME_ARGS,
422428
)
423429

424430
@mock.patch(HOOK_STR.format("DataFusionHook._cdap_request"))

0 commit comments

Comments
 (0)