2121from unittest import mock
2222
2323import aiohttp
24+ import google .auth .transport
2425import pytest
2526from aiohttp .helpers import TimerNoop
2627from yarl import URL
@@ -340,42 +341,38 @@ def test_list_pipelines_should_fail_if_status_not_200(self, mock_request, hook):
340341 @mock .patch (HOOK_STR .format ("DataFusionHook._cdap_request" ))
341342 def test_start_pipeline (self , mock_request , hook ):
342343 run_id = 1234
343- mock_request .return_value = mock .MagicMock (status = 200 , data = f'[{{"runId":{ run_id } }}]' )
344+ mock_request .return_value = mock .MagicMock (
345+ spec = google .auth .transport .Response , status = 200 , data = f'{{"runId":{ run_id } }}'
346+ )
344347
345- hook .start_pipeline (pipeline_name = PIPELINE_NAME , instance_url = INSTANCE_URL , runtime_args = RUNTIME_ARGS )
346- body = [
347- {
348- "appId" : PIPELINE_NAME ,
349- "programType" : "workflow" ,
350- "programId" : "DataPipelineWorkflow" ,
351- "runtimeargs" : RUNTIME_ARGS ,
352- }
353- ]
348+ result = hook .start_pipeline (
349+ pipeline_name = PIPELINE_NAME , instance_url = INSTANCE_URL , runtime_args = RUNTIME_ARGS
350+ )
351+ assert result == str (run_id )
354352 mock_request .assert_called_once_with (
355- url = f"{ INSTANCE_URL } /v3/namespaces/default/start" , method = "POST" , body = body
353+ url = f"{ INSTANCE_URL } /v3/namespaces/default/apps/{ PIPELINE_NAME } /workflows/DataPipelineWorkflow/start" ,
354+ method = "POST" ,
355+ body = RUNTIME_ARGS ,
356356 )
357357
358358 @mock .patch (HOOK_STR .format ("DataFusionHook._cdap_request" ))
359359 def test_start_pipeline_stream (self , mock_request , hook ):
360- run_id = 1234
361- mock_request .return_value = mock .MagicMock (status = 200 , data = f'[{{"runId":{ run_id } }}]' )
360+ run_id = "test-run-123"
361+ mock_request .return_value = mock .MagicMock (
362+ spec = google .auth .transport .Response , status = 200 , data = f'{{"runId":"{ run_id } "}}'
363+ )
362364
363- hook .start_pipeline (
365+ result = hook .start_pipeline (
364366 pipeline_name = PIPELINE_NAME ,
365367 instance_url = INSTANCE_URL ,
366368 runtime_args = RUNTIME_ARGS ,
367369 pipeline_type = DataFusionPipelineType .STREAM ,
368370 )
369- body = [
370- {
371- "appId" : PIPELINE_NAME ,
372- "programType" : "spark" ,
373- "programId" : "DataStreamsSparkStreaming" ,
374- "runtimeargs" : RUNTIME_ARGS ,
375- }
376- ]
371+ assert result == run_id
377372 mock_request .assert_called_once_with (
378- url = f"{ INSTANCE_URL } /v3/namespaces/default/start" , method = "POST" , body = body
373+ url = f"{ INSTANCE_URL } /v3/namespaces/default/apps/{ PIPELINE_NAME } /sparks/DataStreamsSparkStreaming/start" ,
374+ method = "POST" ,
375+ body = RUNTIME_ARGS ,
379376 )
380377
381378 @mock .patch (HOOK_STR .format ("DataFusionHook._cdap_request" ))
@@ -390,16 +387,10 @@ def test_start_pipeline_should_fail_if_empty_data_response(self, mock_request, h
390387 hook .start_pipeline (
391388 pipeline_name = PIPELINE_NAME , instance_url = INSTANCE_URL , runtime_args = RUNTIME_ARGS
392389 )
393- body = [
394- {
395- "appId" : PIPELINE_NAME ,
396- "programType" : "workflow" ,
397- "programId" : "DataPipelineWorkflow" ,
398- "runtimeargs" : RUNTIME_ARGS ,
399- }
400- ]
401390 mock_request .assert_called_once_with (
402- url = f"{ INSTANCE_URL } /v3/namespaces/default/start" , method = "POST" , body = body
391+ url = f"{ INSTANCE_URL } /v3/namespaces/default/apps/{ PIPELINE_NAME } /workflows/DataPipelineWorkflow/start" ,
392+ method = "POST" ,
393+ body = RUNTIME_ARGS ,
403394 )
404395
405396 @mock .patch (HOOK_STR .format ("DataFusionHook._cdap_request" ))
@@ -409,16 +400,31 @@ def test_start_pipeline_should_fail_if_status_not_200(self, mock_request, hook):
409400 hook .start_pipeline (
410401 pipeline_name = PIPELINE_NAME , instance_url = INSTANCE_URL , runtime_args = RUNTIME_ARGS
411402 )
412- body = [
413- {
414- "appId" : PIPELINE_NAME ,
415- "programType" : "workflow" ,
416- "programId" : "DataPipelineWorkflow" ,
417- "runtimeargs" : RUNTIME_ARGS ,
418- }
419- ]
420403 mock_request .assert_called_once_with (
421- url = f"{ INSTANCE_URL } /v3/namespaces/default/start" , method = "POST" , body = body
404+ url = f"{ INSTANCE_URL } /v3/namespaces/default/apps/{ PIPELINE_NAME } /workflows/DataPipelineWorkflow/start" ,
405+ method = "POST" ,
406+ body = RUNTIME_ARGS ,
407+ )
408+
409+ @mock .patch (HOOK_STR .format ("DataFusionHook._cdap_request" ))
410+ def test_start_pipeline_should_fail_if_no_run_id (self , mock_request , hook ):
411+ """Test that start_pipeline fails gracefully when response doesn't contain runId."""
412+ error_response = '{"error": "Invalid runtime arguments"}'
413+ mock_request .return_value = mock .MagicMock (
414+ spec = google .auth .transport .Response , status = 200 , data = error_response
415+ )
416+ with pytest .raises (
417+ AirflowException ,
418+ match = r"Failed to start pipeline 'shrubberyPipeline'. "
419+ r"The response does not contain a runId. Error: Invalid runtime arguments" ,
420+ ):
421+ hook .start_pipeline (
422+ pipeline_name = PIPELINE_NAME , instance_url = INSTANCE_URL , runtime_args = RUNTIME_ARGS
423+ )
424+ mock_request .assert_called_once_with (
425+ url = f"{ INSTANCE_URL } /v3/namespaces/default/apps/{ PIPELINE_NAME } /workflows/DataPipelineWorkflow/start" ,
426+ method = "POST" ,
427+ body = RUNTIME_ARGS ,
422428 )
423429
424430 @mock .patch (HOOK_STR .format ("DataFusionHook._cdap_request" ))
0 commit comments