|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +# or more contributor license agreements. See the NOTICE file |
| 3 | +# distributed with this work for additional information |
| 4 | +# regarding copyright ownership. The ASF licenses this file |
| 5 | +# to you under the Apache License, Version 2.0 (the |
| 6 | +# "License"); you may not use this file except in compliance |
| 7 | +# with the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, |
| 12 | +# software distributed under the License is distributed on an |
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +# KIND, either express or implied. See the License for the |
| 15 | +# specific language governing permissions and limitations |
| 16 | +# under the License. |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +import json |
| 20 | +import subprocess |
| 21 | +from unittest.mock import MagicMock, patch |
| 22 | + |
| 23 | +import pytest |
| 24 | + |
| 25 | +from airflow.providers.amazon.aws.executors.aws_lambda.docker import app |
| 26 | + |
| 27 | + |
| 28 | +class TestLambdaApp: |
| 29 | + def test_get_queue_url_primary(self, monkeypatch): |
| 30 | + monkeypatch.setenv( |
| 31 | + "AIRFLOW__AWS_LAMBDA_EXECUTOR__QUEUE_URL", "https://sqs.us-east-1.amazonaws.com/123/primary" |
| 32 | + ) |
| 33 | + monkeypatch.setenv("QUEUE_URL", "https://sqs.us-east-1.amazonaws.com/123/fallback") |
| 34 | + assert app.get_queue_url() == "https://sqs.us-east-1.amazonaws.com/123/primary" |
| 35 | + |
| 36 | + def test_get_queue_url_fallback(self, monkeypatch): |
| 37 | + monkeypatch.delenv("AIRFLOW__AWS_LAMBDA_EXECUTOR__QUEUE_URL", raising=False) |
| 38 | + monkeypatch.setenv("QUEUE_URL", "https://sqs.us-east-1.amazonaws.com/123/fallback") |
| 39 | + assert app.get_queue_url() == "https://sqs.us-east-1.amazonaws.com/123/fallback" |
| 40 | + |
| 41 | + def test_get_queue_url_missing_raises(self, monkeypatch): |
| 42 | + monkeypatch.delenv("AIRFLOW__AWS_LAMBDA_EXECUTOR__QUEUE_URL", raising=False) |
| 43 | + monkeypatch.delenv("QUEUE_URL", raising=False) |
| 44 | + with pytest.raises(RuntimeError, match="No Queue URL detected"): |
| 45 | + app.get_queue_url() |
| 46 | + |
| 47 | + @patch("boto3.client") |
| 48 | + def test_get_sqs_client(self, mock_boto_client): |
| 49 | + mock_client = MagicMock() |
| 50 | + mock_boto_client.return_value = mock_client |
| 51 | + result = app.get_sqs_client() |
| 52 | + mock_boto_client.assert_called_once_with("sqs") |
| 53 | + assert result == mock_client |
| 54 | + |
| 55 | + @patch.object(app, "get_sqs_client") |
| 56 | + @patch.object(app, "get_queue_url") |
| 57 | + @patch("subprocess.run") |
| 58 | + def test_run_and_report_success(self, mock_subprocess_run, mock_get_queue, mock_get_sqs): |
| 59 | + mock_subprocess_run.return_value = subprocess.CompletedProcess( |
| 60 | + args=["airflow", "tasks", "run"], |
| 61 | + returncode=0, |
| 62 | + stdout=b"Task ran successfully", |
| 63 | + ) |
| 64 | + mock_get_queue.return_value = "https://sqs.us-east-1.amazonaws.com/123/queue" |
| 65 | + mock_sqs = MagicMock() |
| 66 | + mock_get_sqs.return_value = mock_sqs |
| 67 | + |
| 68 | + app.run_and_report(["airflow", "tasks", "run"], "dag.task.run_id") |
| 69 | + |
| 70 | + mock_subprocess_run.assert_called_once_with( |
| 71 | + ["airflow", "tasks", "run"], |
| 72 | + check=False, |
| 73 | + shell=False, |
| 74 | + stdout=subprocess.PIPE, |
| 75 | + stderr=subprocess.STDOUT, |
| 76 | + ) |
| 77 | + expected_body = json.dumps({"task_key": "dag.task.run_id", "return_code": 0}) |
| 78 | + mock_sqs.send_message.assert_called_once_with( |
| 79 | + QueueUrl="https://sqs.us-east-1.amazonaws.com/123/queue", |
| 80 | + MessageBody=expected_body, |
| 81 | + ) |
| 82 | + |
| 83 | + @patch.object(app, "get_sqs_client") |
| 84 | + @patch.object(app, "get_queue_url") |
| 85 | + @patch("subprocess.run") |
| 86 | + def test_run_and_report_subprocess_exception(self, mock_subprocess_run, mock_get_queue, mock_get_sqs): |
| 87 | + mock_subprocess_run.side_effect = OSError("Execution failed") |
| 88 | + mock_get_queue.return_value = "https://sqs.us-east-1.amazonaws.com/123/queue" |
| 89 | + mock_sqs = MagicMock() |
| 90 | + mock_get_sqs.return_value = mock_sqs |
| 91 | + |
| 92 | + app.run_and_report("airflow tasks run string_cmd", "dag.task.run_id") |
| 93 | + |
| 94 | + expected_body = json.dumps({"task_key": "dag.task.run_id", "return_code": 1}) |
| 95 | + mock_sqs.send_message.assert_called_once_with( |
| 96 | + QueueUrl="https://sqs.us-east-1.amazonaws.com/123/queue", |
| 97 | + MessageBody=expected_body, |
| 98 | + ) |
| 99 | + |
| 100 | + @patch.object(app, "get_sqs_client") |
| 101 | + @patch.object(app, "get_queue_url") |
| 102 | + @patch("subprocess.run") |
| 103 | + def test_run_and_report_sqs_failure_swallowed(self, mock_subprocess_run, mock_get_queue, mock_get_sqs): |
| 104 | + mock_subprocess_run.return_value = subprocess.CompletedProcess( |
| 105 | + args=["airflow"], |
| 106 | + returncode=0, |
| 107 | + stdout=b"ok", |
| 108 | + ) |
| 109 | + mock_get_queue.return_value = "https://sqs.us-east-1.amazonaws.com/123/queue" |
| 110 | + mock_sqs = MagicMock() |
| 111 | + mock_sqs.send_message.side_effect = RuntimeError("SQS unavailable") |
| 112 | + mock_get_sqs.return_value = mock_sqs |
| 113 | + |
| 114 | + # Should log exception but not re-raise |
| 115 | + app.run_and_report(["airflow"], "key") |
| 116 | + mock_sqs.send_message.assert_called_once() |
| 117 | + |
| 118 | + @patch("boto3.resource") |
| 119 | + def test_fetch_dags_from_s3(self, mock_boto_resource, monkeypatch, tmp_path): |
| 120 | + mock_s3 = MagicMock() |
| 121 | + mock_boto_resource.return_value = mock_s3 |
| 122 | + mock_bucket = MagicMock() |
| 123 | + mock_s3.Bucket.return_value = mock_bucket |
| 124 | + |
| 125 | + obj_dir = MagicMock(key="dags/subdir/") |
| 126 | + obj_dag1 = MagicMock(key="dags/example_dag.py") |
| 127 | + obj_dag2 = MagicMock(key="dags/subdir/sub_dag.py") |
| 128 | + mock_bucket.objects.filter.return_value = [obj_dir, obj_dag1, obj_dag2] |
| 129 | + |
| 130 | + app.fetch_dags_from_s3("s3://my-test-bucket/dags/") |
| 131 | + |
| 132 | + mock_s3.Bucket.assert_called_once_with("my-test-bucket") |
| 133 | + mock_bucket.objects.filter.assert_called_once_with(Prefix="dags/") |
| 134 | + |
| 135 | + # Verify only files were downloaded, directory skipped |
| 136 | + assert mock_bucket.download_file.call_count == 2 |
| 137 | + |
| 138 | + @patch.object(app, "run_and_report") |
| 139 | + @patch.object(app, "fetch_dags_from_s3") |
| 140 | + def test_lambda_handler_without_s3(self, mock_fetch_dags, mock_run_and_report, monkeypatch): |
| 141 | + monkeypatch.setattr(app, "S3_URI", None) |
| 142 | + event = { |
| 143 | + "command": ["airflow", "tasks", "run"], |
| 144 | + "task_key": "task_1", |
| 145 | + "executor_config": {"memory": 512}, |
| 146 | + } |
| 147 | + app.lambda_handler(event, None) |
| 148 | + |
| 149 | + mock_fetch_dags.assert_not_called() |
| 150 | + mock_run_and_report.assert_called_once_with(["airflow", "tasks", "run"], "task_1") |
| 151 | + |
| 152 | + @patch.object(app, "run_and_report") |
| 153 | + @patch.object(app, "fetch_dags_from_s3") |
| 154 | + def test_lambda_handler_with_s3(self, mock_fetch_dags, mock_run_and_report, monkeypatch): |
| 155 | + monkeypatch.setattr(app, "S3_URI", "s3://my-bucket/dags/") |
| 156 | + event = { |
| 157 | + "command": "airflow tasks run", |
| 158 | + "task_key": "task_2", |
| 159 | + } |
| 160 | + app.lambda_handler(event, None) |
| 161 | + |
| 162 | + mock_fetch_dags.assert_called_once_with("s3://my-bucket/dags/") |
| 163 | + mock_run_and_report.assert_called_once_with("airflow tasks run", "task_2") |
0 commit comments