Skip to content

Commit f23f336

Browse files
Add unit tests for amazon aws lambda executor utils and docker app (apache#72265)
- Add test_utils.py for aws_lambda utils module - Add test_app.py for aws_lambda docker app module - Ensure subpackage __init__.py exists with ASF header - Remove both from OVERLOOKED_TESTS in test_project_structure.py
1 parent ead8618 commit f23f336

4 files changed

Lines changed: 310 additions & 2 deletions

File tree

‎airflow-core/tests/unit/always/test_project_structure.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,6 @@ def test_providers_modules_should_have_tests(self):
6464
"providers/amazon/tests/unit/amazon/aws/auth_manager/datamodels/test_login.py",
6565
"providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor_config.py",
6666
"providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor_config.py",
67-
"providers/amazon/tests/unit/amazon/aws/executors/aws_lambda/test_utils.py",
68-
"providers/amazon/tests/unit/amazon/aws/executors/aws_lambda/docker/test_app.py",
6967
"providers/amazon/tests/unit/amazon/aws/operators/test_emr.py",
7068
"providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker.py",
7169
"providers/amazon/tests/unit/amazon/aws/sensors/test_emr.py",
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
import json
20+
import subprocess
21+
from unittest.mock import MagicMock, patch
22+
23+
import pytest
24+
25+
from airflow.providers.amazon.aws.executors.aws_lambda.docker import app
26+
27+
28+
class TestLambdaApp:
29+
def test_get_queue_url_primary(self, monkeypatch):
30+
monkeypatch.setenv(
31+
"AIRFLOW__AWS_LAMBDA_EXECUTOR__QUEUE_URL", "https://sqs.us-east-1.amazonaws.com/123/primary"
32+
)
33+
monkeypatch.setenv("QUEUE_URL", "https://sqs.us-east-1.amazonaws.com/123/fallback")
34+
assert app.get_queue_url() == "https://sqs.us-east-1.amazonaws.com/123/primary"
35+
36+
def test_get_queue_url_fallback(self, monkeypatch):
37+
monkeypatch.delenv("AIRFLOW__AWS_LAMBDA_EXECUTOR__QUEUE_URL", raising=False)
38+
monkeypatch.setenv("QUEUE_URL", "https://sqs.us-east-1.amazonaws.com/123/fallback")
39+
assert app.get_queue_url() == "https://sqs.us-east-1.amazonaws.com/123/fallback"
40+
41+
def test_get_queue_url_missing_raises(self, monkeypatch):
42+
monkeypatch.delenv("AIRFLOW__AWS_LAMBDA_EXECUTOR__QUEUE_URL", raising=False)
43+
monkeypatch.delenv("QUEUE_URL", raising=False)
44+
with pytest.raises(RuntimeError, match="No Queue URL detected"):
45+
app.get_queue_url()
46+
47+
@patch("boto3.client")
48+
def test_get_sqs_client(self, mock_boto_client):
49+
mock_client = MagicMock()
50+
mock_boto_client.return_value = mock_client
51+
result = app.get_sqs_client()
52+
mock_boto_client.assert_called_once_with("sqs")
53+
assert result == mock_client
54+
55+
@patch.object(app, "get_sqs_client")
56+
@patch.object(app, "get_queue_url")
57+
@patch("subprocess.run")
58+
def test_run_and_report_success(self, mock_subprocess_run, mock_get_queue, mock_get_sqs):
59+
mock_subprocess_run.return_value = subprocess.CompletedProcess(
60+
args=["airflow", "tasks", "run"],
61+
returncode=0,
62+
stdout=b"Task ran successfully",
63+
)
64+
mock_get_queue.return_value = "https://sqs.us-east-1.amazonaws.com/123/queue"
65+
mock_sqs = MagicMock()
66+
mock_get_sqs.return_value = mock_sqs
67+
68+
app.run_and_report(["airflow", "tasks", "run"], "dag.task.run_id")
69+
70+
mock_subprocess_run.assert_called_once_with(
71+
["airflow", "tasks", "run"],
72+
check=False,
73+
shell=False,
74+
stdout=subprocess.PIPE,
75+
stderr=subprocess.STDOUT,
76+
)
77+
expected_body = json.dumps({"task_key": "dag.task.run_id", "return_code": 0})
78+
mock_sqs.send_message.assert_called_once_with(
79+
QueueUrl="https://sqs.us-east-1.amazonaws.com/123/queue",
80+
MessageBody=expected_body,
81+
)
82+
83+
@patch.object(app, "get_sqs_client")
84+
@patch.object(app, "get_queue_url")
85+
@patch("subprocess.run")
86+
def test_run_and_report_subprocess_exception(self, mock_subprocess_run, mock_get_queue, mock_get_sqs):
87+
mock_subprocess_run.side_effect = OSError("Execution failed")
88+
mock_get_queue.return_value = "https://sqs.us-east-1.amazonaws.com/123/queue"
89+
mock_sqs = MagicMock()
90+
mock_get_sqs.return_value = mock_sqs
91+
92+
app.run_and_report("airflow tasks run string_cmd", "dag.task.run_id")
93+
94+
expected_body = json.dumps({"task_key": "dag.task.run_id", "return_code": 1})
95+
mock_sqs.send_message.assert_called_once_with(
96+
QueueUrl="https://sqs.us-east-1.amazonaws.com/123/queue",
97+
MessageBody=expected_body,
98+
)
99+
100+
@patch.object(app, "get_sqs_client")
101+
@patch.object(app, "get_queue_url")
102+
@patch("subprocess.run")
103+
def test_run_and_report_sqs_failure_swallowed(self, mock_subprocess_run, mock_get_queue, mock_get_sqs):
104+
mock_subprocess_run.return_value = subprocess.CompletedProcess(
105+
args=["airflow"],
106+
returncode=0,
107+
stdout=b"ok",
108+
)
109+
mock_get_queue.return_value = "https://sqs.us-east-1.amazonaws.com/123/queue"
110+
mock_sqs = MagicMock()
111+
mock_sqs.send_message.side_effect = RuntimeError("SQS unavailable")
112+
mock_get_sqs.return_value = mock_sqs
113+
114+
# Should log exception but not re-raise
115+
app.run_and_report(["airflow"], "key")
116+
mock_sqs.send_message.assert_called_once()
117+
118+
@patch("boto3.resource")
119+
def test_fetch_dags_from_s3(self, mock_boto_resource, monkeypatch, tmp_path):
120+
mock_s3 = MagicMock()
121+
mock_boto_resource.return_value = mock_s3
122+
mock_bucket = MagicMock()
123+
mock_s3.Bucket.return_value = mock_bucket
124+
125+
obj_dir = MagicMock(key="dags/subdir/")
126+
obj_dag1 = MagicMock(key="dags/example_dag.py")
127+
obj_dag2 = MagicMock(key="dags/subdir/sub_dag.py")
128+
mock_bucket.objects.filter.return_value = [obj_dir, obj_dag1, obj_dag2]
129+
130+
app.fetch_dags_from_s3("s3://my-test-bucket/dags/")
131+
132+
mock_s3.Bucket.assert_called_once_with("my-test-bucket")
133+
mock_bucket.objects.filter.assert_called_once_with(Prefix="dags/")
134+
135+
# Verify only files were downloaded, directory skipped
136+
assert mock_bucket.download_file.call_count == 2
137+
138+
@patch.object(app, "run_and_report")
139+
@patch.object(app, "fetch_dags_from_s3")
140+
def test_lambda_handler_without_s3(self, mock_fetch_dags, mock_run_and_report, monkeypatch):
141+
monkeypatch.setattr(app, "S3_URI", None)
142+
event = {
143+
"command": ["airflow", "tasks", "run"],
144+
"task_key": "task_1",
145+
"executor_config": {"memory": 512},
146+
}
147+
app.lambda_handler(event, None)
148+
149+
mock_fetch_dags.assert_not_called()
150+
mock_run_and_report.assert_called_once_with(["airflow", "tasks", "run"], "task_1")
151+
152+
@patch.object(app, "run_and_report")
153+
@patch.object(app, "fetch_dags_from_s3")
154+
def test_lambda_handler_with_s3(self, mock_fetch_dags, mock_run_and_report, monkeypatch):
155+
monkeypatch.setattr(app, "S3_URI", "s3://my-bucket/dags/")
156+
event = {
157+
"command": "airflow tasks run",
158+
"task_key": "task_2",
159+
}
160+
app.lambda_handler(event, None)
161+
162+
mock_fetch_dags.assert_called_once_with("s3://my-bucket/dags/")
163+
mock_run_and_report.assert_called_once_with("airflow tasks run", "task_2")
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
import datetime
20+
from unittest import mock
21+
22+
from airflow.providers.amazon.aws.executors.aws_lambda.utils import (
23+
CONFIG_GROUP_NAME,
24+
INVALID_CREDENTIALS_EXCEPTIONS,
25+
AllLambdaConfigKeys,
26+
InvokeLambdaKwargsConfigKeys,
27+
LambdaQueuedTask,
28+
)
29+
30+
31+
class TestAwsLambdaExecutorUtilsConstants:
32+
def test_config_group_name(self):
33+
assert CONFIG_GROUP_NAME == "aws_lambda_executor"
34+
35+
def test_invalid_credentials_exceptions(self):
36+
assert "ExpiredTokenException" in INVALID_CREDENTIALS_EXCEPTIONS
37+
assert "InvalidClientTokenId" in INVALID_CREDENTIALS_EXCEPTIONS
38+
assert "UnrecognizedClientException" in INVALID_CREDENTIALS_EXCEPTIONS
39+
assert len(INVALID_CREDENTIALS_EXCEPTIONS) == 3
40+
41+
42+
class TestLambdaConfigKeys:
43+
def test_invoke_lambda_kwargs_config_keys(self):
44+
assert InvokeLambdaKwargsConfigKeys.FUNCTION_NAME == "function_name"
45+
assert InvokeLambdaKwargsConfigKeys.QUALIFIER == "function_qualifier"
46+
assert set(InvokeLambdaKwargsConfigKeys()) == {
47+
"function_name",
48+
"function_qualifier",
49+
}
50+
51+
def test_all_lambda_config_keys(self):
52+
assert AllLambdaConfigKeys.AWS_CONN_ID == "conn_id"
53+
assert AllLambdaConfigKeys.CHECK_HEALTH_ON_STARTUP == "check_health_on_startup"
54+
assert AllLambdaConfigKeys.MAX_INVOKE_ATTEMPTS == "max_invoke_attempts"
55+
assert AllLambdaConfigKeys.REGION_NAME == "region_name"
56+
assert AllLambdaConfigKeys.QUEUE_URL == "queue_url"
57+
assert AllLambdaConfigKeys.DLQ_URL == "dead_letter_queue_url"
58+
assert AllLambdaConfigKeys.END_WAIT_TIMEOUT == "end_wait_timeout"
59+
assert AllLambdaConfigKeys.FUNCTION_NAME == "function_name"
60+
assert AllLambdaConfigKeys.QUALIFIER == "function_qualifier"
61+
62+
expected_keys = {
63+
"conn_id",
64+
"check_health_on_startup",
65+
"max_invoke_attempts",
66+
"region_name",
67+
"queue_url",
68+
"dead_letter_queue_url",
69+
"end_wait_timeout",
70+
"function_name",
71+
"function_qualifier",
72+
}
73+
assert set(AllLambdaConfigKeys()) == expected_keys
74+
75+
76+
class TestLambdaQueuedTask:
77+
def test_lambda_queued_task_creation(self):
78+
key = mock.MagicMock()
79+
command = ["airflow", "tasks", "run", "dag_id", "task_id"]
80+
queue = "lambda-queue"
81+
executor_config = {"memory": 512}
82+
attempt_number = 2
83+
next_attempt_time = datetime.datetime.now(datetime.UTC)
84+
85+
queued_task = LambdaQueuedTask(
86+
key=key,
87+
command=command,
88+
queue=queue,
89+
executor_config=executor_config,
90+
attempt_number=attempt_number,
91+
next_attempt_time=next_attempt_time,
92+
)
93+
94+
assert queued_task.key is key
95+
assert queued_task.command == command
96+
assert queued_task.queue == queue
97+
assert queued_task.executor_config == executor_config
98+
assert queued_task.attempt_number == attempt_number
99+
assert queued_task.next_attempt_time is next_attempt_time
100+
101+
def test_lambda_queued_task_equality(self):
102+
key = mock.MagicMock()
103+
now = datetime.datetime(2026, 1, 1, 12, 0, 0, tzinfo=datetime.UTC)
104+
task1 = LambdaQueuedTask(
105+
key=key,
106+
command=["cmd"],
107+
queue="q",
108+
executor_config={},
109+
attempt_number=1,
110+
next_attempt_time=now,
111+
)
112+
task2 = LambdaQueuedTask(
113+
key=key,
114+
command=["cmd"],
115+
queue="q",
116+
executor_config={},
117+
attempt_number=1,
118+
next_attempt_time=now,
119+
)
120+
task3 = LambdaQueuedTask(
121+
key=key,
122+
command=["different_cmd"],
123+
queue="q",
124+
executor_config={},
125+
attempt_number=1,
126+
next_attempt_time=now,
127+
)
128+
129+
assert task1 == task2
130+
assert task1 != task3

0 commit comments

Comments
 (0)