Skip to content

Commit a95341e

Browse files
TalUsvyatskyduncanista
TalUsvyatsky
and
duncanista
authored
Detect exceptions raised outside of the handler (#475)
* emit enhanced error metric and create span when an exception is raised outside of the handler function * rename handler * move fallback handler to wrapper.py * fix duration * respect DD_TRACE_ENABLED * emit telemetry and raise during init * Update datadog_lambda/tags.py Co-authored-by: jordan gonzález <[email protected]> --------- Co-authored-by: jordan gonzález <[email protected]>
1 parent 3c79531 commit a95341e

7 files changed

Lines changed: 198 additions & 16 deletions

File tree

‎datadog_lambda/handler.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
from importlib import import_module
88

99
import os
10+
from time import time_ns
11+
12+
from datadog_lambda.tracing import emit_telemetry_on_exception_outside_of_handler
1013
from datadog_lambda.wrapper import datadog_lambda_wrapper
1114
from datadog_lambda.module_name import modify_module_name
1215

@@ -27,5 +30,17 @@ class HandlerError(Exception):
2730

2831
(mod_name, handler_name) = parts
2932
modified_mod_name = modify_module_name(mod_name)
30-
handler_module = import_module(modified_mod_name)
31-
handler = datadog_lambda_wrapper(getattr(handler_module, handler_name))
33+
34+
try:
35+
handler_load_start_time_ns = time_ns()
36+
handler_module = import_module(modified_mod_name)
37+
handler_func = getattr(handler_module, handler_name)
38+
except Exception as e:
39+
emit_telemetry_on_exception_outside_of_handler(
40+
e,
41+
modified_mod_name,
42+
handler_load_start_time_ns,
43+
)
44+
raise
45+
46+
handler = datadog_lambda_wrapper(handler_func)

‎datadog_lambda/metric.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ def submit_enhanced_metric(metric_name, lambda_context):
100100
101101
Args:
102102
metric_name (str): metric name w/o enhanced prefix i.e. "invocations" or "errors"
103-
lambda_context (dict): Lambda context dict passed to the function by AWS
103+
lambda_context (object): Lambda context dict passed to the function by AWS
104104
"""
105105
if not enhanced_metrics_enabled:
106106
logger.debug(
@@ -118,7 +118,7 @@ def submit_invocations_metric(lambda_context):
118118
"""Increment aws.lambda.enhanced.invocations by 1, applying runtime, layer, and cold_start tags
119119
120120
Args:
121-
lambda_context (dict): Lambda context dict passed to the function by AWS
121+
lambda_context (object): Lambda context dict passed to the function by AWS
122122
"""
123123
submit_enhanced_metric("invocations", lambda_context)
124124

@@ -127,6 +127,6 @@ def submit_errors_metric(lambda_context):
127127
"""Increment aws.lambda.enhanced.errors by 1, applying runtime, layer, and cold_start tags
128128
129129
Args:
130-
lambda_context (dict): Lambda context dict passed to the function by AWS
130+
lambda_context (object): Lambda context dict passed to the function by AWS
131131
"""
132132
submit_enhanced_metric("errors", lambda_context)

‎datadog_lambda/tags.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,11 @@ def parse_lambda_tags_from_arn(lambda_context):
5555

5656
def get_enhanced_metrics_tags(lambda_context):
5757
"""Get the list of tags to apply to enhanced metrics"""
58-
tags = parse_lambda_tags_from_arn(lambda_context)
58+
tags = []
59+
if lambda_context:
60+
tags = parse_lambda_tags_from_arn(lambda_context)
61+
tags.append(f"memorysize:{lambda_context.memory_limit_in_mb}")
5962
tags.append(get_cold_start_tag())
60-
tags.append(f"memorysize:{lambda_context.memory_limit_in_mb}")
6163
tags.append(runtime_tag)
6264
tags.append(library_version_tag)
6365
return tags

‎datadog_lambda/tracing.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import logging
77
import os
88
import base64
9+
import traceback
910
import ujson as json
1011
from datetime import datetime, timezone
1112
from typing import Optional, Dict
@@ -1320,3 +1321,34 @@ def is_async(span: Span) -> bool:
13201321
e,
13211322
)
13221323
return False
1324+
1325+
1326+
def emit_telemetry_on_exception_outside_of_handler(
1327+
exception, resource_name, handler_load_start_time_ns
1328+
):
1329+
"""
1330+
Emit an enhanced error metric and create a span for exceptions occurring outside the handler
1331+
"""
1332+
submit_errors_metric(None)
1333+
if dd_tracing_enabled:
1334+
span = tracer.trace(
1335+
"aws.lambda",
1336+
service="aws.lambda",
1337+
resource=resource_name,
1338+
span_type="serverless",
1339+
)
1340+
span.start_ns = handler_load_start_time_ns
1341+
1342+
tags = {
1343+
"error.status": 500,
1344+
"error.type": type(exception).__name__,
1345+
"error.message": exception,
1346+
"error.stack": traceback.format_exc(),
1347+
"resource_names": resource_name,
1348+
"resource.name": resource_name,
1349+
"operation_name": "aws.lambda",
1350+
"status": "error",
1351+
}
1352+
span.set_tags(tags)
1353+
span.error = 1
1354+
span.finish()

‎tests/test_handler.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import os
2+
import sys
3+
import unittest
4+
from unittest.mock import patch
5+
6+
from tests.utils import get_mock_context
7+
8+
9+
class TestHandler(unittest.TestCase):
10+
def tearDown(self):
11+
for mod in sys.modules.copy():
12+
if mod.startswith("datadog_lambda.handler"):
13+
del sys.modules[mod]
14+
15+
def test_dd_lambda_handler_env_var_none(self):
16+
with self.assertRaises(Exception) as context:
17+
import datadog_lambda.handler as handler
18+
19+
assert context.exception == handler.HandlerError(
20+
"DD_LAMBDA_HANDLER is not defined. Can't use prebuilt datadog handler"
21+
)
22+
23+
@patch.dict(os.environ, {"DD_LAMBDA_HANDLER": "malformed"}, clear=True)
24+
def test_dd_lambda_handler_env_var_malformed(self):
25+
with self.assertRaises(Exception) as context:
26+
import datadog_lambda.handler as handler
27+
28+
assert context.exception == handler.HandlerError(
29+
"Value malformed for DD_LAMBDA_HANDLER has invalid format."
30+
)
31+
32+
@patch.dict(os.environ, {"DD_LAMBDA_HANDLER": "nonsense.nonsense"}, clear=True)
33+
@patch("datadog_lambda.tracing.emit_telemetry_on_exception_outside_of_handler")
34+
@patch("time.time_ns", return_value=42)
35+
def test_exception_importing_module(self, mock_time, mock_emit_telemetry):
36+
with self.assertRaises(ModuleNotFoundError) as test_context:
37+
import datadog_lambda.handler
38+
39+
mock_emit_telemetry.assert_called_once_with(
40+
test_context.exception, "nonsense", 42
41+
)
42+
43+
@patch.dict(os.environ, {"DD_LAMBDA_HANDLER": "nonsense.nonsense"}, clear=True)
44+
@patch("importlib.import_module", return_value=None)
45+
@patch("datadog_lambda.tracing.emit_telemetry_on_exception_outside_of_handler")
46+
@patch("time.time_ns", return_value=42)
47+
def test_exception_getting_handler_func(
48+
self, mock_time, mock_emit_telemetry, mock_import
49+
):
50+
with self.assertRaises(AttributeError) as test_context:
51+
import datadog_lambda.handler
52+
53+
mock_emit_telemetry.assert_called_once_with(
54+
test_context.exception, "nonsense", 42
55+
)
56+
57+
@patch.dict(os.environ, {"DD_LAMBDA_HANDLER": "nonsense.nonsense"}, clear=True)
58+
@patch("importlib.import_module")
59+
@patch("datadog_lambda.tracing.emit_telemetry_on_exception_outside_of_handler")
60+
@patch("datadog_lambda.wrapper.datadog_lambda_wrapper")
61+
def test_handler_success(
62+
self, mock_lambda_wrapper, mock_emit_telemetry, mock_import
63+
):
64+
def nonsense():
65+
pass
66+
67+
mock_import.nonsense.return_value = nonsense
68+
69+
import datadog_lambda.handler
70+
71+
mock_emit_telemetry.assert_not_called()
72+
mock_lambda_wrapper.assert_called_once_with(mock_import().nonsense)

‎tests/test_tracing.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import copy
22
import functools
33
import json
4+
import traceback
45
import pytest
56
import os
67
import unittest
@@ -36,6 +37,7 @@
3637
determine_service_name,
3738
service_mapping as global_service_mapping,
3839
propagator,
40+
emit_telemetry_on_exception_outside_of_handler,
3941
)
4042
from datadog_lambda.trigger import EventTypes
4143

@@ -1999,3 +2001,62 @@ def test_deterministic_m5_hash__always_leading_with_zero(self):
19992001
# Leading zeros will be omitted, so only test for full 64 bits present
20002002
if len(result_in_binary) == 66: # "0b" + 64 bits.
20012003
self.assertTrue(result_in_binary.startswith("0b0"))
2004+
2005+
2006+
class TestExceptionOutsideHandler(unittest.TestCase):
2007+
@patch("datadog_lambda.tracing.dd_tracing_enabled", True)
2008+
@patch("datadog_lambda.tracing.submit_errors_metric")
2009+
@patch("time.time_ns", return_value=42)
2010+
def test_exception_outside_handler_tracing_enabled(
2011+
self, mock_time, mock_submit_errors_metric
2012+
):
2013+
fake_error = ValueError("Some error message")
2014+
resource_name = "my_handler"
2015+
span_type = "aws.lambda"
2016+
mock_span = Mock()
2017+
with patch(
2018+
"datadog_lambda.tracing.tracer.trace", return_value=mock_span
2019+
) as mock_trace:
2020+
emit_telemetry_on_exception_outside_of_handler(
2021+
fake_error, resource_name, 42
2022+
)
2023+
2024+
mock_submit_errors_metric.assert_called_once_with(None)
2025+
2026+
mock_trace.assert_called_once_with(
2027+
span_type,
2028+
service="aws.lambda",
2029+
resource=resource_name,
2030+
span_type="serverless",
2031+
)
2032+
mock_span.set_tags.assert_called_once_with(
2033+
{
2034+
"error.status": 500,
2035+
"error.type": "ValueError",
2036+
"error.message": fake_error,
2037+
"error.stack": traceback.format_exc(),
2038+
"resource_names": resource_name,
2039+
"resource.name": resource_name,
2040+
"operation_name": span_type,
2041+
"status": "error",
2042+
}
2043+
)
2044+
mock_span.finish.assert_called_once()
2045+
assert mock_span.error == 1
2046+
assert mock_span.start_ns == 42
2047+
2048+
@patch("datadog_lambda.tracing.dd_tracing_enabled", False)
2049+
@patch("datadog_lambda.tracing.submit_errors_metric")
2050+
@patch("time.time_ns", return_value=42)
2051+
def test_exception_outside_handler_tracing_disabled(
2052+
self, mock_time, mock_submit_errors_metric
2053+
):
2054+
fake_error = ValueError("Some error message")
2055+
resource_name = "my_handler"
2056+
with patch("datadog_lambda.tracing.tracer.trace") as mock_trace:
2057+
emit_telemetry_on_exception_outside_of_handler(
2058+
fake_error, resource_name, 42
2059+
)
2060+
2061+
mock_submit_errors_metric.assert_called_once_with(None)
2062+
mock_trace.assert_not_called()

‎tests/test_wrapper.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -220,8 +220,8 @@ def lambda_handler(event, context):
220220
"account_id:123457598159",
221221
"functionname:python-layer-test",
222222
"resource:python-layer-test",
223-
"cold_start:true",
224223
"memorysize:256",
224+
"cold_start:true",
225225
"runtime:python3.9",
226226
"datadog_lambda:v6.6.6",
227227
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -251,8 +251,8 @@ def lambda_handler(event, context):
251251
"account_id:123457598159",
252252
"functionname:python-layer-test",
253253
"resource:python-layer-test",
254-
"cold_start:true",
255254
"memorysize:256",
255+
"cold_start:true",
256256
"runtime:python3.9",
257257
"datadog_lambda:v6.6.6",
258258
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -267,8 +267,8 @@ def lambda_handler(event, context):
267267
"account_id:123457598159",
268268
"functionname:python-layer-test",
269269
"resource:python-layer-test",
270-
"cold_start:true",
271270
"memorysize:256",
271+
"cold_start:true",
272272
"runtime:python3.9",
273273
"datadog_lambda:v6.6.6",
274274
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -306,8 +306,8 @@ def lambda_handler(event, context):
306306
"account_id:123457598159",
307307
"functionname:python-layer-test",
308308
"resource:python-layer-test",
309-
"cold_start:true",
310309
"memorysize:256",
310+
"cold_start:true",
311311
"runtime:python3.9",
312312
"datadog_lambda:v6.6.6",
313313
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -322,8 +322,8 @@ def lambda_handler(event, context):
322322
"account_id:123457598159",
323323
"functionname:python-layer-test",
324324
"resource:python-layer-test",
325-
"cold_start:true",
326325
"memorysize:256",
326+
"cold_start:true",
327327
"runtime:python3.9",
328328
"datadog_lambda:v6.6.6",
329329
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -358,8 +358,8 @@ def lambda_handler(event, context):
358358
"account_id:123457598159",
359359
"functionname:python-layer-test",
360360
"resource:python-layer-test",
361-
"cold_start:true",
362361
"memorysize:256",
362+
"cold_start:true",
363363
"runtime:python3.9",
364364
"datadog_lambda:v6.6.6",
365365
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -374,8 +374,8 @@ def lambda_handler(event, context):
374374
"account_id:123457598159",
375375
"functionname:python-layer-test",
376376
"resource:python-layer-test",
377-
"cold_start:false",
378377
"memorysize:256",
378+
"cold_start:false",
379379
"runtime:python3.9",
380380
"datadog_lambda:v6.6.6",
381381
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -408,8 +408,8 @@ def lambda_handler(event, context):
408408
"account_id:123457598159",
409409
"functionname:python-layer-test",
410410
"resource:python-layer-test:Latest",
411-
"cold_start:true",
412411
"memorysize:256",
412+
"cold_start:true",
413413
"runtime:python3.9",
414414
"datadog_lambda:v6.6.6",
415415
"dd_lambda_layer:datadog-python39_X.X.X",
@@ -442,8 +442,8 @@ def lambda_handler(event, context):
442442
"functionname:python-layer-test",
443443
"executedversion:1",
444444
"resource:python-layer-test:My_alias-1",
445-
"cold_start:true",
446445
"memorysize:256",
446+
"cold_start:true",
447447
"runtime:python3.9",
448448
"datadog_lambda:v6.6.6",
449449
"dd_lambda_layer:datadog-python39_X.X.X",

0 commit comments

Comments
 (0)