Fix Lambda CloudWatch metric recording upon invocation error (#9752)

joe4dev · web-flow · commit e13bb88300d4 · 2023-11-29T09:30:31.000+01:00
diff --git a/localstack/config.py b/localstack/config.py
@@ -896,7 +896,7 @@ def populate_edge_configuration(
 )
 # SEMI-PUBLIC: not actively communicated
 LAMBDA_LIMITS_TOTAL_CODE_SIZE = int(os.environ.get("LAMBDA_LIMITS_TOTAL_CODE_SIZE", 80_530_636_800))
-# SEMI-PUBLIC: not actively communicated
+# PUBLIC: documented after AWS changed validation around 2023-11
 LAMBDA_LIMITS_CODE_SIZE_ZIPPED = int(os.environ.get("LAMBDA_LIMITS_CODE_SIZE_ZIPPED", 52_428_800))
 # SEMI-PUBLIC: not actively communicated
 LAMBDA_LIMITS_CODE_SIZE_UNZIPPED = int(
diff --git a/localstack/services/lambda_/invocation/version_manager.py b/localstack/services/lambda_/invocation/version_manager.py
@@ -25,7 +25,10 @@
     VersionState,
 )
 from localstack.services.lambda_.invocation.logs import LogHandler, LogItem
-from localstack.services.lambda_.invocation.metrics import record_cw_metric_invocation
+from localstack.services.lambda_.invocation.metrics import (
+    record_cw_metric_error,
+    record_cw_metric_invocation,
+)
 from localstack.services.lambda_.invocation.runtime_executor import get_runtime_executor
 from localstack.utils.strings import truncate
 from localstack.utils.threads import FuncThread, start_thread
@@ -215,14 +218,24 @@ def invoke(self, *, invocation: Invocation) -> InvocationResult:
                 )
 
         function_id = self.function_version.id
-        # MAYBE: reuse threads
-        start_thread(
-            lambda *args, **kwargs: record_cw_metric_invocation(
-                function_name=self.function.function_name,
-                region_name=self.function_version.id.region,
-            ),
-            name=f"record-cloudwatch-metric-{function_id.function_name}:{function_id.qualifier}",
-        )
+        # Record CloudWatch metrics in separate threads
+        # MAYBE reuse threads rather than starting new threads upon every invocation
+        if invocation_result.is_error:
+            start_thread(
+                lambda *args, **kwargs: record_cw_metric_error(
+                    function_name=self.function.function_name,
+                    region_name=self.function_version.id.region,
+                ),
+                name=f"record-cloudwatch-metric-error-{function_id.function_name}:{function_id.qualifier}",
+            )
+        else:
+            start_thread(
+                lambda *args, **kwargs: record_cw_metric_invocation(
+                    function_name=self.function.function_name,
+                    region_name=self.function_version.id.region,
+                ),
+                name=f"record-cloudwatch-metric-{function_id.function_name}:{function_id.qualifier}",
+            )
         # MAYBE: consider using the same prefix logging as in error case for execution environment.
         #   possibly as separate named logger.
         LOG.debug("Got logs for invocation '%s'", invocation.request_id)
diff --git a/localstack/services/lambda_/runtimes.py b/localstack/services/lambda_/runtimes.py
@@ -18,6 +18,8 @@
 # 6. Review special tests including:
 # a) [ext] tests.aws.services.lambda_.test_lambda_endpoint_injection
 # 7. Before merging, run the ext integration tests to cover transparent endpoint injection testing.
+# 8. Add the new runtime to the K8 image build: https://github.com/localstack/lambda-cve-mitigation
+# 9. Inform the web team to update the resource browser (consider offering an endpoint in the future)
 
 # Mapping from a) AWS Lambda runtime identifier => b) official AWS image on Amazon ECR Public
 # a) AWS Lambda runtimes: https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
diff --git a/tests/aws/services/cloudwatch/test_cloudwatch_metrics.py b/tests/aws/services/cloudwatch/test_cloudwatch_metrics.py
@@ -1,8 +1,6 @@
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING
 
-import pytest
-
 from localstack.testing.pytest import markers
 from localstack.testing.pytest.snapshot import is_aws
 from localstack.utils.strings import short_uid
@@ -56,7 +54,6 @@ def test_lambda_invoke_successful(self, aws_client, create_lambda_function, snap
         )
         snapshot.match("get-metric-data", result)
 
-    @pytest.mark.skipif(not is_aws(), reason="'Errors' metrics not reported by LS")
     @markers.aws.validated
     def test_lambda_invoke_error(self, aws_client, create_lambda_function, snapshot):
         """

Original file line number	Diff line number	Diff line change
`@@ -896,7 +896,7 @@ def populate_edge_configuration(`
`896`	`896`	`)`
`897`	`897`	`# SEMI-PUBLIC: not actively communicated`
`898`	`898`	`LAMBDA_LIMITS_TOTAL_CODE_SIZE = int(os.environ.get("LAMBDA_LIMITS_TOTAL_CODE_SIZE", 80_530_636_800))`
`899`		`-# SEMI-PUBLIC: not actively communicated`
	`899`	`+# PUBLIC: documented after AWS changed validation around 2023-11`
`900`	`900`	`LAMBDA_LIMITS_CODE_SIZE_ZIPPED = int(os.environ.get("LAMBDA_LIMITS_CODE_SIZE_ZIPPED", 52_428_800))`
`901`	`901`	`# SEMI-PUBLIC: not actively communicated`
`902`	`902`	`LAMBDA_LIMITS_CODE_SIZE_UNZIPPED = int(`