Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

Commit 77b14d7

Browse files
cloutierMatcloutierMat
authored
fix vtl resolver patch (#11886)
1 parent 642e645 commit 77b14d7

File tree

7 files changed

+125
-99
lines changed

7 files changed

+125
-99
lines changed

‎localstack-core/localstack/services/apigateway/legacy/templates.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from localstack.constants import APPLICATION_JSON, APPLICATION_XML
1313
from localstack.services.apigateway.legacy.context import ApiInvocationContext
1414
from localstack.services.apigateway.legacy.helpers import select_integration_response
15-
from localstack.utils.aws.templating import VelocityUtil, VtlTemplate
15+
from localstack.utils.aws.templating import APIGW_SOURCE, VelocityUtil, VtlTemplate
1616
from localstack.utils.json import extract_jsonpath, json_safe, try_json
1717
from localstack.utils.strings import to_str
1818

@@ -184,8 +184,8 @@ def __repr__(self):
184184
class ApiGatewayVtlTemplate(VtlTemplate):
185185
"""Util class for rendering VTL templates with API Gateway specific extensions"""
186186

187-
def prepare_namespace(self, variables) -> Dict[str, Any]:
188-
namespace = super().prepare_namespace(variables)
187+
def prepare_namespace(self, variables, source: str = APIGW_SOURCE) -> Dict[str, Any]:
188+
namespace = super().prepare_namespace(variables, source)
189189
if stage_var := variables.get("stage_variables") or {}:
190190
namespace["stageVariables"] = stage_var
191191
input_var = variables.get("input") or {}

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/template_mapping.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
ContextVarsRequestOverride,
2828
ContextVarsResponseOverride,
2929
)
30-
from localstack.utils.aws.templating import VelocityUtil, VtlTemplate
30+
from localstack.utils.aws.templating import APIGW_SOURCE, VelocityUtil, VtlTemplate
3131
from localstack.utils.json import extract_jsonpath, json_safe
3232

3333
LOG = logging.getLogger(__name__)
@@ -173,8 +173,8 @@ def __repr__(self):
173173
class ApiGatewayVtlTemplate(VtlTemplate):
174174
"""Util class for rendering VTL templates with API Gateway specific extensions"""
175175

176-
def prepare_namespace(self, variables) -> dict[str, Any]:
177-
namespace = super().prepare_namespace(variables)
176+
def prepare_namespace(self, variables, source: str = APIGW_SOURCE) -> dict[str, Any]:
177+
namespace = super().prepare_namespace(variables, source)
178178
input_var = variables.get("input") or {}
179179
variables = {
180180
"input": VelocityInput(input_var.get("body"), input_var.get("params")),

‎localstack-core/localstack/utils/aws/templating.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,21 @@
77
from localstack.utils.objects import recurse_object
88
from localstack.utils.patch import patch
99

10+
SOURCE_NAMESPACE_VARIABLE = "__LOCALSTACK_SERVICE_SOURCE__"
11+
APIGW_SOURCE = "APIGW"
12+
APPSYNC_SOURCE = "APPSYNC"
13+
1014

11-
# remove this patch fails test_api_gateway_kinesis_integration
12-
# we need to validate against AWS behavior before removing this patch
1315
@patch(airspeed.operators.VariableExpression.calculate)
14-
def calculate(fn, self, *args, **kwarg):
15-
result = fn(self, *args, **kwarg)
16-
result = "" if result is None else result
16+
def calculate(fn, self, namespace, loader, global_namespace=None):
17+
result = fn(self, namespace, loader, global_namespace)
18+
19+
if global_namespace is None:
20+
global_namespace = namespace
21+
if (source := global_namespace.top().get(SOURCE_NAMESPACE_VARIABLE)) and source == APIGW_SOURCE:
22+
# Apigateway does not return None but returns an empty string instead
23+
result = "" if result is None else result
24+
1725
return result
1826

1927

@@ -117,11 +125,12 @@ def apply(obj, **_):
117125
rendered_template = json.loads(rendered_template)
118126
return rendered_template
119127

120-
def prepare_namespace(self, variables: Dict[str, Any]) -> Dict:
128+
def prepare_namespace(self, variables: Dict[str, Any], source: str = "") -> Dict:
121129
namespace = dict(variables or {})
122130
namespace.setdefault("context", {})
123131
if not namespace.get("util"):
124132
namespace["util"] = VelocityUtil()
133+
namespace[SOURCE_NAMESPACE_VARIABLE] = source
125134
return namespace
126135

127136

‎tests/aws/services/apigateway/test_apigateway_basic.py‎

Lines changed: 0 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -94,25 +94,6 @@
9494
"path_with_replace",
9595
],
9696
)
97-
# template used to transform incoming requests at the API Gateway (stream name to be filled in later)
98-
APIGATEWAY_DATA_INBOUND_TEMPLATE = """{
99-
"StreamName": "%s",
100-
"Records": [
101-
#set( $numRecords = $input.path('$.records').size() )
102-
#if($numRecords > 0)
103-
#set( $maxIndex = $numRecords - 1 )
104-
#foreach( $idx in [0..$maxIndex] )
105-
#set( $elem = $input.path("$.records[${idx}]") )
106-
#set( $elemJsonB64 = $util.base64Encode($elem.data) )
107-
{
108-
"Data": "$elemJsonB64",
109-
"PartitionKey": #if( $elem.partitionKey != '')"$elem.partitionKey"
110-
#else"$elemJsonB64.length()"#end
111-
}#if($foreach.hasNext),#end
112-
#end
113-
#end
114-
]
115-
}"""
11697

11798
API_PATH_LAMBDA_PROXY_BACKEND = "/lambda/foo1"
11899
API_PATH_LAMBDA_PROXY_BACKEND_WITH_PATH_PARAM = "/lambda/{test_param1}"
@@ -1784,57 +1765,6 @@ def test_api_gateway_http_integrations(
17841765
assert expected == content["data"]
17851766
assert ctype == headers["content-type"]
17861767

1787-
# ==================
1788-
# Helper methods
1789-
# TODO: replace with fixtures, to allow passing aws_client and enable snapshot testing
1790-
# ==================
1791-
1792-
def connect_api_gateway_to_kinesis(
1793-
self,
1794-
client,
1795-
gateway_name: str,
1796-
kinesis_stream: str,
1797-
region_name: str,
1798-
role_arn: str,
1799-
):
1800-
template = APIGATEWAY_DATA_INBOUND_TEMPLATE % kinesis_stream
1801-
resources = {
1802-
"data": [
1803-
{
1804-
"httpMethod": "POST",
1805-
"authorizationType": "NONE",
1806-
"requestModels": {"application/json": "Empty"},
1807-
"integrations": [
1808-
{
1809-
"type": "AWS",
1810-
"uri": f"arn:aws:apigateway:{region_name}:kinesis:action/PutRecords",
1811-
"requestTemplates": {"application/json": template},
1812-
"credentials": role_arn,
1813-
}
1814-
],
1815-
},
1816-
{
1817-
"httpMethod": "GET",
1818-
"authorizationType": "NONE",
1819-
"requestModels": {"application/json": "Empty"},
1820-
"integrations": [
1821-
{
1822-
"type": "AWS",
1823-
"uri": f"arn:aws:apigateway:{region_name}:kinesis:action/ListStreams",
1824-
"requestTemplates": {"application/json": "{}"},
1825-
"credentials": role_arn,
1826-
}
1827-
],
1828-
},
1829-
]
1830-
}
1831-
return resource_util.create_api_gateway(
1832-
name=gateway_name,
1833-
resources=resources,
1834-
stage_name=TEST_STAGE_NAME,
1835-
client=client,
1836-
)
1837-
18381768
def connect_api_gateway_to_http(
18391769
self, int_type, gateway_name, target_url, methods=None, path=None
18401770
):

‎tests/aws/services/apigateway/test_apigateway_kinesis.py‎

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import json
1+
import pytest
22

33
from localstack.testing.pytest import markers
44
from localstack.utils.http import safe_requests as requests
@@ -7,9 +7,35 @@
77
from tests.aws.services.apigateway.apigateway_fixtures import api_invoke_url
88
from tests.aws.services.apigateway.conftest import DEFAULT_STAGE_NAME
99

10+
KINESIS_PUT_RECORDS_INTEGRATION = """{
11+
"StreamName": "%s",
12+
"Records": [
13+
#set( $numRecords = $input.path('$.records').size() )
14+
#if($numRecords > 0)
15+
#set( $maxIndex = $numRecords - 1 )
16+
#foreach( $idx in [0..$maxIndex] )
17+
#set( $elem = $input.path("$.records[${idx}]") )
18+
#set( $elemJsonB64 = $util.base64Encode($elem.data) )
19+
{
20+
"Data": "$elemJsonB64",
21+
"PartitionKey": #if( $foo.bar.stuff != '')"$elem.partitionKey"#else"$elemJsonB64.length()"#end
22+
}#if($foreach.hasNext),#end
23+
#end
24+
#end
25+
]
26+
}"""
27+
28+
KINESIS_PUT_RECORD_INTEGRATION = """
29+
{
30+
"StreamName": "%s",
31+
"Data": "$util.base64Encode($input.body)",
32+
"PartitionKey": "test"
33+
}"""
34+
1035

1136
# PutRecord does not return EncryptionType, but it's documented as such.
1237
# xxx requires further investigation
38+
@pytest.mark.parametrize("action", ("PutRecord", "PutRecords"))
1339
@markers.snapshot.skip_snapshot_verify(paths=["$..EncryptionType", "$..ChildShards"])
1440
@markers.aws.validated
1541
def test_apigateway_to_kinesis(
@@ -19,10 +45,26 @@ def test_apigateway_to_kinesis(
1945
snapshot,
2046
region_name,
2147
aws_client,
48+
action,
2249
):
2350
snapshot.add_transformer(snapshot.transform.apigateway_api())
2451
snapshot.add_transformer(snapshot.transform.kinesis_api())
2552

53+
if action == "PutRecord":
54+
template = KINESIS_PUT_RECORD_INTEGRATION
55+
payload = {"kinesis": "snapshot"}
56+
expected_key = "SequenceNumber"
57+
else:
58+
template = KINESIS_PUT_RECORDS_INTEGRATION
59+
payload = {
60+
"records": [
61+
{"data": '{"foo": "bar1"}'},
62+
{"data": '{"foo": "bar2"}'},
63+
{"data": '{"foo": "bar3"}'},
64+
]
65+
}
66+
expected_key = "Records"
67+
2668
# create stream
2769
stream_name = f"kinesis-stream-{short_uid()}"
2870
kinesis_create_stream(StreamName=stream_name, ShardCount=1)
@@ -35,16 +77,8 @@ def test_apigateway_to_kinesis(
3577
shard_id = first_stream_shard_data["ShardId"]
3678

3779
# create REST API with Kinesis integration
38-
integration_uri = f"arn:aws:apigateway:{region_name}:kinesis:action/PutRecord"
39-
request_templates = {
40-
"application/json": json.dumps(
41-
{
42-
"StreamName": stream_name,
43-
"Data": "$util.base64Encode($input.body)",
44-
"PartitionKey": "test",
45-
}
46-
)
47-
}
80+
integration_uri = f"arn:aws:apigateway:{region_name}:kinesis:action/{action}"
81+
request_templates = {"application/json": template % stream_name}
4882
api_id = create_rest_api_with_integration(
4983
integration_uri=integration_uri,
5084
req_templates=request_templates,
@@ -53,10 +87,10 @@ def test_apigateway_to_kinesis(
5387

5488
def _invoke_apigw_to_kinesis() -> dict:
5589
url = api_invoke_url(api_id, stage=DEFAULT_STAGE_NAME, path="/test")
56-
_response = requests.post(url, json={"kinesis": "snapshot"})
90+
_response = requests.post(url, json=payload)
5791
assert _response.ok
5892
json_resp = _response.json()
59-
assert "SequenceNumber" in json_resp
93+
assert expected_key in json_resp
6094
return json_resp
6195

6296
# push events to Kinesis via API

‎tests/aws/services/apigateway/test_apigateway_kinesis.snapshot.json‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"tests/aws/services/apigateway/test_apigateway_kinesis.py::test_apigateway_to_kinesis": {
3-
"recorded-date": "12-07-2024, 20:32:13",
2+
"tests/aws/services/apigateway/test_apigateway_kinesis.py::test_apigateway_to_kinesis[PutRecord]": {
3+
"recorded-date": "20-11-2024, 05:29:53",
44
"recorded-content": {
55
"apigateway_response": {
66
"SequenceNumber": "<sequence_number:1>",
@@ -23,5 +23,55 @@
2323
}
2424
}
2525
}
26+
},
27+
"tests/aws/services/apigateway/test_apigateway_kinesis.py::test_apigateway_to_kinesis[PutRecords]": {
28+
"recorded-date": "20-11-2024, 06:33:51",
29+
"recorded-content": {
30+
"apigateway_response": {
31+
"FailedRecordCount": 0,
32+
"Records": [
33+
{
34+
"SequenceNumber": "<sequence_number:1>",
35+
"ShardId": "<shard_id:1>"
36+
},
37+
{
38+
"SequenceNumber": "<sequence_number:2>",
39+
"ShardId": "<shard_id:1>"
40+
},
41+
{
42+
"SequenceNumber": "<sequence_number:3>",
43+
"ShardId": "<shard_id:1>"
44+
}
45+
]
46+
},
47+
"kinesis_records": {
48+
"MillisBehindLatest": 0,
49+
"NextShardIterator": "<next_shard_iterator:1>",
50+
"Records": [
51+
{
52+
"ApproximateArrivalTimestamp": "timestamp",
53+
"Data": "b'{\"foo\": \"bar1\"}'",
54+
"PartitionKey": "20",
55+
"SequenceNumber": "<sequence_number:1>"
56+
},
57+
{
58+
"ApproximateArrivalTimestamp": "timestamp",
59+
"Data": "b'{\"foo\": \"bar2\"}'",
60+
"PartitionKey": "20",
61+
"SequenceNumber": "<sequence_number:2>"
62+
},
63+
{
64+
"ApproximateArrivalTimestamp": "timestamp",
65+
"Data": "b'{\"foo\": \"bar3\"}'",
66+
"PartitionKey": "20",
67+
"SequenceNumber": "<sequence_number:3>"
68+
}
69+
],
70+
"ResponseMetadata": {
71+
"HTTPHeaders": {},
72+
"HTTPStatusCode": 200
73+
}
74+
}
75+
}
2676
}
2777
}

‎tests/aws/services/apigateway/test_apigateway_kinesis.validation.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
{
2-
"tests/aws/services/apigateway/test_apigateway_kinesis.py::test_apigateway_to_kinesis": {
3-
"last_validated_date": "2024-07-12T20:32:13+00:00"
2+
"tests/aws/services/apigateway/test_apigateway_kinesis.py::test_apigateway_to_kinesis[PutRecord]": {
3+
"last_validated_date": "2024-11-20T05:29:53+00:00"
4+
},
5+
"tests/aws/services/apigateway/test_apigateway_kinesis.py::test_apigateway_to_kinesis[PutRecords]": {
6+
"last_validated_date": "2024-11-20T06:33:51+00:00"
47
}
58
}

0 commit comments

Comments
 (0)