Skip to content

Commit a1ae8f0

Browse files
simonrwsimonrw
authored
CFNv2: Standardise on test reporting (#12984)
1 parent 6cc56f7 commit a1ae8f0

33 files changed

+100
-68
lines changed

‎tests/aws/services/cloudformation/api/test_changesets.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ def test_create_change_set_missing_stackname(aws_client):
386386
)
387387

388388

389-
@skip_if_v2_provider(reason="CFNV2:Resolve")
389+
@skip_if_v2_provider("Resolve")
390390
@markers.aws.validated
391391
def test_create_change_set_with_ssm_parameter(
392392
cleanup_changesets,
@@ -634,7 +634,7 @@ def test_create_and_then_remove_non_supported_resource_change_set(deploy_cfn_tem
634634
)
635635

636636

637-
@skip_if_v2_provider(reason="test needs fixing")
637+
@skip_if_v2_provider("Test", reason="test needs fixing")
638638
@markers.aws.needs_fixing
639639
def test_create_and_then_update_refreshes_template_metadata(
640640
aws_client,

‎tests/aws/services/cloudformation/api/test_nested_stacks.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ def test_nested_stacks_conditions(deploy_cfn_template, s3_create_bucket, aws_cli
295295
assert ":" not in nested_stack["Stacks"][0]["StackName"]
296296

297297

298-
@skip_if_v2_provider(reason="CFNV2:Deletion")
298+
@skip_if_v2_provider("Deletion")
299299
@markers.aws.validated
300300
def test_deletion_of_failed_nested_stack(s3_create_bucket, aws_client, region_name, snapshot):
301301
"""

‎tests/aws/services/cloudformation/api/test_resources.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from localstack.testing.pytest import markers
99

1010

11-
@skip_if_v1_provider(reason="Not implemented for v1")
11+
@skip_if_v1_provider("Not implemented for v1")
1212
@markers.aws.validated
1313
def test_describe_non_existent_stack(aws_client, deploy_cfn_template, snapshot):
1414
with pytest.raises(ClientError) as err:
@@ -35,7 +35,7 @@ def test_describe_non_existent_resource(aws_client, deploy_cfn_template, snapsho
3535
snapshot.match("error", err.value)
3636

3737

38-
@skip_if_v1_provider(reason="Not implemented for v1")
38+
@skip_if_v1_provider("Not implemented for v1")
3939
@markers.aws.validated
4040
def test_invalid_logical_resource_id(deploy_cfn_template, snapshot):
4141
template = {

‎tests/aws/services/cloudformation/api/test_stacks.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ def test_update_stack_with_same_template_withoutchange(
295295

296296
snapshot.match("no_change_exception", ctx.value.response)
297297

298-
@skip_if_v2_provider(reason="CFNV2:Transform")
298+
@skip_if_v2_provider("Transform")
299299
@markers.aws.validated
300300
def test_update_stack_with_same_template_withoutchange_transformation(
301301
self, deploy_cfn_template, aws_client
@@ -951,7 +951,7 @@ def test_stack_deploy_order(deploy_cfn_template, aws_client, snapshot, deploy_or
951951
snapshot.match("events", filtered_events)
952952

953953

954-
@skip_if_v1_provider(reason="Not supported with v1 provider")
954+
@skip_if_v1_provider("Not supported with v1 provider")
955955
@markers.aws.validated
956956
@pytest.mark.parametrize(
957957
"deletions",
@@ -1033,7 +1033,7 @@ def test_stack_deletion_order(
10331033
snapshot.match("all-events", to_snapshot)
10341034

10351035

1036-
@skip_if_v2_provider(reason="CFNV2:DescribeStack(noecho)")
1036+
@skip_if_v2_provider("DescribeStack")
10371037
@markers.snapshot.skip_snapshot_verify(
10381038
paths=[
10391039
# TODO: this property is present in the response from LocalStack when
@@ -1184,7 +1184,6 @@ def test_stack_resource_not_found(deploy_cfn_template, aws_client, snapshot):
11841184

11851185

11861186
@markers.aws.validated
1187-
@skip_if_v2_provider(reason="Error message does not match")
11881187
def test_non_existing_stack_message(aws_client, snapshot):
11891188
with pytest.raises(botocore.exceptions.ClientError) as ex:
11901189
aws_client.cloudformation.describe_stacks(StackName="non-existing")
@@ -1193,7 +1192,7 @@ def test_non_existing_stack_message(aws_client, snapshot):
11931192
snapshot.match("Error", ex.value.response)
11941193

11951194

1196-
@skip_if_v1_provider(reason="Not implemented for V1 provider")
1195+
@skip_if_v1_provider("Not implemented for V1 provider")
11971196
@markers.aws.validated
11981197
def test_no_parameters_given(aws_client, deploy_cfn_template, snapshot):
11991198
template_path = os.path.join(

‎tests/aws/services/cloudformation/api/test_transformers.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from localstack.utils.strings import short_uid, to_bytes
1515

1616

17-
@skip_if_v2_provider(reason="transform not implemented")
17+
@skip_if_v2_provider("Transform")
1818
@markers.aws.validated
1919
@markers.snapshot.skip_snapshot_verify(paths=["$..tags"])
2020
def test_duplicate_resources(deploy_cfn_template, s3_bucket, snapshot, aws_client):
@@ -74,10 +74,8 @@ def test_duplicate_resources(deploy_cfn_template, s3_bucket, snapshot, aws_clien
7474

7575

7676
@skip_if_v2_provider(
77-
reason=(
78-
"CFNV2:AWS::Include the transformation is run however the "
79-
"physical resource id for the resource is not available"
80-
)
77+
"AWS::Include",
78+
reason="The transformation is run however the physical resource id for the resource is not available",
8179
)
8280
@markers.aws.validated
8381
def test_transformer_property_level(deploy_cfn_template, s3_bucket, aws_client, snapshot):
@@ -131,7 +129,7 @@ def test_transformer_property_level(deploy_cfn_template, s3_bucket, aws_client,
131129
snapshot.match("processed_template", processed_template)
132130

133131

134-
@skip_if_v2_provider(reason="CFNV2:Transform")
132+
@skip_if_v2_provider("Transform")
135133
@markers.aws.validated
136134
def test_transformer_individual_resource_level(deploy_cfn_template, s3_bucket, aws_client):
137135
api_spec = textwrap.dedent("""
@@ -223,7 +221,7 @@ def transform(template: str, parameters: dict[str, str] | None = None) -> Transf
223221
call_safe(lambda: aws_client.cloudformation.delete_stack(StackName=stack_id))
224222

225223

226-
@skip_if_v2_provider(reason="CFNV2:LanguageExtensions")
224+
@skip_if_v2_provider("LanguageExtensions")
227225
class TestLanguageExtensionsTransform:
228226
"""
229227
Manual testing of the language extensions trasnform

‎tests/aws/services/cloudformation/conftest.py‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import re
2+
from collections import Counter
13
from collections.abc import Iterable
24
from typing import TypeVar
35

@@ -7,17 +9,49 @@
79
from localstack.testing.aws.util import is_aws_cloud
810
from localstack.utils.collections import optional_list
911

12+
SKIP_TYPE_RE = re.compile(r"^CFNV2\((?P<reason>[^\)]+)\)")
1013

11-
def skip_if_v2_provider(reason: str):
14+
15+
def skip_if_v2_provider(*types: str, reason: str = ""):
16+
if reason:
17+
reason = f"CFNV2({','.join(types)}): {reason}"
18+
else:
19+
reason = f"CFNV2({','.join(types)})"
1220
return pytest.mark.skipif(condition=is_v2_engine() and not is_aws_cloud(), reason=reason)
1321

1422

15-
def skip_if_v1_provider(*, reason: str):
16-
return pytest.mark.skipif(condition=not is_v2_engine() and not is_aws_cloud(), reason=reason)
23+
def skip_if_v1_provider(reason: str):
24+
return pytest.mark.skipif(
25+
condition=not is_v2_engine() and not is_aws_cloud(), reason=f"CFNV1: {reason}"
26+
)
1727

1828

1929
_T = TypeVar("_T")
2030

2131

2232
def skipped_v2_items(*items: Iterable[_T]) -> list[_T]:
2333
return optional_list(is_v2_engine(), items)
34+
35+
36+
def pytest_report_collectionfinish(config, start_path, startdir, items):
37+
if not is_v2_engine():
38+
return
39+
40+
v1_items, v2_skip_types = [], []
41+
for item in items:
42+
if skip := item.get_closest_marker("skipif"):
43+
if reason := skip.kwargs.get("reason"):
44+
if "CFNV2" in reason:
45+
if match := SKIP_TYPE_RE.match(reason):
46+
v2_skip_types.append(match.group("reason"))
47+
else:
48+
v2_skip_types.append("Other")
49+
elif "CFNV1" in reason:
50+
v1_items.append(item.nodeid)
51+
header = f"CFNV2: skipped {len(v1_items)} v1 skips and {len(v2_skip_types)} v2 skips"
52+
types_count_parts = []
53+
for skip_type, count in sorted(
54+
Counter(v2_skip_types).items(), key=lambda e: e[1], reverse=True
55+
):
56+
types_count_parts.append("- " + ": ".join([f"CFNV2({skip_type})", str(count)]))
57+
return [header] + types_count_parts

‎tests/aws/services/cloudformation/engine/test_mappings.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def test_mapping_with_nonexisting_key(self, aws_client, cleanups, snapshot):
7070
)
7171
snapshot.match("mapping_nonexisting_key_exc", e.value.response)
7272

73-
@skip_if_v2_provider(reason="CFNV2:Validation replaced with v2 test below")
73+
@skip_if_v2_provider("Validation", reason="replaced with v2 test below")
7474
@markers.aws.only_localstack
7575
def test_async_mapping_error_first_level(self, deploy_cfn_template):
7676
"""
@@ -94,7 +94,7 @@ def test_async_mapping_error_first_level(self, deploy_cfn_template):
9494
assert "Cannot find map key 'C' in mapping 'TopicSuffixMap'" in str(exc_info.value)
9595

9696
@markers.aws.validated
97-
@skip_if_v1_provider(reason="V1 provider is not in parity with AWS")
97+
@skip_if_v1_provider("V1 provider is not in parity with AWS")
9898
def test_async_mapping_error_first_level_v2(self, aws_client, snapshot):
9999
snapshot.add_transformer(snapshot.transform.cloudformation_api())
100100
topic_name = f"test-topic-{short_uid()}"
@@ -120,7 +120,7 @@ def test_async_mapping_error_first_level_v2(self, aws_client, snapshot):
120120

121121
snapshot.match("error", exc_info.value)
122122

123-
@skip_if_v2_provider(reason="CFNV2:Validation replaced with v2 test below")
123+
@skip_if_v2_provider("Validation", reason="replaced with v2 test below")
124124
@markers.aws.only_localstack
125125
def test_async_mapping_error_second_level(self, deploy_cfn_template):
126126
"""
@@ -146,7 +146,7 @@ def test_async_mapping_error_second_level(self, deploy_cfn_template):
146146
)
147147

148148
@markers.aws.validated
149-
@skip_if_v1_provider(reason="V1 provider is not in parity with AWS")
149+
@skip_if_v1_provider("V1 provider is not in parity with AWS")
150150
def test_async_mapping_error_second_level_v2(self, aws_client, snapshot):
151151
"""
152152
Similar to the `test_async_mapping_error_first_level` test above, but

‎tests/aws/services/cloudformation/resource_providers/iam/aws_iam_user/test_parity.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class TestParity:
2222
- Negative test: missing required properties
2323
"""
2424

25-
@skip_if_v2_provider(reason="CFNV2:Engine possible resource dependency issue")
25+
@skip_if_v2_provider("Engine", reason="possible resource dependency issue")
2626
@markers.aws.validated
2727
@markers.snapshot.skip_snapshot_verify(
2828
paths=["$..IsTruncated"]

‎tests/aws/services/cloudformation/resource_providers/iam/test_iam.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ def test_update_inline_policy(deploy_cfn_template, snapshot, aws_client):
202202
snapshot.match("role_updated_inline_policy", role_updated_inline_policy_resource)
203203

204204

205-
@skip_if_v2_provider(reason="CFNV2:Engine Ref: AWS::NoValue")
205+
@skip_if_v2_provider("Engine", reason="Ref: AWS::NoValue")
206206
@markers.aws.validated
207207
@markers.snapshot.skip_snapshot_verify(
208208
paths=[

‎tests/aws/services/cloudformation/resources/test_apigateway.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,9 @@ def _invoke():
167167

168168

169169
@skip_if_v2_provider(
170+
"Provider",
170171
reason="The v2 provider appears to instead return the correct url: "
171-
"https://e1i3grfiws.execute-api.us-east-1.localhost.localstack.cloud/prod/"
172+
"https://e1i3grfiws.execute-api.us-east-1.localhost.localstack.cloud/prod/",
172173
)
173174
@markers.aws.only_localstack
174175
def test_url_output(httpserver, deploy_cfn_template):
@@ -330,7 +331,6 @@ def test_cfn_deploy_apigateway_integration(deploy_cfn_template, snapshot, aws_cl
330331
# TODO: snapshot the authorizer too? it's not attached to the REST API
331332

332333

333-
# @skip_if_v2_provider(reason="")
334334
@markers.aws.validated
335335
@markers.snapshot.skip_snapshot_verify(
336336
paths=[
@@ -582,7 +582,7 @@ def test_api_gateway_with_policy_as_dict(deploy_cfn_template, snapshot, aws_clie
582582

583583

584584
@skip_if_v2_provider(
585-
reason="CFNV2:Other lambda function fails on creation due to invalid function name"
585+
"Other", reason="lambda function fails on creation due to invalid function name"
586586
)
587587
@markers.aws.validated
588588
@markers.snapshot.skip_snapshot_verify(

0 commit comments

Comments
 (0)