Skip to content

Commit d33106a

Browse files
simonrwsimonrw
committed
Merge branch 'main' into cp/cfn/v2/import-exports
2 parents 81c2ab3 + 6e03b92 commit d33106a

23 files changed

+12411
-5068
lines changed

‎.github/workflows/asf-updates.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,8 @@ jobs:
9393
bin/release-helper.sh set-dep-ver awscli "==$AWSCLI_VERSION"
9494
9595
# upgrade the requirements files only for the botocore package
96-
pip install pip-tools
96+
# FIXME remove pin on pip-tools when https://github.com/jazzband/pip-tools/issues/2215 us resolved
97+
pip install "pip-tools<7.5.0"
9798
pip-compile --strip-extras --upgrade-package "botocore==$BOTOCORE_VERSION" --upgrade-package "boto3==$BOTOCORE_VERSION" --extra base-runtime -o requirements-base-runtime.txt pyproject.toml
9899
pip-compile --strip-extras --upgrade-package "botocore==$BOTOCORE_VERSION" --upgrade-package "boto3==$BOTOCORE_VERSION" --upgrade-package "awscli==$AWSCLI_VERSION" --extra runtime -o requirements-runtime.txt pyproject.toml
99100
pip-compile --strip-extras --upgrade-package "botocore==$BOTOCORE_VERSION" --upgrade-package "boto3==$BOTOCORE_VERSION" --upgrade-package "awscli==$AWSCLI_VERSION" --extra test -o requirements-test.txt pyproject.toml

‎.pre-commit-config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@
33
repos:
44
- repo: https://github.com/astral-sh/ruff-pre-commit
55
# Ruff version.
6-
rev: v0.12.5
6+
rev: v0.12.7
77
hooks:
88
- id: ruff
99
args: [--fix, --exit-non-zero-on-fix]
1010
# Run the formatter.
1111
- id: ruff-format
1212

1313
- repo: https://github.com/pre-commit/mirrors-mypy
14-
rev: v1.17.0
14+
rev: v1.17.1
1515
hooks:
1616
- id: mypy
1717
entry: bash -c 'cd localstack-core && mypy --install-types --non-interactive'

‎.test_durations‎

Lines changed: 4919 additions & 4813 deletions
Large diffs are not rendered by default.

‎Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ freeze: ## Run pip freeze -l in the virtual environment
3636
@$(VENV_RUN); pip freeze -l
3737

3838
upgrade-pinned-dependencies: venv
39-
$(VENV_RUN); $(PIP_CMD) install --upgrade pip-tools pre-commit
39+
# FIXME remove pin on pip-tools when https://github.com/jazzband/pip-tools/issues/2215 us resolved
40+
$(VENV_RUN); $(PIP_CMD) install --upgrade "pip-tools<7.5.0" pre-commit
4041
$(VENV_RUN); pip-compile --strip-extras --upgrade -o requirements-basic.txt pyproject.toml
4142
$(VENV_RUN); pip-compile --strip-extras --upgrade --extra runtime -o requirements-runtime.txt pyproject.toml
4243
$(VENV_RUN); pip-compile --strip-extras --upgrade --extra test -o requirements-test.txt pyproject.toml

‎localstack-core/localstack/aws/api/ec2/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9113,6 +9113,7 @@ class Route(TypedDict, total=False):
91139113
VpcPeeringConnectionId: Optional[String]
91149114
CoreNetworkArn: Optional[CoreNetworkArn]
91159115
OdbNetworkArn: Optional[OdbNetworkArn]
9116+
IpAddress: Optional[String]
91169117

91179118

91189119
RouteList = List[Route]
@@ -20678,6 +20679,7 @@ class TerminateClientVpnConnectionsResult(TypedDict, total=False):
2067820679

2067920680
class TerminateInstancesRequest(ServiceRequest):
2068020681
InstanceIds: InstanceIdStringList
20682+
Force: Optional[Boolean]
2068120683
SkipOsShutdown: Optional[Boolean]
2068220684
DryRun: Optional[Boolean]
2068320685

@@ -29116,6 +29118,7 @@ def terminate_instances(
2911629118
self,
2911729119
context: RequestContext,
2911829120
instance_ids: InstanceIdStringList,
29121+
force: Boolean | None = None,
2911929122
skip_os_shutdown: Boolean | None = None,
2912029123
dry_run: Boolean | None = None,
2912129124
**kwargs,

‎localstack-core/localstack/aws/api/opensearch/__init__.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
ErrorType = str
4040
GUID = str
4141
HostedZoneId = str
42+
IAMFederationRolesKey = str
43+
IAMFederationSubjectKey = str
4244
Id = str
4345
IdentityCenterApplicationARN = str
4446
IdentityCenterInstanceARN = str
@@ -836,6 +838,12 @@ class AdvancedOptionsStatus(TypedDict, total=False):
836838
DisableTimestamp = datetime
837839

838840

841+
class IAMFederationOptionsOutput(TypedDict, total=False):
842+
Enabled: Optional[Boolean]
843+
SubjectKey: Optional[IAMFederationSubjectKey]
844+
RolesKey: Optional[IAMFederationRolesKey]
845+
846+
839847
class JWTOptionsOutput(TypedDict, total=False):
840848
Enabled: Optional[Boolean]
841849
SubjectKey: Optional[String]
@@ -861,10 +869,17 @@ class AdvancedSecurityOptions(TypedDict, total=False):
861869
InternalUserDatabaseEnabled: Optional[Boolean]
862870
SAMLOptions: Optional[SAMLOptionsOutput]
863871
JWTOptions: Optional[JWTOptionsOutput]
872+
IAMFederationOptions: Optional[IAMFederationOptionsOutput]
864873
AnonymousAuthDisableDate: Optional[DisableTimestamp]
865874
AnonymousAuthEnabled: Optional[Boolean]
866875

867876

877+
class IAMFederationOptionsInput(TypedDict, total=False):
878+
Enabled: Optional[Boolean]
879+
SubjectKey: Optional[IAMFederationSubjectKey]
880+
RolesKey: Optional[IAMFederationRolesKey]
881+
882+
868883
class JWTOptionsInput(TypedDict, total=False):
869884
Enabled: Optional[Boolean]
870885
SubjectKey: Optional[SubjectKey]
@@ -894,6 +909,7 @@ class AdvancedSecurityOptionsInput(TypedDict, total=False):
894909
MasterUserOptions: Optional[MasterUserOptions]
895910
SAMLOptions: Optional[SAMLOptionsInput]
896911
JWTOptions: Optional[JWTOptionsInput]
912+
IAMFederationOptions: Optional[IAMFederationOptionsInput]
897913
AnonymousAuthEnabled: Optional[Boolean]
898914

899915

‎localstack-core/localstack/aws/api/s3control/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -857,6 +857,7 @@ class CreateAccessPointRequest(ServiceRequest):
857857
PublicAccessBlockConfiguration: Optional[PublicAccessBlockConfiguration]
858858
BucketAccountId: Optional[AccountId]
859859
Scope: Optional[Scope]
860+
Tags: Optional[TagList]
860861

861862

862863
class CreateAccessPointResult(TypedDict, total=False):
@@ -2423,6 +2424,7 @@ def create_access_point(
24232424
public_access_block_configuration: PublicAccessBlockConfiguration | None = None,
24242425
bucket_account_id: AccountId | None = None,
24252426
scope: Scope | None = None,
2427+
tags: TagList | None = None,
24262428
**kwargs,
24272429
) -> CreateAccessPointResult:
24282430
raise NotImplementedError

‎localstack-core/localstack/services/cloudformation/engine/v2/change_set_model_preproc.py‎

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,6 @@
1010
from localstack import config
1111
from localstack.aws.api.ec2 import AvailabilityZoneList, DescribeAvailabilityZonesResult
1212
from localstack.aws.connect import connect_to
13-
from localstack.services.cloudformation.engine.transformers import (
14-
Transformer,
15-
execute_macro,
16-
transformers,
17-
)
1813
from localstack.services.cloudformation.engine.v2.change_set_model import (
1914
ChangeSetEntity,
2015
ChangeType,
@@ -49,7 +44,6 @@
4944
)
5045
from localstack.services.cloudformation.stores import (
5146
exports_map,
52-
get_cloudformation_store,
5347
)
5448
from localstack.services.cloudformation.v2.entities import ChangeSet
5549
from localstack.utils.aws.arns import get_partition
@@ -612,65 +606,6 @@ def _compute_fn_not(arg: bool) -> bool:
612606
)
613607
return delta
614608

615-
def _compute_fn_transform(self, args: dict[str, Any]) -> Any:
616-
# TODO: add typing to arguments before this level.
617-
# TODO: add schema validation
618-
# TODO: add support for other transform types
619-
620-
account_id = self._change_set.account_id
621-
region_name = self._change_set.region_name
622-
transform_name: str = args.get("Name")
623-
if not isinstance(transform_name, str):
624-
raise RuntimeError("Invalid or missing Fn::Transform 'Name' argument")
625-
transform_parameters: dict = args.get("Parameters")
626-
if not isinstance(transform_parameters, dict):
627-
raise RuntimeError("Invalid or missing Fn::Transform 'Parameters' argument")
628-
629-
if transform_name in transformers:
630-
# TODO: port and refactor this 'transformers' logic to this package.
631-
builtin_transformer_class = transformers[transform_name]
632-
builtin_transformer: Transformer = builtin_transformer_class()
633-
transform_output: Any = builtin_transformer.transform(
634-
account_id=account_id, region_name=region_name, parameters=transform_parameters
635-
)
636-
return transform_output
637-
638-
macros_store = get_cloudformation_store(
639-
account_id=account_id, region_name=region_name
640-
).macros
641-
if transform_name in macros_store:
642-
# TODO: this formatting of stack parameters is odd but required to integrate with v1 execute_macro util.
643-
# consider porting this utils and passing the plain list of parameters instead.
644-
stack_parameters = {
645-
parameter["ParameterKey"]: parameter
646-
for parameter in self._change_set.stack.parameters
647-
}
648-
transform_output: Any = execute_macro(
649-
account_id=account_id,
650-
region_name=region_name,
651-
parsed_template=dict(), # TODO: review the requirements for this argument.
652-
macro=args, # TODO: review support for non dict bindings (v1).
653-
stack_parameters=stack_parameters,
654-
transformation_parameters=transform_parameters,
655-
is_intrinsic=True,
656-
)
657-
return transform_output
658-
659-
raise RuntimeError(
660-
f"Unsupported transform function '{transform_name}' in '{self._change_set.stack.stack_name}'"
661-
)
662-
663-
def visit_node_intrinsic_function_fn_transform(
664-
self, node_intrinsic_function: NodeIntrinsicFunction
665-
) -> PreprocEntityDelta:
666-
arguments_delta = self.visit(node_intrinsic_function.arguments)
667-
delta = self._cached_apply(
668-
scope=node_intrinsic_function.scope,
669-
arguments_delta=arguments_delta,
670-
resolver=self._compute_fn_transform,
671-
)
672-
return delta
673-
674609
def visit_node_intrinsic_function_fn_sub(
675610
self, node_intrinsic_function: NodeIntrinsicFunction
676611
) -> PreprocEntityDelta:

‎localstack-core/localstack/services/cloudformation/engine/v2/change_set_model_transform.py‎

Lines changed: 51 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@
44
from typing import Any, Final, Optional, TypedDict
55

66
import boto3
7+
from botocore.exceptions import ClientError
78
from samtranslator.translator.transform import transform as transform_sam
89

10+
from localstack.aws.connect import connect_to
911
from localstack.services.cloudformation.engine.policy_loader import create_policy_loader
12+
from localstack.services.cloudformation.engine.template_preparer import parse_template
1013
from localstack.services.cloudformation.engine.transformers import (
1114
FailedTransformationException,
1215
execute_macro,
@@ -27,12 +30,14 @@
2730
)
2831
from localstack.services.cloudformation.stores import get_cloudformation_store
2932
from localstack.services.cloudformation.v2.entities import ChangeSet
33+
from localstack.utils import testutil
3034

3135
LOG = logging.getLogger(__name__)
3236

3337
SERVERLESS_TRANSFORM = "AWS::Serverless-2016-10-31"
3438
EXTENSIONS_TRANSFORM = "AWS::LanguageExtensions"
3539
SECRETSMANAGER_TRANSFORM = "AWS::SecretsManager-2020-07-23"
40+
INCLUDE_TRANSFORM = "AWS::Include"
3641

3742
_SCOPE_TRANSFORM_TEMPLATE_OUTCOME: Final[Scope] = Scope("TRANSFORM_TEMPLATE_OUTCOME")
3843

@@ -138,6 +143,32 @@ def _apply_global_serverless_transformation(
138143
if region_before is not None:
139144
os.environ["AWS_DEFAULT_REGION"] = region_before
140145

146+
@staticmethod
147+
def _apply_global_include(
148+
global_transform: GlobalTransform, template: dict, parameters: dict, account_id, region_name
149+
) -> dict:
150+
location = global_transform.parameters.get("Location")
151+
if not location or not location.startswith("s3://"):
152+
raise FailedTransformationException(
153+
transformation=INCLUDE_TRANSFORM,
154+
message="Unexpected Location parameter for AWS::Include transformer: %s" % location,
155+
)
156+
157+
s3_client = connect_to(aws_access_key_id=account_id, region_name=region_name).s3
158+
bucket, _, path = location.removeprefix("s3://").partition("/")
159+
try:
160+
content = testutil.download_s3_object(s3_client, bucket, path)
161+
except ClientError:
162+
raise FailedTransformationException(
163+
transformation=INCLUDE_TRANSFORM,
164+
message="Error downloading S3 object '%s/%s'" % (bucket, path),
165+
)
166+
try:
167+
template_to_include = parse_template(content)
168+
except Exception as e:
169+
raise FailedTransformationException(transformation=INCLUDE_TRANSFORM, message=str(e))
170+
return {**template, **template_to_include}
171+
141172
@staticmethod
142173
def _apply_global_macro_transformation(
143174
account_id: str,
@@ -182,6 +213,14 @@ def _apply_global_transform(
182213
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/transform-aws-secretsmanager.html
183214
LOG.warning("%s is not yet supported. Ignoring.", SECRETSMANAGER_TRANSFORM)
184215
transformed_template = template
216+
elif transform_name == INCLUDE_TRANSFORM:
217+
transformed_template = self._apply_global_include(
218+
global_transform=global_transform,
219+
region_name=self._change_set.region_name,
220+
account_id=self._change_set.account_id,
221+
template=template,
222+
parameters=parameters,
223+
)
185224
else:
186225
transformed_template = self._apply_global_macro_transformation(
187226
account_id=self._change_set.account_id,
@@ -213,11 +252,12 @@ def transform(self) -> tuple[dict, dict]:
213252
if not transformed_before_template:
214253
transformed_before_template = self._before_template
215254
for before_global_transform in transform_before:
216-
transformed_before_template = self._apply_global_transform(
217-
global_transform=before_global_transform,
218-
parameters=parameters_before,
219-
template=transformed_before_template,
220-
)
255+
if not is_nothing(before_global_transform.name):
256+
transformed_before_template = self._apply_global_transform(
257+
global_transform=before_global_transform,
258+
parameters=parameters_before,
259+
template=transformed_before_template,
260+
)
221261
self._before_cache[_SCOPE_TRANSFORM_TEMPLATE_OUTCOME] = transformed_before_template
222262

223263
transformed_after_template = self._after_template
@@ -226,11 +266,12 @@ def transform(self) -> tuple[dict, dict]:
226266
if not transformed_after_template:
227267
transformed_after_template = self._after_template
228268
for after_global_transform in transform_after:
229-
transformed_after_template = self._apply_global_transform(
230-
global_transform=after_global_transform,
231-
parameters=parameters_after,
232-
template=transformed_after_template,
233-
)
269+
if not is_nothing(after_global_transform.name):
270+
transformed_after_template = self._apply_global_transform(
271+
global_transform=after_global_transform,
272+
parameters=parameters_after,
273+
template=transformed_after_template,
274+
)
234275
self._after_cache[_SCOPE_TRANSFORM_TEMPLATE_OUTCOME] = transformed_after_template
235276

236277
self._save_runtime_cache()

‎localstack-core/localstack/testing/pytest/cloudformation/fixtures.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ def inner(
131131
ChangeSetName=change_set_name,
132132
TemplateBody=t1,
133133
ChangeSetType="CREATE",
134+
Capabilities=[
135+
"CAPABILITY_IAM",
136+
"CAPABILITY_NAMED_IAM",
137+
"CAPABILITY_AUTO_EXPAND",
138+
],
134139
Parameters=[{"ParameterKey": k, "ParameterValue": v} for (k, v) in p1.items()],
135140
)
136141
snapshot.match("create-change-set-1", change_set_details)
@@ -189,6 +194,11 @@ def inner(
189194
TemplateBody=t2,
190195
ChangeSetType="UPDATE",
191196
Parameters=[{"ParameterKey": k, "ParameterValue": v} for (k, v) in p2.items()],
197+
Capabilities=[
198+
"CAPABILITY_IAM",
199+
"CAPABILITY_NAMED_IAM",
200+
"CAPABILITY_AUTO_EXPAND",
201+
],
192202
)
193203
snapshot.match("create-change-set-2", change_set_details)
194204
stack_id = change_set_details["StackId"]

0 commit comments

Comments
 (0)