Skip to content

Commit 8e3cbeb

Browse files
authored
Glue: Add MLTransform support and tagging for DevEndpoints/Workflows (#10181)
1 parent 62bf212 commit 8e3cbeb

5 files changed

Lines changed: 318 additions & 8 deletions

File tree

‎IMPLEMENTATION_COVERAGE.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5235,7 +5235,7 @@
52355235
- [ ] create_integration_resource_property
52365236
- [ ] create_integration_table_properties
52375237
- [X] create_job
5238-
- [ ] create_ml_transform
5238+
- [X] create_ml_transform
52395239
- [X] create_partition
52405240
- [ ] create_partition_index
52415241
- [X] create_registry
@@ -5267,7 +5267,7 @@
52675267
- [ ] delete_integration_resource_property
52685268
- [ ] delete_integration_table_properties
52695269
- [X] delete_job
5270-
- [ ] delete_ml_transform
5270+
- [X] delete_ml_transform
52715271
- [X] delete_partition
52725272
- [ ] delete_partition_index
52735273
- [X] delete_registry
@@ -5332,8 +5332,8 @@
53325332
- [ ] get_materialized_view_refresh_task_run
53335333
- [ ] get_ml_task_run
53345334
- [ ] get_ml_task_runs
5335-
- [ ] get_ml_transform
5336-
- [ ] get_ml_transforms
5335+
- [X] get_ml_transform
5336+
- [X] get_ml_transforms
53375337
- [X] get_partition
53385338
- [ ] get_partition_indexes
53395339
- [X] get_partitions

‎docs/docs/services/glue.rst‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ glue
5252
- [ ] create_integration_resource_property
5353
- [ ] create_integration_table_properties
5454
- [X] create_job
55-
- [ ] create_ml_transform
55+
- [X] create_ml_transform
5656
- [X] create_partition
5757
- [ ] create_partition_index
5858
- [X] create_registry
@@ -88,7 +88,7 @@ The following parameters/features are not yet implemented: Glue Schema Registry:
8888
- [ ] delete_integration_resource_property
8989
- [ ] delete_integration_table_properties
9090
- [X] delete_job
91-
- [ ] delete_ml_transform
91+
- [X] delete_ml_transform
9292
- [X] delete_partition
9393
- [ ] delete_partition_index
9494
- [X] delete_registry
@@ -153,8 +153,8 @@ The following parameters/features are not yet implemented: Glue Schema Registry:
153153
- [ ] get_materialized_view_refresh_task_run
154154
- [ ] get_ml_task_run
155155
- [ ] get_ml_task_runs
156-
- [ ] get_ml_transform
157-
- [ ] get_ml_transforms
156+
- [X] get_ml_transform
157+
- [X] get_ml_transforms
158158
- [X] get_partition
159159
- [ ] get_partition_indexes
160160
- [X] get_partitions

‎moto/glue/models.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,70 @@ def as_dict(self) -> dict[str, Any]:
153153
return response
154154

155155

156+
class FakeMLTransform(BaseModel):
157+
def __init__(
158+
self,
159+
name: str,
160+
input_record_tables: list[dict[str, Any]],
161+
parameters: dict[str, Any],
162+
role: str,
163+
backend: "GlueBackend",
164+
description: str | None = None,
165+
glue_version: str | None = None,
166+
max_capacity: float | None = None,
167+
worker_type: str | None = None,
168+
number_of_workers: int | None = None,
169+
timeout: int | None = None,
170+
max_retries: int | None = None,
171+
transform_encryption: dict[str, Any] | None = None,
172+
):
173+
self.transform_id = f"tfm-{mock_random.get_random_hex(32)}"
174+
self.name = name
175+
self.input_record_tables = input_record_tables or []
176+
self.parameters = parameters or {}
177+
self.role = role
178+
self.description = description
179+
self.glue_version = glue_version or "1.0"
180+
self.max_capacity = max_capacity
181+
self.worker_type = worker_type
182+
self.number_of_workers = number_of_workers
183+
self.timeout = timeout or 2880
184+
self.max_retries = max_retries if max_retries is not None else 0
185+
self.transform_encryption = transform_encryption
186+
self.status = "READY"
187+
self.created_on = utcnow()
188+
self.last_modified_on = self.created_on
189+
self.label_count = 0
190+
self.backend = backend
191+
self.arn = f"arn:{get_partition(backend.region_name)}:glue:{backend.region_name}:{backend.account_id}:mlTransform/{self.transform_id}"
192+
193+
def as_dict(self) -> dict[str, Any]:
194+
response: dict[str, Any] = {
195+
"TransformId": self.transform_id,
196+
"Name": self.name,
197+
"Description": self.description,
198+
"Status": self.status,
199+
"CreatedOn": self.created_on,
200+
"LastModifiedOn": self.last_modified_on,
201+
"InputRecordTables": self.input_record_tables,
202+
"Parameters": self.parameters,
203+
"Role": self.role,
204+
"GlueVersion": self.glue_version,
205+
"Timeout": self.timeout,
206+
"MaxRetries": self.max_retries,
207+
"LabelCount": self.label_count,
208+
}
209+
if self.max_capacity is not None:
210+
response["MaxCapacity"] = self.max_capacity
211+
if self.worker_type is not None:
212+
response["WorkerType"] = self.worker_type
213+
if self.number_of_workers is not None:
214+
response["NumberOfWorkers"] = self.number_of_workers
215+
if self.transform_encryption is not None:
216+
response["TransformEncryption"] = self.transform_encryption
217+
return response
218+
219+
156220
class GlueBackend(BaseBackend, TaggableResourcesMixin):
157221
SERVICE_NAMESPACE = "glue"
158222

@@ -229,6 +293,12 @@ class GlueBackend(BaseBackend, TaggableResourcesMixin):
229293
"limit_default": 100,
230294
"unique_attribute": "workflow_run_id",
231295
},
296+
"get_ml_transforms": {
297+
"input_token": "next_token",
298+
"limit_key": "max_results",
299+
"limit_default": 100,
300+
"unique_attribute": "transform_id",
301+
},
232302
}
233303

234304
def __init__(self, region_name: str, account_id: str):
@@ -251,6 +321,7 @@ def __init__(self, region_name: str, account_id: str):
251321
self.num_schemas = 0
252322
self.num_schema_versions = 0
253323
self.dev_endpoints: dict[str, FakeDevEndpoint] = OrderedDict()
324+
self.ml_transforms: dict[str, FakeMLTransform] = OrderedDict()
254325
self.data_catalog_encryption_settings: dict[str, dict[str, Any]] = {}
255326
self.resource_policies: dict[str, dict[str, Any]] = {}
256327
self.default_catalog_arn = f"arn:{get_partition(self.region_name)}:glue:{self.region_name}:{self.account_id}:catalog"
@@ -1319,6 +1390,8 @@ def create_dev_endpoint(
13191390
dev_endpoint.public_keys = public_keys
13201391

13211392
self.dev_endpoints[endpoint_name] = dev_endpoint
1393+
resource_arn = f"arn:{get_partition(self.region_name)}:glue:{self.region_name}:{self.account_id}:devEndpoint/{endpoint_name}"
1394+
self.tag_resource(resource_arn, tags)
13221395
return dev_endpoint
13231396

13241397
def get_dev_endpoint(self, endpoint_name: str) -> FakeDevEndpoint:
@@ -1332,6 +1405,60 @@ def delete_dev_endpoint(self, endpoint_name: str) -> None:
13321405
del self.dev_endpoints[endpoint_name]
13331406
except KeyError:
13341407
raise EntityNotFoundException(f"DevEndpoint {endpoint_name} not found")
1408+
resource_arn = f"arn:{get_partition(self.region_name)}:glue:{self.region_name}:{self.account_id}:devEndpoint/{endpoint_name}"
1409+
self.tagger.delete_all_tags_for_resource(resource_arn)
1410+
1411+
def create_ml_transform(
1412+
self,
1413+
name: str,
1414+
input_record_tables: list[dict[str, Any]],
1415+
parameters: dict[str, Any],
1416+
role: str,
1417+
description: str | None = None,
1418+
glue_version: str | None = None,
1419+
max_capacity: float | None = None,
1420+
worker_type: str | None = None,
1421+
number_of_workers: int | None = None,
1422+
timeout: int | None = None,
1423+
max_retries: int | None = None,
1424+
tags: dict[str, str] | None = None,
1425+
transform_encryption: dict[str, Any] | None = None,
1426+
) -> str:
1427+
ml_transform = FakeMLTransform(
1428+
name=name,
1429+
input_record_tables=input_record_tables,
1430+
parameters=parameters,
1431+
role=role,
1432+
description=description,
1433+
glue_version=glue_version,
1434+
max_capacity=max_capacity,
1435+
worker_type=worker_type,
1436+
number_of_workers=number_of_workers,
1437+
timeout=timeout,
1438+
max_retries=max_retries,
1439+
transform_encryption=transform_encryption,
1440+
backend=self,
1441+
)
1442+
self.ml_transforms[ml_transform.transform_id] = ml_transform
1443+
self.tag_resource(ml_transform.arn, tags)
1444+
return ml_transform.transform_id
1445+
1446+
def get_ml_transform(self, transform_id: str) -> "FakeMLTransform":
1447+
try:
1448+
return self.ml_transforms[transform_id]
1449+
except KeyError:
1450+
raise EntityNotFoundException(f"MLTransform {transform_id} not found")
1451+
1452+
@paginate(pagination_model=PAGINATION_MODEL)
1453+
def get_ml_transforms(self) -> list["FakeMLTransform"]:
1454+
return list(self.ml_transforms.values())
1455+
1456+
def delete_ml_transform(self, transform_id: str) -> str:
1457+
if transform_id not in self.ml_transforms:
1458+
raise EntityNotFoundException(f"MLTransform {transform_id} not found")
1459+
ml_transform = self.ml_transforms.pop(transform_id)
1460+
self.tagger.delete_all_tags_for_resource(ml_transform.arn)
1461+
return transform_id
13351462

13361463
def create_connection(
13371464
self,
@@ -1516,6 +1643,8 @@ def create_workflow(
15161643
self.workflows[name] = FakeWorkflow(
15171644
name, default_run_properties, description, max_concurrent_runs, tags
15181645
)
1646+
resource_arn = f"arn:{get_partition(self.region_name)}:glue:{self.region_name}:{self.account_id}:workflow/{name}"
1647+
self.tag_resource(resource_arn, tags)
15191648
return name
15201649

15211650
def get_workflow(
@@ -1558,6 +1687,8 @@ def delete_workflow(
15581687
) -> str:
15591688
if self.workflows.get(name):
15601689
del self.workflows[name]
1690+
resource_arn = f"arn:{get_partition(self.region_name)}:glue:{self.region_name}:{self.account_id}:workflow/{name}"
1691+
self.tagger.delete_all_tags_for_resource(resource_arn)
15611692
return name
15621693

15631694
def get_workflow_run(

‎moto/glue/responses.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -870,6 +870,61 @@ def delete_dev_endpoint(self) -> ActionResult:
870870
self.glue_backend.delete_dev_endpoint(endpoint_name)
871871
return EmptyResult()
872872

873+
def create_ml_transform(self) -> ActionResult:
874+
name = self._get_param("Name")
875+
input_record_tables = self._get_param("InputRecordTables")
876+
parameters = self._get_param("Parameters")
877+
role = self._get_param("Role")
878+
description = self._get_param("Description")
879+
glue_version = self._get_param("GlueVersion")
880+
max_capacity = self._get_param("MaxCapacity")
881+
worker_type = self._get_param("WorkerType")
882+
number_of_workers = self._get_int_param("NumberOfWorkers")
883+
timeout = self._get_int_param("Timeout")
884+
max_retries = self._get_int_param("MaxRetries")
885+
tags = self._get_param("Tags")
886+
transform_encryption = self._get_param("TransformEncryption")
887+
888+
transform_id = self.glue_backend.create_ml_transform(
889+
name=name,
890+
input_record_tables=input_record_tables,
891+
parameters=parameters,
892+
role=role,
893+
description=description,
894+
glue_version=glue_version,
895+
max_capacity=max_capacity,
896+
worker_type=worker_type,
897+
number_of_workers=number_of_workers,
898+
timeout=timeout,
899+
max_retries=max_retries,
900+
tags=tags,
901+
transform_encryption=transform_encryption,
902+
)
903+
return ActionResult({"TransformId": transform_id})
904+
905+
def get_ml_transform(self) -> ActionResult:
906+
transform_id = self._get_param("TransformId")
907+
ml_transform = self.glue_backend.get_ml_transform(transform_id)
908+
return ActionResult(ml_transform.as_dict())
909+
910+
def get_ml_transforms(self) -> ActionResult:
911+
next_token = self._get_param("NextToken")
912+
max_results = self._get_int_param("MaxResults")
913+
ml_transforms, next_token = self.glue_backend.get_ml_transforms(
914+
next_token=next_token, max_results=max_results
915+
)
916+
return ActionResult(
917+
{
918+
"Transforms": [transform.as_dict() for transform in ml_transforms],
919+
"NextToken": next_token,
920+
}
921+
)
922+
923+
def delete_ml_transform(self) -> ActionResult:
924+
transform_id = self._get_param("TransformId")
925+
transform_id = self.glue_backend.delete_ml_transform(transform_id)
926+
return ActionResult({"TransformId": transform_id})
927+
873928
def create_connection(self) -> ActionResult:
874929
catalog_id = self._get_param("CatalogId")
875930
connection_input = self._get_param("ConnectionInput")

0 commit comments

Comments
 (0)