Skip to content

Commit 6dc620b

Browse files
nfxnfx
authored
Decouple PermissionsManager from GroupMigrationToolkit (#407)
1 parent f391092 commit 6dc620b

File tree

4 files changed

+114
-13
lines changed

4 files changed

+114
-13
lines changed

‎src/databricks/labs/ucx/runtime.py‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
from databricks.labs.ucx.hive_metastore import GrantsCrawler, TablesCrawler
1212
from databricks.labs.ucx.hive_metastore.data_objects import ExternalLocationCrawler
1313
from databricks.labs.ucx.hive_metastore.mounts import Mounts
14-
from databricks.labs.ucx.workspace_access import GroupMigrationToolkit
14+
from databricks.labs.ucx.workspace_access.groups import GroupManager
15+
from databricks.labs.ucx.workspace_access.manager import PermissionManager
1516

1617
logger = logging.getLogger(__name__)
1718

@@ -132,9 +133,16 @@ def crawl_permissions(cfg: WorkspaceConfig):
132133
custom access configurations, and any specialized policies governing resource access. The results of this
133134
meticulous scan are methodically stored within the `$inventory.permissions` table, which serves as a central
134135
repository for preserving and managing these crucial access control details."""
135-
toolkit = GroupMigrationToolkit(cfg)
136-
toolkit.cleanup_inventory_table()
137-
toolkit.inventorize_permissions()
136+
ws = WorkspaceClient(config=cfg.to_databricks_config())
137+
permission_manager = PermissionManager.factory(
138+
ws,
139+
RuntimeBackend(),
140+
cfg.inventory_database,
141+
num_threads=cfg.num_threads,
142+
workspace_start_path=cfg.workspace_start_path,
143+
)
144+
permission_manager.cleanup()
145+
permission_manager.inventorize_permissions()
138146

139147

140148
@task(
@@ -188,21 +196,32 @@ def migrate_permissions(cfg: WorkspaceConfig):
188196
organization.
189197
190198
See [interactive tutorial here](https://app.getreprise.com/launch/myM3VNn/)."""
191-
toolkit = GroupMigrationToolkit(cfg)
192-
toolkit.prepare_environment()
193-
if toolkit.has_groups():
194-
toolkit.apply_permissions_to_backup_groups()
195-
toolkit.replace_workspace_groups_with_account_groups()
196-
toolkit.apply_permissions_to_account_groups()
197-
else:
199+
ws = WorkspaceClient(config=cfg.to_databricks_config())
200+
group_manager = GroupManager(ws, cfg.groups)
201+
group_manager.prepare_groups_in_environment()
202+
if not group_manager.has_groups():
198203
logger.info("Skipping group migration as no groups were found.")
204+
return
205+
206+
permission_manager = PermissionManager.factory(
207+
ws,
208+
RuntimeBackend(),
209+
cfg.inventory_database,
210+
num_threads=cfg.num_threads,
211+
workspace_start_path=cfg.workspace_start_path,
212+
)
213+
214+
permission_manager.apply_group_permissions(group_manager.migration_groups_provider, destination="backup")
215+
group_manager.replace_workspace_groups_with_account_groups()
216+
permission_manager.apply_group_permissions(group_manager.migration_groups_provider, destination="account")
199217

200218

201219
@task("migrate-groups-cleanup", depends_on=[migrate_permissions])
202220
def delete_backup_groups(cfg: WorkspaceConfig):
203221
"""Removes workspace-level backup groups"""
204-
toolkit = GroupMigrationToolkit(cfg)
205-
toolkit.delete_backup_groups()
222+
ws = WorkspaceClient(config=cfg.to_databricks_config())
223+
group_manager = GroupManager(ws, cfg.groups)
224+
group_manager.delete_backup_groups()
206225

207226

208227
@task("destroy-schema")

‎src/databricks/labs/ucx/workspace_access/manager.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
import logging
2+
import os
23
from collections.abc import Callable, Iterator
34
from itertools import groupby
45
from typing import Literal
56

7+
from databricks.sdk import WorkspaceClient
8+
from databricks.sdk.service import sql
9+
610
from databricks.labs.ucx.framework.crawlers import CrawlerBase, SqlBackend
711
from databricks.labs.ucx.framework.parallel import Threads
12+
from databricks.labs.ucx.workspace_access import generic, redash, scim, secrets
813
from databricks.labs.ucx.workspace_access.base import Applier, Crawler, Permissions
914
from databricks.labs.ucx.workspace_access.groups import GroupMigrationState
1015

@@ -19,6 +24,74 @@ def __init__(
1924
self._crawlers = crawlers
2025
self._appliers = appliers
2126

27+
@classmethod
28+
def factory(
29+
cls,
30+
ws: WorkspaceClient,
31+
sql_backend: SqlBackend,
32+
inventory_database: str,
33+
*,
34+
num_threads: int | None = None,
35+
workspace_start_path: str = "/",
36+
) -> "PermissionManager":
37+
if num_threads is None:
38+
num_threads = os.cpu_count() * 2
39+
generic_acl_listing = [
40+
generic.listing_wrapper(ws.clusters.list, "cluster_id", "clusters"),
41+
generic.listing_wrapper(ws.cluster_policies.list, "policy_id", "cluster-policies"),
42+
generic.listing_wrapper(ws.instance_pools.list, "instance_pool_id", "instance-pools"),
43+
generic.listing_wrapper(ws.warehouses.list, "id", "sql/warehouses"),
44+
generic.listing_wrapper(ws.jobs.list, "job_id", "jobs"),
45+
generic.listing_wrapper(ws.pipelines.list_pipelines, "pipeline_id", "pipelines"),
46+
generic.listing_wrapper(generic.experiments_listing(ws), "experiment_id", "experiments"),
47+
generic.listing_wrapper(generic.models_listing(ws), "id", "registered-models"),
48+
generic.workspace_listing(ws, num_threads=num_threads, start_path=workspace_start_path),
49+
generic.authorization_listing(),
50+
]
51+
redash_acl_listing = [
52+
redash.redash_listing_wrapper(ws.alerts.list, sql.ObjectTypePlural.ALERTS),
53+
redash.redash_listing_wrapper(ws.dashboards.list, sql.ObjectTypePlural.DASHBOARDS),
54+
redash.redash_listing_wrapper(ws.queries.list, sql.ObjectTypePlural.QUERIES),
55+
]
56+
generic_support = generic.GenericPermissionsSupport(ws, generic_acl_listing)
57+
sql_support = redash.SqlPermissionsSupport(ws, redash_acl_listing)
58+
secrets_support = secrets.SecretScopesSupport(ws)
59+
scim_support = scim.ScimSupport(ws)
60+
return cls(
61+
sql_backend,
62+
inventory_database,
63+
[generic_support, sql_support, secrets_support, scim_support],
64+
cls._object_type_appliers(generic_support, sql_support, secrets_support, scim_support),
65+
)
66+
67+
@staticmethod
68+
def _object_type_appliers(generic_support, sql_support, secrets_support, scim_support):
69+
return {
70+
# SCIM-based API
71+
"entitlements": scim_support,
72+
"roles": scim_support,
73+
# Generic Permissions API
74+
"authorization": generic_support,
75+
"clusters": generic_support,
76+
"cluster-policies": generic_support,
77+
"instance-pools": generic_support,
78+
"sql/warehouses": generic_support,
79+
"jobs": generic_support,
80+
"pipelines": generic_support,
81+
"experiments": generic_support,
82+
"registered-models": generic_support,
83+
"notebooks": generic_support,
84+
"files": generic_support,
85+
"directories": generic_support,
86+
"repos": generic_support,
87+
# Redash equivalent of Generic Permissions API
88+
"alerts": sql_support,
89+
"queries": sql_support,
90+
"dashboards": sql_support,
91+
# Secret Scope ACL API
92+
"secrets": secrets_support,
93+
}
94+
2295
def inventorize_permissions(self):
2396
logger.debug("Crawling permissions")
2497
crawler_tasks = list(self._get_crawler_tasks())

‎src/databricks/labs/ucx/workspace_access/migration.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from databricks.labs.ucx.workspace_access.verification import VerificationManager
2929

3030

31+
# TODO: fully remove this obsolete class later
3132
class GroupMigrationToolkit:
3233
def __init__(self, config: WorkspaceConfig, *, warehouse_id=None):
3334
self._configure_logger(config.log_level)
@@ -65,6 +66,8 @@ def __init__(self, config: WorkspaceConfig, *, warehouse_id=None):
6566
self._object_type_appliers(generic_support, sql_support, secrets_support, scim_support),
6667
)
6768
self._group_manager = GroupManager(ws, config.groups)
69+
# TODO: remove VerificationManager in scope of https://github.com/databrickslabs/ucx/issues/36,
70+
# where we probably should add verify() abstract method to every Applier
6871
self._verification_manager = VerificationManager(ws, secrets_support)
6972

7073
@staticmethod

‎tests/unit/workspace_access/test_manager.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,3 +170,9 @@ def test_unregistered_support():
170170
)
171171
pm = PermissionManager(b, "test", [], {})
172172
pm.apply_group_permissions(migration_state=MagicMock(), destination="backup")
173+
174+
175+
def test_factory(mocker):
176+
ws = mocker.Mock()
177+
b = MockBackend()
178+
PermissionManager.factory(ws, b, "test")

0 commit comments

Comments
 (0)