Skip to content

Commit fa2eb28

Browse files
dipankarkush-dbdipankarkush-db
committed
Added checking for index out of range.
Fixes #430
1 parent 1c2f715 commit fa2eb28

File tree

2 files changed

+25
-17
lines changed

2 files changed

+25
-17
lines changed

‎tests/integration/assessment/test_assessment.py‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,34 +6,33 @@
66
JobsCrawler,
77
PipelinesCrawler,
88
)
9-
from databricks.labs.ucx.mixins.fixtures import make_secret_scope
109

1110
logger = logging.getLogger(__name__)
1211

1312
_PIPELINE_CONF = {
1413
"spark.hadoop.fs.azure.account.oauth2.client.id.storage_acct_1.dfs.core.windows.net": ""
15-
"pipeline_dummy_application_id",
14+
"pipeline_dummy_application_id",
1615
"spark.hadoop.fs.azure.account.oauth2.client.endpoint.storage_acct_1.dfs.core.windows.net": ""
17-
"https://login"
18-
".microsoftonline.com/directory_12345/oauth2/token",
16+
"https://login"
17+
".microsoftonline.com/directory_12345/oauth2/token",
1918
}
2019

2120
_PIPELINE_CONF_WITH_SECRET = {
2221
"fs.azure.account.oauth2.client.id.abcde.dfs.core.windows.net": "{{secrets/reallyasecret123/sp_app_client_id}}",
2322
"fs.azure.account.oauth2.client.endpoint.abcde.dfs.core.windows.net": "https://login.microsoftonline.com"
24-
"/dummy_application/token",
23+
"/dummy_application/token",
2524
}
2625

2726
_SPARK_CONF = {
2827
"spark.databricks.cluster.profile": "singleNode",
2928
"spark.master": "local[*]",
3029
"fs.azure.account.auth.type.storage_acct_1.dfs.core.windows.net": "OAuth",
3130
"fs.azure.account.oauth.provider.type.storage_acct_1.dfs.core.windows.net": "org.apache.hadoop.fs"
32-
".azurebfs.oauth2.ClientCredsTokenProvider",
31+
".azurebfs.oauth2.ClientCredsTokenProvider",
3332
"fs.azure.account.oauth2.client.id.storage_acct_1.dfs.core.windows.net": "dummy_application_id",
3433
"fs.azure.account.oauth2.client.secret.storage_acct_1.dfs.core.windows.net": "dummy",
3534
"fs.azure.account.oauth2.client.endpoint.storage_acct_1.dfs.core.windows.net": "https://login"
36-
".microsoftonline.com/directory_12345/oauth2/token",
35+
".microsoftonline.com/directory_12345/oauth2/token",
3736
}
3837

3938

@@ -143,18 +142,19 @@ def test_spn_crawler_with_pipeline_unavlbl_secret(ws, inventory_schema, make_job
143142
assert results[0].tenant_id == "directory_12345"
144143

145144

146-
def test_spn_crawler_with_available_secrets(ws, inventory_schema, make_job, make_pipeline, sql_backend,
147-
make_secret_scope):
145+
def test_spn_crawler_with_available_secrets(
146+
ws, inventory_schema, make_job, make_pipeline, sql_backend, make_secret_scope
147+
):
148148
secret_scope = make_secret_scope()
149149
secret_key = "spn_client_id"
150150
ws.secrets.put_secret(scope=secret_scope, key=secret_key, string_value="New_Application_Id")
151151
_pipeline_conf_with_avlbl_secret = {}
152-
_pipeline_conf_with_avlbl_secret["fs.azure.account.oauth2.client.id.SA1.dfs.core.windows.net"] = "{" + (f"{{"
153-
f"secrets/{secret_scope}/{secret_key}}}") + "}"
154-
_pipeline_conf_with_avlbl_secret["fs.azure.account.oauth2.client.endpoint.SA1.dfs.core.windows.net"] = (
155-
"https://login"
156-
".microsoftonline.com"
157-
"/dummy_tenant/oauth2/token")
152+
_pipeline_conf_with_avlbl_secret["fs.azure.account.oauth2.client.id.SA1.dfs.core.windows.net"] = (
153+
"{" + (f"{{secrets/{secret_scope}/{secret_key}}}") + "}"
154+
)
155+
_pipeline_conf_with_avlbl_secret[
156+
"fs.azure.account.oauth2.client.endpoint.SA1.dfs.core.windows.net"
157+
] = "https://login.microsoftonline.com/dummy_tenant/oauth2/token"
158158
make_job()
159159
make_pipeline(configuration=_pipeline_conf_with_avlbl_secret)
160160
spn_crawler = AzureServicePrincipalCrawler(ws=ws, sbe=sql_backend, schema=inventory_schema)

‎tests/unit/assessment/test_assessment.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
from tests.unit.framework.mocks import MockBackend
3838

3939
_SECRET_PATTERN = r"{{(secrets.*?)}}"
40+
_SECRET_VALUE = b"SGVsbG8sIFdvcmxkIQ=="
4041

4142

4243
def test_external_locations():
@@ -812,6 +813,7 @@ def test_azure_service_principal_info_crawl(mocker):
812813
),
813814
]
814815
ws.pipelines.get().spec.configuration = config_dict
816+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
815817
ws.cluster_policies.get().policy_family_definition_overrides = None
816818
ws.jobs.list.return_value = sample_jobs
817819
spn_crawler = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._crawl()
@@ -1440,6 +1442,7 @@ def test_list_all_cluster_with_spn_in_spark_conf_with_secret(mocker):
14401442

14411443
ws = mocker.Mock()
14421444
ws.clusters.list.return_value = sample_clusters
1445+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
14431446
ws.cluster_policies.get().policy_family_definition_overrides = None
14441447
crawler = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._list_all_cluster_with_spn_in_spark_conf()
14451448
result_set = list(crawler)
@@ -1536,6 +1539,7 @@ def test_list_all_wh_config_with_spn_and_secret(mocker):
15361539
),
15371540
]
15381541
mocker.Mock().secrets.get_secret()
1542+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
15391543
result_set = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._list_all_spn_in_sql_warehouses_spark_conf()
15401544

15411545
assert len(result_set) == 2
@@ -1567,6 +1571,7 @@ def test_list_all_clusters_spn_in_spark_conf_with_tenant(mocker):
15671571
ws = mocker.Mock()
15681572
ws.clusters.list.return_value = sample_clusters
15691573
ws.cluster_policies.get().policy_family_definition_overrides = None
1574+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
15701575
result_set = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._list_all_cluster_with_spn_in_spark_conf()
15711576

15721577
assert len(result_set) == 1
@@ -1701,6 +1706,7 @@ def test_azure_service_principal_info_policy_conf(mocker):
17011706
value="https://login.microsoftonline.com/dummy_tenant_id2/oauth2/token",
17021707
),
17031708
]
1709+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
17041710
spn_crawler = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._crawl()
17051711

17061712
assert len(spn_crawler) == 4
@@ -1777,6 +1783,7 @@ def test_azure_service_principal_info_dedupe(mocker):
17771783
config_dict = {}
17781784
ws.pipelines.get().spec.configuration = config_dict
17791785
ws.jobs.list.return_value = sample_jobs
1786+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
17801787
ws.cluster_policies.get().definition = json.dumps(
17811788
{
17821789
"spark_conf.fs.azure.account.auth.type": {"type": "fixed", "value": "OAuth", "hidden": "true"},
@@ -1926,6 +1933,7 @@ def test_list_all_pipeline_with_conf_spn_secret(mocker):
19261933
"spark.hadoop.fs.azure.sas.fixed.token.abcde.dfs.core.windows.net": "{{secrets/abcde_access/sasFixedToken}}",
19271934
}
19281935
ws.pipelines.get().spec.configuration = config_dict
1936+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
19291937
result_set = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._list_all_pipeline_with_spn_in_spark_conf()
19301938

19311939
assert len(result_set) == 1
@@ -2499,10 +2507,10 @@ def test_list_all_pipeline_with_conf_spn_secret_avlb(mocker):
24992507
"spark.hadoop.fs.azure.sas.fixed.token.abcde.dfs.core.windows.net": "{{secrets/abcde_access/sasFixedToken}}",
25002508
}
25012509
ws.pipelines.get().spec.configuration = config_dict
2502-
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value="dmlkeWEuYXNob2s=")
2510+
ws.secrets.get_secret.return_value = GetSecretResponse(key="username", value=_SECRET_VALUE)
25032511
result_set = AzureServicePrincipalCrawler(ws, MockBackend(), "ucx")._list_all_pipeline_with_spn_in_spark_conf()
25042512

25052513
assert len(result_set) > 0
2506-
assert result_set[0].get("application_id") == "dmlkeWEuYXNob2s="
2514+
assert result_set[0].get("application_id") == "Hello, World!"
25072515
assert result_set[0].get("tenant_id") == "directory_12345"
25082516
assert result_set[0].get("storage_account") == "newstorageacct"

0 commit comments

Comments
 (0)