Skip to content

Commit 9ba48cc

Browse files
mgyuchtxinjiezhen-db
mgyucht
and
xinjiezhen-db
authored
Paginate all SCIM list requests in the SDK (#440)
## Changes This PR incorporates two hard-coded changes for the SCIM API in the Python SDK: 1. startIndex starts at 1 for SCIM APIs, not 0. However, the existing .Pagination.Increment controls both the start index as well as whether the pagination is per-page or per-resource. Later, we should replace this extension with two independent OpenAPI options: `one_indexed` (defaulting to `false`) and `pagination_basis` (defaulting to `resource` but can be overridden to `page`). 2. If users don't specify a limit, the SDK will include a hard-coded limit of 100 resources per request. We could add this to the OpenAPI spec as an option `default_limit`, which is useful for any non-paginated APIs that later expose pagination options and allow the SDK to gracefully support those. However, we don't want to encourage folks to use this pattern: all new list APIs are required to be paginated from the start. ## Tests <!-- How is this tested? Please see the checklist below and also describe any other relevant tests --> - [ ] `make test` run locally - [ ] `make fmt` applied - [ ] relevant integration tests applied --------- Co-authored-by: Xinjie Zheng <[email protected]>
1 parent 50c71a1 commit 9ba48cc

File tree

4 files changed

+63
-22
lines changed

4 files changed

+63
-22
lines changed

‎.codegen/service.py.tmpl‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,16 @@ class {{.Name}}API:{{if .Description}}
247247
# deduplicate items that may have been added during iteration
248248
seen = set()
249249
{{- end}}{{if and .Pagination.Offset (not (eq .Path "/api/2.0/clusters/events")) }}
250-
query['{{.Pagination.Offset.Name}}'] = {{if eq .Pagination.Increment 1}}1{{else}}0{{end}}{{end}}
250+
query['{{.Pagination.Offset.Name}}'] =
251+
{{- if eq .Pagination.Increment 1 -}}
252+
1
253+
{{- else if contains .Path "/scim/v2/" -}}
254+
1
255+
{{- else -}}
256+
0
257+
{{- end}}{{end}}{{if and .Pagination.Limit (contains .Path "/scim/v2/")}}
258+
if "{{.Pagination.Limit.Name}}" not in query: query['{{.Pagination.Limit.Name}}'] = 100
259+
{{- end}}
251260
while True:
252261
json = {{template "method-do" .}}
253262
if '{{.Pagination.Results.Name}}' not in json or not json['{{.Pagination.Results.Name}}']:

‎databricks/sdk/service/iam.py‎

Lines changed: 12 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/integration/test_groups.py‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎tests/integration/test_iam.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import pytest
2+
3+
from databricks.sdk.core import DatabricksError
4+
5+
6+
def test_filtering_groups(w, random):
7+
all = w.groups.list(filter=f'displayName eq any-{random(12)}')
8+
found = len(list(all))
9+
assert found == 0
10+
11+
12+
def test_scim_error_unmarshall(w, random):
13+
with pytest.raises(DatabricksError) as exc_info:
14+
w.groups.list(filter=random(12))
15+
assert 'Given filter operator is not supported' in str(exc_info.value)
16+
17+
18+
@pytest.mark.parametrize(
19+
"path,call",
20+
[("/api/2.0/preview/scim/v2/Users", lambda w: w.users.list(count=10)),
21+
("/api/2.0/preview/scim/v2/Groups", lambda w: w.groups.list(count=4)),
22+
("/api/2.0/preview/scim/v2/ServicePrincipals", lambda w: w.service_principals.list(count=1)), ])
23+
def test_workspace_users_list_pagination(w, path, call):
24+
raw = w.api_client.do('GET', path)
25+
total = raw['totalResults']
26+
all = call(w)
27+
found = len(list(all))
28+
assert found == total
29+
30+
31+
@pytest.mark.parametrize(
32+
"path,call",
33+
[("/api/2.0/accounts/%s/scim/v2/Users", lambda a: a.users.list(count=3000)),
34+
("/api/2.0/accounts/%s/scim/v2/Groups", lambda a: a.groups.list(count=5)),
35+
("/api/2.0/accounts/%s/scim/v2/ServicePrincipals", lambda a: a.service_principals.list(count=1000)), ])
36+
def test_account_users_list_pagination(a, path, call):
37+
raw = a.api_client.do('GET', path.replace("%s", a.config.account_id))
38+
total = raw['totalResults']
39+
all = call(a)
40+
found = len(list(all))
41+
assert found == total

0 commit comments

Comments
 (0)