Skip to content

Commit 5aa2f49

Browse files
committed
compute: stop re-decoding server list pages in IsEmpty and NextPageURL
ServerPage.IsEmpty and ServerPage.NextPageURL both went through ExtractInto, which re-encodes the page body that pagination.PageResultFrom has already decoded and then parses it again. Together with the ExtractServers call in the EachPage handler, every page returned by servers.List was serialised three times and parsed four times, which gets expensive with many servers or large user_data. Read the servers list and servers_links from the decoded body instead and only fall back to ExtractInto when the body is not the decoded map. ExtractServers is now the only full decode per page. Add a two page List fixture, which was not covered before, and a test that counts body re-encodes. Fix: gophercloud#3867 Signed-off-by: Youssef Omar Bouden <[email protected]>
1 parent 2856661 commit 5aa2f49

4 files changed

Lines changed: 129 additions & 0 deletions

File tree

‎openstack/compute/v2/servers/results.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,13 +436,34 @@ func (r ServerPage) IsEmpty() (bool, error) {
436436
return true, nil
437437
}
438438

439+
// Body is already decoded by pagination.PageResultFrom; avoid re-encoding it.
440+
if body, ok := r.Body.(map[string]any); ok {
441+
if s, ok := body["servers"].([]any); ok {
442+
return len(s) == 0, nil
443+
}
444+
}
445+
439446
s, err := ExtractServers(r)
440447
return len(s) == 0, err
441448
}
442449

443450
// NextPageURL uses the response's embedded link reference to navigate to the
444451
// next page of results.
445452
func (r ServerPage) NextPageURL(endpointURL string) (string, error) {
453+
if body, ok := r.Body.(map[string]any); ok {
454+
var links []gophercloud.Link
455+
if raw, ok := body["servers_links"].([]any); ok {
456+
for _, l := range raw {
457+
if m, ok := l.(map[string]any); ok {
458+
href, _ := m["href"].(string)
459+
rel, _ := m["rel"].(string)
460+
links = append(links, gophercloud.Link{Href: href, Rel: rel})
461+
}
462+
}
463+
}
464+
return gophercloud.ExtractNextURL(links)
465+
}
466+
446467
var s struct {
447468
Links []gophercloud.Link `json:"servers_links"`
448469
}

‎openstack/compute/v2/servers/testing/fixtures_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,3 +1377,45 @@ func HandleServerHostnameUpdateSuccessfully(t *testing.T, fakeServer th.FakeServ
13771377
fmt.Fprint(w, SingleServerBody)
13781378
})
13791379
}
1380+
1381+
// HandleServerListPaginatedSuccessfully sets up the test server to respond to
1382+
// a server List request with two pages linked through servers_links.
1383+
func HandleServerListPaginatedSuccessfully(t *testing.T, fakeServer th.FakeServer) {
1384+
fakeServer.Mux.HandleFunc("/servers/detail", func(w http.ResponseWriter, r *http.Request) {
1385+
th.TestMethod(t, r, "GET")
1386+
th.TestHeader(t, r, "X-Auth-Token", client.TokenID)
1387+
1388+
w.Header().Add("Content-Type", "application/json")
1389+
if err := r.ParseForm(); err != nil {
1390+
t.Errorf("Failed to parse request form %v", err)
1391+
}
1392+
marker := r.Form.Get("marker")
1393+
switch marker {
1394+
case "":
1395+
fmt.Fprintf(w, `
1396+
{
1397+
"servers": [
1398+
{"id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", "name": "herp"},
1399+
{"id": "9e5476bd-a4ec-4653-93d6-72c93aa682ba", "name": "derp"}
1400+
],
1401+
"servers_links": [
1402+
{
1403+
"href": "%s/servers/detail?marker=9e5476bd-a4ec-4653-93d6-72c93aa682ba",
1404+
"rel": "next"
1405+
}
1406+
]
1407+
}
1408+
`, fakeServer.Server.URL)
1409+
case "9e5476bd-a4ec-4653-93d6-72c93aa682ba":
1410+
fmt.Fprint(w, `
1411+
{
1412+
"servers": [
1413+
{"id": "9e5476bd-a4ec-4653-93d6-72c93aa682bb", "name": "merp"}
1414+
]
1415+
}
1416+
`)
1417+
default:
1418+
t.Fatalf("/servers/detail invoked with unexpected marker=[%s]", marker)
1419+
}
1420+
})
1421+
}

‎openstack/compute/v2/servers/testing/requests_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1208,3 +1208,27 @@ func TestUpdateServerHostname(t *testing.T) {
12081208

12091209
th.CheckDeepEquals(t, ServerDerp, *actual)
12101210
}
1211+
1212+
func TestListServersPaginated(t *testing.T) {
1213+
fakeServer := th.SetupHTTP()
1214+
defer fakeServer.Teardown()
1215+
HandleServerListPaginatedSuccessfully(t, fakeServer)
1216+
1217+
var names []string
1218+
pages := 0
1219+
err := servers.List(client.ServiceClient(fakeServer), servers.ListOpts{}).EachPage(context.TODO(), func(_ context.Context, page pagination.Page) (bool, error) {
1220+
pages++
1221+
1222+
actual, err := servers.ExtractServers(page)
1223+
if err != nil {
1224+
return false, err
1225+
}
1226+
for _, s := range actual {
1227+
names = append(names, s.Name)
1228+
}
1229+
return true, nil
1230+
})
1231+
th.AssertNoErr(t, err)
1232+
th.AssertEquals(t, 2, pages)
1233+
th.AssertDeepEquals(t, []string{"herp", "derp", "merp"}, names)
1234+
}

‎openstack/compute/v2/servers/testing/results_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"github.com/gophercloud/gophercloud/v2"
1111
"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers"
12+
"github.com/gophercloud/gophercloud/v2/pagination"
1213
th "github.com/gophercloud/gophercloud/v2/testhelper"
1314
"github.com/gophercloud/gophercloud/v2/testhelper/client"
1415
"golang.org/x/crypto/ssh"
@@ -111,3 +112,44 @@ func TestListAddressesAllPages(t *testing.T) {
111112
_, err = servers.ExtractAddresses(allPages)
112113
th.AssertNoErr(t, err)
113114
}
115+
116+
type marshalCounter struct {
117+
n *int
118+
}
119+
120+
func (m marshalCounter) MarshalJSON() ([]byte, error) {
121+
*m.n++
122+
return []byte("null"), nil
123+
}
124+
125+
// ExtractInto re-encodes the whole page body before decoding it again, so
126+
// each MarshalJSON call on the probe is one extra pass over the page.
127+
func TestServerPageIsEmptyNextPageURLNoReencode(t *testing.T) {
128+
reencodes := 0
129+
page := servers.ServerPage{LinkedPageBase: pagination.LinkedPageBase{PageResult: pagination.PageResult{Result: gophercloud.Result{
130+
Body: map[string]any{
131+
"servers": []any{
132+
map[string]any{"id": "ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", "name": "herp"},
133+
},
134+
"servers_links": []any{
135+
map[string]any{"href": "http://example.com/servers/detail?marker=ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", "rel": "next"},
136+
},
137+
"probe": marshalCounter{&reencodes},
138+
},
139+
}}}}
140+
141+
empty, err := page.IsEmpty()
142+
th.AssertNoErr(t, err)
143+
th.AssertEquals(t, false, empty)
144+
145+
next, err := page.NextPageURL("")
146+
th.AssertNoErr(t, err)
147+
th.AssertEquals(t, "http://example.com/servers/detail?marker=ef079b0c-e610-4dfb-b1aa-b49f07ac48e5", next)
148+
th.AssertEquals(t, 0, reencodes)
149+
150+
actual, err := servers.ExtractServers(page)
151+
th.AssertNoErr(t, err)
152+
th.AssertEquals(t, 1, len(actual))
153+
th.AssertEquals(t, "herp", actual[0].Name)
154+
th.AssertEquals(t, 1, reencodes)
155+
}

0 commit comments

Comments
 (0)