Skip to content

Commit 68df8dc

Browse files
DrisDaryDrisDary
authored
Implement pagination to localstacks SQS list_queues method (#12397)
1 parent 7fdfe01 commit 68df8dc

File tree

10 files changed

+145
-12
lines changed

10 files changed

+145
-12
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
htmlcov
88
*.orig
99

10+
# ignore .vs files that store temproray cache of visual studio workspace settings
11+
.vs
12+
1013
.cache
1114
.filesystem
1215
/infra/

‎localstack-core/localstack/services/sqs/constants.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,9 @@
4848
# HTTP headers used to override internal SQS ReceiveMessage
4949
HEADER_LOCALSTACK_SQS_OVERRIDE_MESSAGE_COUNT = "x-localstack-sqs-override-message-count"
5050
HEADER_LOCALSTACK_SQS_OVERRIDE_WAIT_TIME_SECONDS = "x-localstack-sqs-override-wait-time-seconds"
51+
52+
# response includes a default maximum of 1,000 results
53+
MAX_RESULT_LIMIT = 1000
54+
55+
# SQS string seed value for uuid generation
56+
SQS_UUID_STRING_SEED = "123e4567-e89b-12d3-a456-426614174000"

‎localstack-core/localstack/services/sqs/provider.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
from localstack.services.sqs.constants import (
8181
HEADER_LOCALSTACK_SQS_OVERRIDE_MESSAGE_COUNT,
8282
HEADER_LOCALSTACK_SQS_OVERRIDE_WAIT_TIME_SECONDS,
83+
MAX_RESULT_LIMIT,
8384
)
8485
from localstack.services.sqs.exceptions import (
8586
InvalidParameterValueException,
@@ -101,6 +102,7 @@
101102
is_fifo_queue,
102103
is_message_deduplication_id_required,
103104
parse_queue_url,
105+
token_generator,
104106
)
105107
from localstack.services.stores import AccountRegionBundle
106108
from localstack.utils.aws.arns import parse_arn
@@ -111,6 +113,7 @@
111113
publish_sqs_metric,
112114
publish_sqs_metric_batch,
113115
)
116+
from localstack.utils.collections import PaginatedList
114117
from localstack.utils.run import FuncThread
115118
from localstack.utils.scheduler import Scheduler
116119
from localstack.utils.strings import md5
@@ -994,17 +997,17 @@ def list_queues(
994997
else:
995998
urls = [queue.url(context) for queue in store.queues.values()]
996999

997-
if max_results:
998-
# FIXME: also need to solve pagination with stateful iterators: If the total number of items available is
999-
# more than the value specified, a NextToken is provided in the command's output. To resume pagination,
1000-
# provide the NextToken value in the starting-token argument of a subsequent command. Do not use the
1001-
# NextToken response element directly outside of the AWS CLI.
1002-
urls = urls[:max_results]
1000+
paginated_list = PaginatedList(urls)
1001+
1002+
page_size = max_results if max_results else MAX_RESULT_LIMIT
1003+
paginated_urls, next_token = paginated_list.get_page(
1004+
token_generator=token_generator, next_token=next_token, page_size=page_size
1005+
)
10031006

10041007
if len(urls) == 0:
10051008
return ListQueuesResult()
10061009

1007-
return ListQueuesResult(QueueUrls=urls)
1010+
return ListQueuesResult(QueueUrls=paginated_urls, NextToken=next_token)
10081011

10091012
def change_message_visibility(
10101013
self,

‎localstack-core/localstack/services/sqs/utils.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,9 @@ def global_message_sequence():
175175

176176
def generate_message_id():
177177
return long_uid()
178+
179+
180+
def token_generator(item: str) -> str:
181+
base64_bytes = base64.b64encode(item.encode("utf-8"))
182+
next_token = base64_bytes.decode("utf-8")
183+
return next_token

‎localstack-core/localstack/utils/collections.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,8 @@ def get_page(
132132
if page_size is None:
133133
page_size = self.DEFAULT_PAGE_SIZE
134134

135-
if len(result_list) <= page_size:
135+
# returns all or remaining elements in final page.
136+
if len(result_list) <= page_size and next_token is None:
136137
return result_list, None
137138

138139
start_idx = 0

‎localstack-core/localstack/utils/common.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
long_uid,
158158
md5,
159159
short_uid,
160+
short_uid_from_seed,
160161
snake_to_camel_case,
161162
str_insert,
162163
str_remove,

‎localstack-core/localstack/utils/strings.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,12 @@ def short_uid() -> str:
134134
return str(uuid.uuid4())[0:8]
135135

136136

137+
def short_uid_from_seed(seed: str) -> str:
138+
hash = hashlib.sha1(seed.encode("utf-8")).hexdigest()
139+
truncated_hash = hash[:32]
140+
return str(uuid.UUID(truncated_hash))[0:8]
141+
142+
137143
def long_uid() -> str:
138144
return str(uuid.uuid4())
139145

‎tests/aws/services/sqs/test_sqs.py‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@
1313

1414
from localstack import config
1515
from localstack.aws.api.lambda_ import Runtime
16-
from localstack.services.sqs.constants import DEFAULT_MAXIMUM_MESSAGE_SIZE
16+
from localstack.services.sqs.constants import DEFAULT_MAXIMUM_MESSAGE_SIZE, SQS_UUID_STRING_SEED
1717
from localstack.services.sqs.models import sqs_stores
1818
from localstack.services.sqs.provider import MAX_NUMBER_OF_MESSAGES
19-
from localstack.services.sqs.utils import parse_queue_url
19+
from localstack.services.sqs.utils import parse_queue_url, token_generator
2020
from localstack.testing.aws.util import is_aws_cloud
2121
from localstack.testing.config import (
2222
SECONDARY_TEST_AWS_ACCESS_KEY_ID,
@@ -28,7 +28,7 @@
2828
from localstack.utils.aws import arns
2929
from localstack.utils.aws.arns import get_partition
3030
from localstack.utils.aws.request_context import mock_aws_request_headers
31-
from localstack.utils.common import poll_condition, retry, short_uid, to_str
31+
from localstack.utils.common import poll_condition, retry, short_uid, short_uid_from_seed, to_str
3232
from localstack.utils.urls import localstack_host
3333
from tests.aws.services.lambda_.functions import lambda_integration
3434
from tests.aws.services.lambda_.test_lambda import TEST_LAMBDA_PYTHON
@@ -143,6 +143,60 @@ def test_list_queues(self, sqs_create_queue, aws_client):
143143
result = aws_client.sqs.list_queues(QueueNamePrefix="nonexisting-queue-")
144144
assert "QueueUrls" not in result
145145

146+
@markers.aws.validated
147+
def test_list_queues_pagination(self, sqs_create_queue, aws_client, snapshot):
148+
queue_list_length = 10
149+
# ensures test is unique and prevents conflict in case of parrallel test runs
150+
test_output_identifier = short_uid_from_seed(SQS_UUID_STRING_SEED)
151+
max_result_1 = 2
152+
max_result_2 = 10
153+
154+
queue_names = [f"{test_output_identifier}-test-queue-{i}" for i in range(queue_list_length)]
155+
156+
queue_urls = []
157+
for name in queue_names:
158+
sqs_create_queue(QueueName=name)
159+
queue_url = aws_client.sqs.get_queue_url(QueueName=name)["QueueUrl"]
160+
assert queue_url.endswith(name)
161+
queue_urls.append(queue_url)
162+
163+
list_all = aws_client.sqs.list_queues(QueueNamePrefix=test_output_identifier)
164+
assert "QueueUrls" in list_all
165+
assert len(list_all["QueueUrls"]) == queue_list_length
166+
snapshot.match("list_all", list_all)
167+
168+
list_two_max = aws_client.sqs.list_queues(
169+
MaxResults=max_result_1, QueueNamePrefix=test_output_identifier
170+
)
171+
assert "QueueUrls" in list_two_max
172+
assert "NextToken" in list_two_max
173+
assert len(list_two_max["QueueUrls"]) == max_result_1
174+
snapshot.match("list_two_max", list_two_max)
175+
next_token = list_two_max["NextToken"]
176+
177+
list_remaining = aws_client.sqs.list_queues(
178+
MaxResults=max_result_2, NextToken=next_token, QueueNamePrefix=test_output_identifier
179+
)
180+
assert "QueueUrls" in list_remaining
181+
assert "NextToken" not in list_remaining
182+
assert len(list_remaining["QueueUrls"]) == max_result_2 - max_result_1
183+
snapshot.match("list_remaining", list_remaining)
184+
185+
snapshot.add_transformer(
186+
snapshot.transform.regex(
187+
r"https://sqs\.(.+?)\.amazonaws\.com",
188+
r"http://sqs.\1.localhost.localstack.cloud:4566",
189+
)
190+
)
191+
192+
url = f"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/{test_output_identifier}-test-queue-{max_result_1 - 1}"
193+
snapshot.add_transformer(
194+
snapshot.transform.regex(
195+
r'("NextToken":\s*")[^"]*(")',
196+
r"\1" + token_generator(url) + r"\2",
197+
)
198+
)
199+
146200
@markers.aws.validated
147201
def test_create_queue_and_get_attributes(self, sqs_queue, aws_sqs_client):
148202
result = aws_sqs_client.get_queue_attributes(

‎tests/aws/services/sqs/test_sqs.snapshot.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3840,5 +3840,55 @@
38403840
}
38413841
}
38423842
}
3843+
},
3844+
"tests/aws/services/sqs/test_sqs.py::TestSqsProvider::test_list_queues_pagination": {
3845+
"recorded-date": "19-03-2025, 21:04:35",
3846+
"recorded-content": {
3847+
"list_all": {
3848+
"QueueUrls": [
3849+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-0",
3850+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-1",
3851+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-2",
3852+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-3",
3853+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-4",
3854+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-5",
3855+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-6",
3856+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-7",
3857+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-8",
3858+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-9"
3859+
],
3860+
"ResponseMetadata": {
3861+
"HTTPHeaders": {},
3862+
"HTTPStatusCode": 200
3863+
}
3864+
},
3865+
"list_two_max": {
3866+
"NextToken": "aHR0cDovL3Nxcy48cmVnaW9uPi5sb2NhbGhvc3QubG9jYWxzdGFjay5jbG91ZDo0NTY2LzExMTExMTExMTExMS83ZjdkZjBmNS10ZXN0LXF1ZXVlLTE=",
3867+
"QueueUrls": [
3868+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-0",
3869+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-1"
3870+
],
3871+
"ResponseMetadata": {
3872+
"HTTPHeaders": {},
3873+
"HTTPStatusCode": 200
3874+
}
3875+
},
3876+
"list_remaining": {
3877+
"QueueUrls": [
3878+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-2",
3879+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-3",
3880+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-4",
3881+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-5",
3882+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-6",
3883+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-7",
3884+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-8",
3885+
"http://sqs.<region>.localhost.localstack.cloud:4566/111111111111/7f7df0f5-test-queue-9"
3886+
],
3887+
"ResponseMetadata": {
3888+
"HTTPHeaders": {},
3889+
"HTTPStatusCode": 200
3890+
}
3891+
}
3892+
}
38433893
}
38443894
}

‎tests/aws/services/sqs/test_sqs.validation.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,9 @@
167167
"tests/aws/services/sqs/test_sqs.py::TestSqsProvider::test_list_queues": {
168168
"last_validated_date": "2024-04-30T13:39:55+00:00"
169169
},
170+
"tests/aws/services/sqs/test_sqs.py::TestSqsProvider::test_list_queues_pagination": {
171+
"last_validated_date": "2025-03-19T21:04:33+00:00"
172+
},
170173
"tests/aws/services/sqs/test_sqs.py::TestSqsProvider::test_marker_serialization_json_protocol[\"{\\\\\"foo\\\\\": \\\\\"ba\\\\rr\\\\\"}\"]": {
171174
"last_validated_date": "2024-05-07T13:33:39+00:00"
172175
},
@@ -422,4 +425,4 @@
422425
"tests/aws/services/sqs/test_sqs.py::TestSqsQueryApi::test_send_message_via_queue_url_with_json_protocol": {
423426
"last_validated_date": "2024-04-30T13:35:11+00:00"
424427
}
425-
}
428+
}

0 commit comments

Comments
 (0)