Skip to content

Commit 504f61a

Browse files
authored
Core: normalize incoming requests (#10343)
Ensure no matter how a request comes into Moto, we normalize it to a single Request class.
1 parent 1aba064 commit 504f61a

27 files changed

Lines changed: 653 additions & 274 deletions

‎moto/apigatewaymanagementapi/responses.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,11 @@
1111
class ApiGatewayManagementApiResponse(BaseResponse):
1212
"""Handler for ApiGatewayManagementApi requests and responses."""
1313

14+
use_raw_body = True
15+
1416
def __init__(self) -> None:
1517
super().__init__(service_name="apigatewaymanagementapi")
1618

17-
def setup_class(
18-
self, request: Any, full_url: str, headers: Any, use_raw_body: bool = False
19-
) -> None:
20-
super().setup_class(request, full_url, headers, use_raw_body=True)
21-
2219
@property
2320
def apigatewaymanagementapi_backend(self) -> ApiGatewayManagementApiBackend:
2421
"""Return backend instance specific for this region."""
@@ -52,7 +49,7 @@ def connect_to_apigateway( # type: ignore[misc]
5249
request: Any, full_url: str, headers: Any
5350
) -> TYPE_RESPONSE:
5451
self = ApiGatewayManagementApiResponse()
55-
self.setup_class(request, full_url, headers, use_raw_body=True)
52+
self.setup_class(request)
5653
if request.method == "GET":
5754
return 200, {}, self.get_connection()
5855
elif request.method == "DELETE":

‎moto/core/authorization.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ def _authenticate_and_authorize_action(
4747
):
4848
return
4949
parsed_url = urlparse(self.uri) # type: ignore[attr-defined]
50-
path = parsed_url.path
50+
path = self.raw_path # type: ignore[attr-defined]
5151
if parsed_url.query:
5252
path += "?" + parsed_url.query
5353
iam_request = iam_request_cls(

‎moto/core/botocore_stubber.py‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from moto.core.common_types import TYPE_RESPONSE
1010
from moto.core.config import passthrough_service, passthrough_url, service_whitelisted
1111
from moto.core.exceptions import ServiceNotWhitelisted
12+
from moto.core.request import Request, normalize_request
1213
from moto.core.utils import get_equivalent_url_in_aws_domain
1314

1415

@@ -34,18 +35,18 @@ def __call__(
3435
) -> AWSResponse | None:
3536
if not self.enabled:
3637
return None
37-
38+
request = normalize_request(request)
3839
response = self.process_request(request)
3940
if response is not None:
4041
status, headers, body = response
4142
return AWSResponse(request.url, status, headers, MockRawResponse(body)) # type: ignore[arg-type]
4243
else:
4344
return response
4445

45-
def process_request(self, request: Any) -> TYPE_RESPONSE | None:
46+
def process_request(self, request: Request) -> TYPE_RESPONSE | None:
4647
# Handle non-standard AWS endpoint hostnames from ISO regions or custom
4748
# S3 endpoints.
48-
parsed_url, _ = get_equivalent_url_in_aws_domain(request.url)
49+
parsed_url, _ = get_equivalent_url_in_aws_domain(request.raw_url)
4950
# Remove the querystring from the URL, as we'll never match on that
5051
clean_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
5152

@@ -75,18 +76,14 @@ def process_request(self, request: Any) -> TYPE_RESPONSE | None:
7576
else:
7677
backend = backend_dict["global"]
7778

78-
for header, value in request.headers.items():
79-
if isinstance(value, bytes):
80-
request.headers[header] = value.decode("utf-8")
81-
8279
for url, method_to_execute in backend.urls.items():
8380
if re.compile(url).match(clean_url):
8481
from moto.moto_api import recorder
8582

8683
try:
8784
recorder._record_request(request)
8885
status, headers, body = method_to_execute(
89-
request, request.url, request.headers
86+
request, request.raw_url, request.headers
9087
)
9188
except HTTPException as e:
9289
status = e.code

‎moto/core/custom_responses_mock.py‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
import types
22
from http.client import responses as http_responses
3-
from io import BytesIO
43
from typing import Any
5-
from urllib.parse import urlparse
64

75
import responses
8-
from werkzeug.wrappers import Request
96

107
from moto.backends import get_service_from_url
118
from moto.core.config import passthrough_service, passthrough_url
9+
from moto.core.request import normalize_request
1210
from moto.core.versions import is_responses_0_17_x
1311

1412
from .responses import TYPE_RESPONSE
@@ -28,31 +26,12 @@ def get_response(self, request: Any) -> responses.HTTPResponse:
2826
"""
2927
Need to override this so we can pass decode_content=False
3028
"""
31-
if not isinstance(request, Request):
32-
url = urlparse(request.url)
33-
if request.body is None:
34-
body = None
35-
elif isinstance(request.body, str):
36-
body = request.body.encode("UTF-8")
37-
elif hasattr(request.body, "read"):
38-
body = request.body.read()
39-
else:
40-
body = request.body
41-
req = Request.from_values(
42-
path="?".join([url.path, url.query]),
43-
input_stream=BytesIO(body) if body else None,
44-
content_length=request.headers.get("Content-Length"),
45-
content_type=request.headers.get("Content-Type"),
46-
method=request.method,
47-
base_url=f"{url.scheme}://{url.netloc}",
48-
headers=[(k, v) for k, v in request.headers.items()],
49-
)
50-
request = req
29+
request = normalize_request(request)
5130
headers = self.get_headers()
5231

5332
from moto.moto_api import recorder
5433

55-
recorder._record_request(request, body)
34+
recorder._record_request(request)
5635

5736
result = self.callback(request)
5837
if isinstance(result, Exception):

‎moto/core/request.py‎

Lines changed: 91 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,108 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING
3+
from typing import TYPE_CHECKING, Any, cast
44
from urllib.parse import urlparse
55

6-
from werkzeug.wrappers import Request
6+
try:
7+
from botocore.httpchecksum import AwsChunkedWrapper
8+
except ModuleNotFoundError:
9+
10+
class AwsChunkedWrapper: # type: ignore[no-redef]
11+
def read(self) -> bytes:
12+
raise NotImplementedError()
13+
14+
15+
from botocore.awsrequest import AWSPreparedRequest
16+
from requests import PreparedRequest
17+
from werkzeug.local import LocalProxy
18+
from werkzeug.wrappers import Request as WerkzeugRequest
719

8-
from moto.core.utils import gzip_decompress
920
from moto.settings import MAX_FORM_MEMORY_SIZE
1021
from moto.utilities.constants import APPLICATION_JSON, JSON_TYPES
1122

1223
if TYPE_CHECKING:
13-
from botocore.awsrequest import AWSPreparedRequest
14-
1524
from moto.core.model import ServiceModel
1625

1726

18-
def normalize_request(request: AWSPreparedRequest | Request) -> Request:
27+
class Request(WerkzeugRequest):
28+
#: True when this request was received by a real WSGI server (Moto Server).
29+
from_wsgi_server = False
30+
31+
def __init__(self, *args: Any, **kwargs: Any) -> None:
32+
super().__init__(*args, **kwargs)
33+
self.max_form_memory_size = MAX_FORM_MEMORY_SIZE
34+
35+
@classmethod
36+
def from_primitives(
37+
cls, method: str, url: str, headers: Any, body: Any = None
38+
) -> Request:
39+
"""Build a request from the basic components of an HTTP message."""
40+
if isinstance(body, AwsChunkedWrapper):
41+
body = body.read()
42+
parsed_url = urlparse(url)
43+
request = cast(
44+
Request,
45+
cls.from_values(
46+
method=method,
47+
base_url=f"{parsed_url.scheme}://{parsed_url.netloc}",
48+
path=parsed_url.path,
49+
query_string=parsed_url.query,
50+
data=body if body is not None else b"",
51+
# The proxy de-chunks before we ever see the request, and we (above) read
52+
# an AwsChunkedWrapper out in full, so Transfer-Encoding no longer applies.
53+
headers=[
54+
(key, value.decode("utf-8") if isinstance(value, bytes) else value)
55+
for key, value in headers.items()
56+
if key.lower() not in ["transfer-encoding"]
57+
],
58+
),
59+
)
60+
# Ensure a Content-Length header is present, even for bodiless requests.
61+
# (Some S3 endpoints return a 411 if this header is missing.)
62+
request.environ.setdefault("CONTENT_LENGTH", "0")
63+
return request
64+
65+
@property
66+
def raw_path(self) -> str:
67+
"""The path as it arrived, before werkzeug percent-decoded it.
68+
69+
S3 keys routinely contain characters - an encoded slash, most awkwardly -
70+
that `path` decodes away, and Moto matches backend URLs against the
71+
encoded form.
72+
"""
73+
# RAW_URI holds either a path or a full URL, depending on the server.
74+
# A path may begin with a double slash, which urlparse would read as the
75+
# start of a netloc, so only parse when there is really a scheme to strip.
76+
raw_uri: str = self.environ.get("RAW_URI", "") or self.path
77+
parsed = urlparse(raw_uri)
78+
raw_path = parsed.path if parsed.scheme else raw_uri.split("?", 1)[0]
79+
if not raw_path:
80+
return "/"
81+
return raw_path if raw_path.startswith("/") else f"/{raw_path}"
82+
83+
@property
84+
def raw_url(self) -> str:
85+
"""The full URL as it arrived. See `raw_path`."""
86+
raw_url = f"{self.url_root.rstrip('/')}{self.raw_path}"
87+
if self.query_string:
88+
raw_url += f"?{self.query_string.decode()}"
89+
return raw_url
90+
91+
92+
def normalize_request(
93+
request: AWSPreparedRequest | LocalProxy[Request] | PreparedRequest | Request,
94+
) -> Request:
95+
"""Turn however this request reached us into the one type the core acts on."""
96+
if isinstance(request, LocalProxy):
97+
request = request._get_current_object()
1998
if isinstance(request, Request):
2099
return request
21-
body = request.body
22-
# Request.from_values() does not automatically handle gzip-encoded bodies,
23-
# like the full WSGI server would, so we need to do it manually.
24-
if request.headers.get("Content-Encoding") == "gzip":
25-
body = gzip_decompress(body) # type: ignore[arg-type]
26-
parsed_url = urlparse(request.url)
27-
Request.max_form_memory_size = MAX_FORM_MEMORY_SIZE
28-
normalized_request = Request.from_values(
29-
method=request.method,
30-
base_url=f"{parsed_url.scheme}://{parsed_url.netloc}",
31-
path=parsed_url.path,
32-
query_string=parsed_url.query,
33-
data=body,
34-
headers=[(k, v) for k, v in request.headers.items()],
35-
)
36-
return normalized_request
100+
if isinstance(request, (AWSPreparedRequest, PreparedRequest)):
101+
assert request.method and request.url
102+
return Request.from_primitives(
103+
request.method, request.url, request.headers, request.body
104+
)
105+
raise TypeError(f"Cannot normalize request of type {type(request).__name__}")
37106

38107

39108
def determine_request_protocol(

0 commit comments

Comments
 (0)