S3: fix aws-global validation in CreateBucket (#13250)

bentsku · web-flow · commit 48d30e0743d7 · 2025-10-14T19:30:13.000+02:00
diff --git a/localstack-core/localstack/aws/api/s3/__init__.py b/localstack-core/localstack/aws/api/s3/__init__.py
@@ -1061,6 +1061,14 @@ class BadDigest(ServiceException):
     CalculatedDigest: Optional[ContentMD5]
 
 
+class AuthorizationHeaderMalformed(ServiceException):
+    code: str = "AuthorizationHeaderMalformed"
+    sender_fault: bool = False
+    status_code: int = 400
+    Region: Optional[BucketRegion]
+    HostId: Optional[HostId]
+
+
 AbortDate = datetime
 
 
diff --git a/localstack-core/localstack/aws/spec-patches.json b/localstack-core/localstack/aws/spec-patches.json
@@ -1329,6 +1329,26 @@
         "documentation": "<p>The Content-MD5 you specified did not match what we received.</p>",
         "exception": true
       }
+    },
+    {
+      "op": "add",
+      "path": "/shapes/AuthorizationHeaderMalformed",
+      "value": {
+        "type": "structure",
+        "members": {
+          "Region": {
+            "shape": "BucketRegion"
+          },
+          "HostId": {
+            "shape": "HostId"
+          }
+        },
+        "error": {
+          "httpStatusCode": 400
+        },
+        "documentation": "<p>The authorization header is malformed.</p>",
+        "exception": true
+      }
     }
   ],
   "apigatewayv2/2018-11-29/service-2": [
diff --git a/localstack-core/localstack/services/s3/constants.py b/localstack-core/localstack/services/s3/constants.py
@@ -21,6 +21,11 @@
 DEFAULT_PRE_SIGNED_ACCESS_KEY_ID = "test"
 DEFAULT_PRE_SIGNED_SECRET_ACCESS_KEY = "test"
 
+S3_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
+"""
+S3 is returning a Host Id as part of its exceptions
+"""
+
 AUTHENTICATED_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
 ALL_USERS_ACL_GROUP = "http://acs.amazonaws.com/groups/global/AllUsers"
 LOG_DELIVERY_ACL_GROUP = "http://acs.amazonaws.com/groups/s3/LogDelivery"
diff --git a/localstack-core/localstack/services/s3/cors.py b/localstack-core/localstack/services/s3/cors.py
@@ -21,13 +21,13 @@
 from localstack.aws.spec import get_service_catalog
 from localstack.config import S3_VIRTUAL_HOSTNAME
 from localstack.http import Request, Response
+from localstack.services.s3.constants import S3_HOST_ID
 from localstack.services.s3.utils import S3_VIRTUAL_HOSTNAME_REGEX
 
 # TODO: add more logging statements
 LOG = logging.getLogger(__name__)
 
 _s3_virtual_host_regex = re.compile(S3_VIRTUAL_HOSTNAME_REGEX)
-FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
 
 # TODO: refactor those to expose the needed methods maybe in another way that both can import
 add_default_headers = CorsResponseEnricher.add_cors_headers
@@ -135,7 +135,7 @@ def stop_options_chain():
             if is_options_request:
                 context.operation = self._get_op_from_request(request)
                 raise BadRequest(
-                    "Insufficient information. Origin request header needed.", HostId=FAKE_HOST_ID
+                    "Insufficient information. Origin request header needed.", HostId=S3_HOST_ID
                 )
             else:
                 # If the header is missing, Amazon S3 doesn't treat the request as a cross-origin request,
@@ -167,7 +167,7 @@ def stop_options_chain():
                     context.operation = self._get_op_from_request(request)
                     raise AccessForbidden(
                         message,
-                        HostId=FAKE_HOST_ID,
+                        HostId=S3_HOST_ID,
                         Method=request.headers.get("Access-Control-Request-Method", "OPTIONS"),
                         ResourceType="BUCKET",
                     )
@@ -182,7 +182,7 @@ def stop_options_chain():
                 context.operation = self._get_op_from_request(request)
                 raise AccessForbidden(
                     "CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted by the resource's CORS spec.",
-                    HostId=FAKE_HOST_ID,
+                    HostId=S3_HOST_ID,
                     Method=request.headers.get("Access-Control-Request-Method"),
                     ResourceType="OBJECT",
                 )
diff --git a/localstack-core/localstack/services/s3/presigned_url.py b/localstack-core/localstack/services/s3/presigned_url.py
@@ -40,6 +40,7 @@
 from localstack.services.s3.constants import (
     DEFAULT_PRE_SIGNED_ACCESS_KEY_ID,
     DEFAULT_PRE_SIGNED_SECRET_ACCESS_KEY,
+    S3_HOST_ID,
     SIGNATURE_V2_PARAMS,
     SIGNATURE_V4_PARAMS,
 )
@@ -85,8 +86,6 @@
     "x-amz-content-sha256",
 ]
 
-FAKE_HOST_ID = "9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg="
-
 HOST_COMBINATION_REGEX = r"^(.*)(:[\d]{0,6})"
 PORT_REPLACEMENT = [":80", ":443", f":{config.GATEWAY_LISTEN[0].port}", ""]
 
@@ -156,7 +155,7 @@ def create_signature_does_not_match_sig_v2(
         "The request signature we calculated does not match the signature you provided. Check your key and signing method."
     )
     ex.AWSAccessKeyId = access_key_id
-    ex.HostId = FAKE_HOST_ID
+    ex.HostId = S3_HOST_ID
     ex.SignatureProvided = request_signature
     ex.StringToSign = string_to_sign
     ex.StringToSignBytes = to_bytes(string_to_sign).hex(sep=" ", bytes_per_sep=2).upper()
@@ -299,7 +298,7 @@ def is_valid_sig_v2(query_args: set) -> bool:
             LOG.info("Presign signature calculation failed")
             raise AccessDenied(
                 "Query-string authentication requires the Signature, Expires and AWSAccessKeyId parameters",
-                HostId=FAKE_HOST_ID,
+                HostId=S3_HOST_ID,
             )
 
         return True
@@ -317,7 +316,7 @@ def is_valid_sig_v4(query_args: set) -> bool:
             LOG.info("Presign signature calculation failed")
             raise AuthorizationQueryParametersError(
                 "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
-                HostId=FAKE_HOST_ID,
+                HostId=S3_HOST_ID,
             )
 
         return True
@@ -351,7 +350,7 @@ def validate_presigned_url_s3(context: RequestContext) -> None:
             )
         else:
             raise AccessDenied(
-                "Request has expired", HostId=FAKE_HOST_ID, Expires=expires, ServerTime=time.time()
+                "Request has expired", HostId=S3_HOST_ID, Expires=expires, ServerTime=time.time()
             )
 
     auth_signer = HmacV1QueryAuthValidation(credentials=signing_credentials, expires=expires)
@@ -450,7 +449,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
         else:
             raise AccessDenied(
                 "There were headers present in the request which were not signed",
-                HostId=FAKE_HOST_ID,
+                HostId=S3_HOST_ID,
                 HeadersNotSigned=", ".join(sigv4_context.missing_signed_headers),
             )
 
@@ -482,7 +481,7 @@ def validate_presigned_url_s3v4(context: RequestContext) -> None:
         else:
             raise AccessDenied(
                 "Request has expired",
-                HostId=FAKE_HOST_ID,
+                HostId=S3_HOST_ID,
                 Expires=expiration_time.timestamp(),
                 ServerTime=time.time(),
                 X_Amz_Expires=x_amz_expires,
@@ -714,7 +713,7 @@ def _get_region_from_x_amz_credential(credential: str) -> str:
         if not (split_creds := credential.split("/")) or len(split_creds) != 5:
             raise AuthorizationQueryParametersError(
                 'Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting "<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request".',
-                HostId=FAKE_HOST_ID,
+                HostId=S3_HOST_ID,
             )
 
         return split_creds[2]
@@ -775,7 +774,7 @@ def validate_post_policy(
             "Bucket POST must contain a field named 'key'.  If it is specified, please check the order of the fields.",
             ArgumentName="key",
             ArgumentValue="",
-            HostId=FAKE_HOST_ID,
+            HostId=S3_HOST_ID,
         )
 
     form_dict = {k.lower(): v for k, v in request_form.items()}
@@ -791,7 +790,7 @@ def validate_post_policy(
 
     if not is_v2 and not is_v4:
         ex: AccessDenied = AccessDenied("Access Denied")
-        ex.HostId = FAKE_HOST_ID
+        ex.HostId = S3_HOST_ID
         raise ex
 
     try:
@@ -810,7 +809,7 @@ def validate_post_policy(
     if expiration := policy_decoded.get("expiration"):
         if is_expired(_parse_policy_expiration_date(expiration)):
             ex: AccessDenied = AccessDenied("Invalid according to Policy: Policy expired.")
-            ex.HostId = FAKE_HOST_ID
+            ex.HostId = S3_HOST_ID
             raise ex
 
     # TODO: validate the signature
@@ -832,7 +831,7 @@ def validate_post_policy(
             str_condition = str(condition).replace("'", '"')
             raise AccessDenied(
                 f"Invalid according to Policy: Policy Condition failed: {str_condition}",
-                HostId=FAKE_HOST_ID,
+                HostId=S3_HOST_ID,
             )
 
 
@@ -885,7 +884,7 @@ def _verify_condition(condition: list | dict, form: dict, additional_policy_meta
                     "Your proposed upload exceeds the maximum allowed size",
                     ProposedSize=size,
                     MaxSizeAllowed=end,
-                    HostId=FAKE_HOST_ID,
+                    HostId=S3_HOST_ID,
                 )
             else:
                 return True
@@ -934,7 +933,7 @@ def _is_match_with_signature_fields(
                     f"Bucket POST must contain a field named '{argument_name}'.  If it is specified, please check the order of the fields.",
                     ArgumentName=argument_name,
                     ArgumentValue="",
-                    HostId=FAKE_HOST_ID,
+                    HostId=S3_HOST_ID,
                 )
 
         return True
diff --git a/localstack-core/localstack/services/s3/provider.py b/localstack-core/localstack/services/s3/provider.py
@@ -23,6 +23,7 @@
     AccountId,
     AnalyticsConfiguration,
     AnalyticsId,
+    AuthorizationHeaderMalformed,
     BadDigest,
     Body,
     Bucket,
@@ -235,6 +236,7 @@
     ARCHIVES_STORAGE_CLASSES,
     CHECKSUM_ALGORITHMS,
     DEFAULT_BUCKET_ENCRYPTION,
+    S3_HOST_ID,
 )
 from localstack.services.s3.cors import S3CorsHandler, s3_cors_request_handler
 from localstack.services.s3.exceptions import (
@@ -470,6 +472,16 @@ def create_bucket(
         context: RequestContext,
         request: CreateBucketRequest,
     ) -> CreateBucketOutput:
+        if context.region == "aws-global":
+            # TODO: extend this logic to probably all the provider, and maybe all services. S3 is the most impacted
+            #  right now so this will help users to properly set a region in their config
+            # See the `TestS3.test_create_bucket_aws_global` test
+            raise AuthorizationHeaderMalformed(
+                f"The authorization header is malformed; the region 'aws-global' is wrong; expecting '{AWS_REGION_US_EAST_1}'",
+                HostId=S3_HOST_ID,
+                Region=AWS_REGION_US_EAST_1,
+            )
+
         bucket_name = request["Bucket"]
 
         if not is_bucket_name_valid(bucket_name):
@@ -637,6 +649,16 @@ def head_bucket(
         expected_bucket_owner: AccountId = None,
         **kwargs,
     ) -> HeadBucketOutput:
+        if context.region == "aws-global":
+            # TODO: extend this logic to probably all the provider, and maybe all services. S3 is the most impacted
+            #  right now so this will help users to properly set a region in their config
+            # See the `TestS3.test_create_bucket_aws_global` test
+            raise AuthorizationHeaderMalformed(
+                f"The authorization header is malformed; the region 'aws-global' is wrong; expecting '{AWS_REGION_US_EAST_1}'",
+                HostId=S3_HOST_ID,
+                Region=AWS_REGION_US_EAST_1,
+            )
+
         store = self.get_store(context.account_id, context.region)
         if not (s3_bucket := store.buckets.get(bucket)):
             if not (account_id := store.global_bucket_map.get(bucket)):
diff --git a/tests/aws/services/s3/test_s3.py b/tests/aws/services/s3/test_s3.py
@@ -4150,6 +4150,90 @@ def test_access_bucket_different_region(self, s3_create_bucket, s3_vhost_client,
         assert response.status_code == 200
         assert response.history[0].status_code == 307
 
+    @markers.aws.validated
+    @markers.requires_in_process  # we're monkeypatching the handler chain
+    @markers.snapshot.skip_snapshot_verify(
+        # missing from `HeadBucket` call
+        paths=["$..AccessPointAlias"],
+    )
+    def test_create_bucket_aws_global(
+        self,
+        aws_client_factory,
+        cleanups,
+        aws_client,
+        snapshot,
+        aws_http_client_factory,
+        monkeypatch,
+    ):
+        """
+        Some tools use the `aws-global` region instead of `us-east-1` when no region are defined. It is considered
+        a valid region by Botocore too when creating the client, as it is a pseudo-region used for endpoint resolving.
+        The client is however supposed to then use `us-east-1` to sign the request, not `aws-global`.
+        Using `aws-global` to sign the request results in an error in AWS.
+        It seems some tools like the Go SDK used by Terraform might have the wrong logic, and skip the part where
+        they are supposed to sign with `us-east-1` if you override the endpoint url and skip the endpoint
+        resolving part, which is how we end up with those kind of requests.
+        """
+        # we need to patch the `DefaultRegionRewriterStrategy` as it wil replace `aws-global` by `us-east-1`, which
+        # is its default region
+        from localstack.aws.handlers.region import DefaultRegionRewriterStrategy
+
+        monkeypatch.setattr(DefaultRegionRewriterStrategy, "apply", lambda *_, **__: None)
+
+        global_region = "aws-global"
+        bucket_prefix = f"global-bucket-{short_uid()}"
+        bucket_name_1 = f"{bucket_prefix}-{short_uid()}"
+        headers = {"x-amz-content-sha256": "UNSIGNED-PAYLOAD"}
+
+        snapshot.add_transformers_list(
+            [
+                snapshot.transform.regex(bucket_name_1, "<bucket-name-1>"),
+                snapshot.transform.regex(bucket_prefix, "<bucket-prefix>"),
+                snapshot.transform.regex(AWS_REGION_US_EAST_1, "<us_east_1_region>"),
+                snapshot.transform.key_value("DisplayName"),
+                snapshot.transform.key_value("ID"),
+                snapshot.transform.key_value("HostId"),
+                snapshot.transform.key_value("RequestId"),
+            ]
+        )
+        http_client = aws_http_client_factory(
+            service="s3", signer_factory=SigV4Auth, region=global_region
+        )
+        # use the global endpoint with no region
+        base_endpoint = _endpoint_url(region=AWS_REGION_US_EAST_1)
+        response = http_client.put(f"{base_endpoint}/{bucket_name_1}", headers=headers)
+        if response.ok:
+            # we still clean up even if we expect it to fail, just in case it doesn't fail in AWS
+            cleanups.append(lambda: aws_client.s3.delete_bucket(Bucket=bucket_name_1))
+
+        assert response.status_code == 400
+        xml_error = xmltodict.parse(response.content)
+        snapshot.match("xml-error-create-bucket", xml_error)
+
+        # botocore is automatically signing the request with `us-east-1`, so we don't have issues
+        s3_client = aws_client_factory(region_name=global_region).s3
+        create_bucket = s3_client.create_bucket(Bucket=bucket_name_1)
+        cleanups.append(lambda: aws_client.s3.delete_bucket(Bucket=bucket_name_1))
+        snapshot.match("create-bucket-global", create_bucket)
+
+        head_bucket = s3_client.head_bucket(Bucket=bucket_name_1)
+        snapshot.match("head-bucket-global", head_bucket)
+
+        get_location_1 = aws_client.s3.get_bucket_location(Bucket=bucket_name_1)
+        # verify that the bucket 1 is created in `us-east-1`
+        snapshot.match("get-location-1", get_location_1)
+
+        list_buckets_per_region = aws_client.s3.list_buckets(
+            BucketRegion=AWS_REGION_US_EAST_1, Prefix=bucket_prefix
+        )
+        snapshot.match("list-buckets", list_buckets_per_region)
+
+        # test that HeadBucket also raises an exception
+        head_response = http_client.head(f"{base_endpoint}/{bucket_name_1}", headers=headers)
+
+        assert head_response.status_code == 400
+        assert not head_response.content
+
     @markers.aws.validated
     def test_bucket_does_not_exist(self, s3_vhost_client, snapshot, aws_client):
         snapshot.add_transformer(snapshot.transform.s3_api())
diff --git a/tests/aws/services/s3/test_s3.snapshot.json b/tests/aws/services/s3/test_s3.snapshot.json
diff --git a/tests/aws/services/s3/test_s3.validation.json b/tests/aws/services/s3/test_s3.validation.json
