Skip to content
This repository was archived by the owner on Mar 31, 2026. It is now read-only.

Commit f4cf041

Browse files
andrewsgandrewsg
authored
feat: support custom universe domains/TPC (#1212)
1 parent a0416a2 commit f4cf041

15 files changed

Lines changed: 415 additions & 96 deletions

File tree

‎google/cloud/storage/_helpers.py‎

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from hashlib import md5
2222
import os
2323
from urllib.parse import urlsplit
24+
from urllib.parse import urlunsplit
2425
from uuid import uuid4
2526

2627
from google import resumable_media
@@ -30,19 +31,24 @@
3031
from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
3132

3233

33-
STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
34+
STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST" # Despite name, includes scheme.
3435
"""Environment variable defining host for Storage emulator."""
3536

36-
_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"
37+
_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE" # Includes scheme.
3738
"""This is an experimental configuration variable. Use api_endpoint instead."""
3839

3940
_API_VERSION_OVERRIDE_ENV_VAR = "API_VERSION_OVERRIDE"
4041
"""This is an experimental configuration variable used for internal testing."""
4142

42-
_DEFAULT_STORAGE_HOST = os.getenv(
43-
_API_ENDPOINT_OVERRIDE_ENV_VAR, "https://storage.googleapis.com"
43+
_DEFAULT_UNIVERSE_DOMAIN = "googleapis.com"
44+
45+
_STORAGE_HOST_TEMPLATE = "storage.{universe_domain}"
46+
47+
_TRUE_DEFAULT_STORAGE_HOST = _STORAGE_HOST_TEMPLATE.format(
48+
universe_domain=_DEFAULT_UNIVERSE_DOMAIN
4449
)
45-
"""Default storage host for JSON API."""
50+
51+
_DEFAULT_SCHEME = "https://"
4652

4753
_API_VERSION = os.getenv(_API_VERSION_OVERRIDE_ENV_VAR, "v1")
4854
"""API version of the default storage host"""
@@ -72,8 +78,39 @@
7278
)
7379

7480

75-
def _get_storage_host():
76-
return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST)
81+
def _get_storage_emulator_override():
82+
return os.environ.get(STORAGE_EMULATOR_ENV_VAR, None)
83+
84+
85+
def _get_default_storage_base_url():
86+
return os.getenv(
87+
_API_ENDPOINT_OVERRIDE_ENV_VAR, _DEFAULT_SCHEME + _TRUE_DEFAULT_STORAGE_HOST
88+
)
89+
90+
91+
def _get_api_endpoint_override():
92+
"""This is an experimental configuration variable. Use api_endpoint instead."""
93+
if _get_default_storage_base_url() != _DEFAULT_SCHEME + _TRUE_DEFAULT_STORAGE_HOST:
94+
return _get_default_storage_base_url()
95+
return None
96+
97+
98+
def _virtual_hosted_style_base_url(url, bucket, trailing_slash=False):
99+
"""Returns the scheme and netloc sections of the url, with the bucket
100+
prepended to the netloc.
101+
102+
Not intended for use with netlocs which include a username and password.
103+
"""
104+
parsed_url = urlsplit(url)
105+
new_netloc = f"{bucket}.{parsed_url.netloc}"
106+
base_url = urlunsplit(
107+
(parsed_url.scheme, new_netloc, "/" if trailing_slash else "", "", "")
108+
)
109+
return base_url
110+
111+
112+
def _use_client_cert():
113+
return os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
77114

78115

79116
def _get_environ_project():

‎google/cloud/storage/_http.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,14 @@
2121

2222

2323
class Connection(_http.JSONConnection):
24-
"""A connection to Google Cloud Storage via the JSON REST API. Mutual TLS feature will be
25-
enabled if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is set to "true".
24+
"""A connection to Google Cloud Storage via the JSON REST API.
25+
26+
Mutual TLS will be enabled if the "GOOGLE_API_USE_CLIENT_CERTIFICATE"
27+
environment variable is set to the exact string "true" (case-sensitive).
28+
29+
Mutual TLS is not compatible with any API endpoint or universe domain
30+
override at this time. If such settings are enabled along with
31+
"GOOGLE_API_USE_CLIENT_CERTIFICATE", a ValueError will be raised.
2632
2733
:type client: :class:`~google.cloud.storage.client.Client`
2834
:param client: The client that owns the current connection.
@@ -34,7 +40,7 @@ class Connection(_http.JSONConnection):
3440
:param api_endpoint: (Optional) api endpoint to use.
3541
"""
3642

37-
DEFAULT_API_ENDPOINT = _helpers._DEFAULT_STORAGE_HOST
43+
DEFAULT_API_ENDPOINT = _helpers._get_default_storage_base_url()
3844
DEFAULT_API_MTLS_ENDPOINT = "https://storage.mtls.googleapis.com"
3945

4046
def __init__(self, client, client_info=None, api_endpoint=None):

‎google/cloud/storage/_signing.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,7 @@ def generate_signed_url_v4(
466466
``tzinfo`` set, it will be assumed to be ``UTC``.
467467
468468
:type api_access_endpoint: str
469-
:param api_access_endpoint: (Optional) URI base. Defaults to
469+
:param api_access_endpoint: URI base. Defaults to
470470
"https://storage.googleapis.com/"
471471
472472
:type method: str

‎google/cloud/storage/blob.py‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,12 @@
5757
from google.cloud.storage._helpers import _raise_if_more_than_one_set
5858
from google.cloud.storage._helpers import _api_core_retry_to_resumable_media_retry
5959
from google.cloud.storage._helpers import _get_default_headers
60+
from google.cloud.storage._helpers import _get_default_storage_base_url
6061
from google.cloud.storage._signing import generate_signed_url_v2
6162
from google.cloud.storage._signing import generate_signed_url_v4
6263
from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
63-
from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
6464
from google.cloud.storage._helpers import _API_VERSION
65+
from google.cloud.storage._helpers import _virtual_hosted_style_base_url
6566
from google.cloud.storage.acl import ACL
6667
from google.cloud.storage.acl import ObjectACL
6768
from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -80,7 +81,6 @@
8081
from google.cloud.storage.fileio import BlobWriter
8182

8283

83-
_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST
8484
_DEFAULT_CONTENT_TYPE = "application/octet-stream"
8585
_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"
8686
_BASE_UPLOAD_TEMPLATE = (
@@ -376,8 +376,12 @@ def public_url(self):
376376
:rtype: `string`
377377
:returns: The public URL for this blob.
378378
"""
379+
if self.client:
380+
endpoint = self.client.api_endpoint
381+
else:
382+
endpoint = _get_default_storage_base_url()
379383
return "{storage_base_url}/{bucket_name}/{quoted_name}".format(
380-
storage_base_url=_API_ACCESS_ENDPOINT,
384+
storage_base_url=endpoint,
381385
bucket_name=self.bucket.name,
382386
quoted_name=_quote(self.name, safe=b"/~"),
383387
)
@@ -416,7 +420,7 @@ def from_string(cls, uri, client=None):
416420
def generate_signed_url(
417421
self,
418422
expiration=None,
419-
api_access_endpoint=_API_ACCESS_ENDPOINT,
423+
api_access_endpoint=None,
420424
method="GET",
421425
content_md5=None,
422426
content_type=None,
@@ -464,7 +468,9 @@ def generate_signed_url(
464468
assumed to be ``UTC``.
465469
466470
:type api_access_endpoint: str
467-
:param api_access_endpoint: (Optional) URI base.
471+
:param api_access_endpoint: (Optional) URI base, for instance
472+
"https://storage.googleapis.com". If not specified, the client's
473+
api_endpoint will be used. Incompatible with bucket_bound_hostname.
468474
469475
:type method: str
470476
:param method: The HTTP verb that will be used when requesting the URL.
@@ -537,21 +543,22 @@ def generate_signed_url(
537543
:param virtual_hosted_style:
538544
(Optional) If true, then construct the URL relative the bucket's
539545
virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
546+
Incompatible with bucket_bound_hostname.
540547
541548
:type bucket_bound_hostname: str
542549
:param bucket_bound_hostname:
543-
(Optional) If passed, then construct the URL relative to the
544-
bucket-bound hostname. Value can be a bare or with scheme, e.g.,
545-
'example.com' or 'http://example.com'. See:
546-
https://cloud.google.com/storage/docs/request-endpoints#cname
550+
(Optional) If passed, then construct the URL relative to the bucket-bound hostname.
551+
Value can be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
552+
Incompatible with api_access_endpoint and virtual_hosted_style.
553+
See: https://cloud.google.com/storage/docs/request-endpoints#cname
547554
548555
:type scheme: str
549556
:param scheme:
550557
(Optional) If ``bucket_bound_hostname`` is passed as a bare
551558
hostname, use this value as the scheme. ``https`` will work only
552559
when using a CDN. Defaults to ``"http"``.
553560
554-
:raises: :exc:`ValueError` when version is invalid.
561+
:raises: :exc:`ValueError` when version is invalid or mutually exclusive arguments are used.
555562
:raises: :exc:`TypeError` when expiration is not a valid type.
556563
:raises: :exc:`AttributeError` if credentials is not an instance
557564
of :class:`google.auth.credentials.Signing`.
@@ -565,25 +572,38 @@ def generate_signed_url(
565572
elif version not in ("v2", "v4"):
566573
raise ValueError("'version' must be either 'v2' or 'v4'")
567574

575+
if (
576+
api_access_endpoint is not None or virtual_hosted_style
577+
) and bucket_bound_hostname:
578+
raise ValueError(
579+
"The bucket_bound_hostname argument is not compatible with "
580+
"either api_access_endpoint or virtual_hosted_style."
581+
)
582+
583+
if api_access_endpoint is None:
584+
client = self._require_client(client)
585+
api_access_endpoint = client.api_endpoint
586+
568587
quoted_name = _quote(self.name, safe=b"/~")
569588

570589
# If you are on Google Compute Engine, you can't generate a signed URL
571590
# using GCE service account.
572591
# See https://github.com/googleapis/google-auth-library-python/issues/50
573592
if virtual_hosted_style:
574-
api_access_endpoint = f"https://{self.bucket.name}.storage.googleapis.com"
593+
api_access_endpoint = _virtual_hosted_style_base_url(
594+
api_access_endpoint, self.bucket.name
595+
)
596+
resource = f"/{quoted_name}"
575597
elif bucket_bound_hostname:
576598
api_access_endpoint = _bucket_bound_hostname_url(
577599
bucket_bound_hostname, scheme
578600
)
601+
resource = f"/{quoted_name}"
579602
else:
580603
resource = f"/{self.bucket.name}/{quoted_name}"
581604

582-
if virtual_hosted_style or bucket_bound_hostname:
583-
resource = f"/{quoted_name}"
584-
585605
if credentials is None:
586-
client = self._require_client(client)
606+
client = self._require_client(client) # May be redundant, but that's ok.
587607
credentials = client._credentials
588608

589609
if version == "v2":

‎google/cloud/storage/bucket.py‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
from google.cloud.storage._signing import generate_signed_url_v2
3737
from google.cloud.storage._signing import generate_signed_url_v4
3838
from google.cloud.storage._helpers import _bucket_bound_hostname_url
39+
from google.cloud.storage._helpers import _virtual_hosted_style_base_url
3940
from google.cloud.storage.acl import BucketACL
4041
from google.cloud.storage.acl import DefaultObjectACL
4142
from google.cloud.storage.blob import Blob
@@ -82,7 +83,6 @@
8283
"valid before the bucket is created. Instead, pass the location "
8384
"to `Bucket.create`."
8485
)
85-
_API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
8686

8787

8888
def _blobs_page_start(iterator, page, response):
@@ -3265,7 +3265,7 @@ def lock_retention_policy(
32653265
def generate_signed_url(
32663266
self,
32673267
expiration=None,
3268-
api_access_endpoint=_API_ACCESS_ENDPOINT,
3268+
api_access_endpoint=None,
32693269
method="GET",
32703270
headers=None,
32713271
query_parameters=None,
@@ -3298,7 +3298,9 @@ def generate_signed_url(
32983298
``tzinfo`` set, it will be assumed to be ``UTC``.
32993299
33003300
:type api_access_endpoint: str
3301-
:param api_access_endpoint: (Optional) URI base.
3301+
:param api_access_endpoint: (Optional) URI base, for instance
3302+
"https://storage.googleapis.com". If not specified, the client's
3303+
api_endpoint will be used. Incompatible with bucket_bound_hostname.
33023304
33033305
:type method: str
33043306
:param method: The HTTP verb that will be used when requesting the URL.
@@ -3322,7 +3324,6 @@ def generate_signed_url(
33223324
:param client: (Optional) The client to use. If not passed, falls back
33233325
to the ``client`` stored on the blob's bucket.
33243326
3325-
33263327
:type credentials: :class:`google.auth.credentials.Credentials` or
33273328
:class:`NoneType`
33283329
:param credentials: The authorization credentials to attach to requests.
@@ -3338,11 +3339,13 @@ def generate_signed_url(
33383339
:param virtual_hosted_style:
33393340
(Optional) If true, then construct the URL relative the bucket's
33403341
virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
3342+
Incompatible with bucket_bound_hostname.
33413343
33423344
:type bucket_bound_hostname: str
33433345
:param bucket_bound_hostname:
3344-
(Optional) If pass, then construct the URL relative to the bucket-bound hostname.
3345-
Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
3346+
(Optional) If passed, then construct the URL relative to the bucket-bound hostname.
3347+
Value can be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
3348+
Incompatible with api_access_endpoint and virtual_hosted_style.
33463349
See: https://cloud.google.com/storage/docs/request-endpoints#cname
33473350
33483351
:type scheme: str
@@ -3351,7 +3354,7 @@ def generate_signed_url(
33513354
this value as the scheme. ``https`` will work only when using a CDN.
33523355
Defaults to ``"http"``.
33533356
3354-
:raises: :exc:`ValueError` when version is invalid.
3357+
:raises: :exc:`ValueError` when version is invalid or mutually exclusive arguments are used.
33553358
:raises: :exc:`TypeError` when expiration is not a valid type.
33563359
:raises: :exc:`AttributeError` if credentials is not an instance
33573360
of :class:`google.auth.credentials.Signing`.
@@ -3365,23 +3368,36 @@ def generate_signed_url(
33653368
elif version not in ("v2", "v4"):
33663369
raise ValueError("'version' must be either 'v2' or 'v4'")
33673370

3371+
if (
3372+
api_access_endpoint is not None or virtual_hosted_style
3373+
) and bucket_bound_hostname:
3374+
raise ValueError(
3375+
"The bucket_bound_hostname argument is not compatible with "
3376+
"either api_access_endpoint or virtual_hosted_style."
3377+
)
3378+
3379+
if api_access_endpoint is None:
3380+
client = self._require_client(client)
3381+
api_access_endpoint = client.api_endpoint
3382+
33683383
# If you are on Google Compute Engine, you can't generate a signed URL
33693384
# using GCE service account.
33703385
# See https://github.com/googleapis/google-auth-library-python/issues/50
33713386
if virtual_hosted_style:
3372-
api_access_endpoint = f"https://{self.name}.storage.googleapis.com"
3387+
api_access_endpoint = _virtual_hosted_style_base_url(
3388+
api_access_endpoint, self.name
3389+
)
3390+
resource = "/"
33733391
elif bucket_bound_hostname:
33743392
api_access_endpoint = _bucket_bound_hostname_url(
33753393
bucket_bound_hostname, scheme
33763394
)
3395+
resource = "/"
33773396
else:
33783397
resource = f"/{self.name}"
33793398

3380-
if virtual_hosted_style or bucket_bound_hostname:
3381-
resource = "/"
3382-
33833399
if credentials is None:
3384-
client = self._require_client(client)
3400+
client = self._require_client(client) # May be redundant, but that's ok.
33853401
credentials = client._credentials
33863402

33873403
if version == "v2":

0 commit comments

Comments
 (0)