# -*- coding: utf-8 -*-

# Copyright 2018 Google Inc. All Rights Reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Shared utility structures and methods that require importing boto.

This module also imports httplib2 (as it is Boto's http transport and closely

tied to some of Boto's core functionality) and oauth2client.

"""

from __future__ import absolute_import

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals

import os

import pkgutil

import tempfile

import textwrap

import six

import boto

from boto import config

import boto.auth

from boto.exception import NoAuthHandlerFound

from boto.gs.connection import GSConnection

from boto.provider import Provider

from boto.pyami.config import BotoConfigLocations

import gslib

from gslib import context_config

from gslib.exception import CommandException

from gslib.utils import system_util

from gslib.utils.constants import DEFAULT_GCS_JSON_API_VERSION

from gslib.utils.constants import DEFAULT_GSUTIL_STATE_DIR

from gslib.utils.constants import SSL_TIMEOUT_SEC

from gslib.utils.constants import UTF8

from gslib.utils.unit_util import HumanReadableToBytes

from gslib.utils.unit_util import ONE_MIB

import httplib2

from oauth2client.client import HAS_CRYPTO

if six.PY3:

long = int

# Globals in this module are set according to values in the boto config.

BOTO_IS_SECURE = config.get('Boto', 'is_secure', True)

CERTIFICATE_VALIDATION_ENABLED = config.get('Boto',

'https_validate_certificates', True)

configured_certs_file = None # Single certs file for use across all processes.

temp_certs_file = None # Temporary certs file for cleanup upon exit.

def ConfigureCertsFile():

"""Configures and returns the CA Certificates file.

If one is already configured, use it. Otherwise, use the cert roots

distributed with gsutil.

Returns:

string filename of the certs file to use.

"""

certs_file = boto.config.get('Boto', 'ca_certificates_file', None)

# The 'system' keyword indicates to use the system installed certs. Some

# Linux distributions patch the stack such that the Python SSL

# infrastructure picks up the system installed certs by default, thus no

# action necessary on our part

if certs_file == 'system':

return None

if not certs_file:

global configured_certs_file, temp_certs_file

if not configured_certs_file:

configured_certs_file = os.path.abspath(

os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt'))

if not os.path.exists(configured_certs_file):

# If the file is not present on disk, this means the gslib module

# doesn't actually exist on disk anywhere. This can happen if it's

# being imported from a zip file. Unfortunately, we have to copy the

# certs file to a local temp file on disk because the underlying SSL

# socket requires it to be a filesystem path.

certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt')

if not certs_data:

raise CommandException('Certificates file not found. Please '

'reinstall gsutil from scratch')

certs_data = six.ensure_str(certs_data)

fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts')

f = os.fdopen(fd, 'w')

f.write(certs_data)

f.close()

temp_certs_file = fname

configured_certs_file = temp_certs_file

certs_file = configured_certs_file

return certs_file

def ConfigureNoOpAuthIfNeeded():

"""Sets up no-op auth handler if no boto credentials are configured."""

if not HasConfiguredCredentials():

if (config.has_option('Credentials', 'gs_service_client_id') and

not HAS_CRYPTO):

if system_util.InvokedViaCloudSdk():

raise CommandException('\n'.join(

textwrap.wrap(

'Your gsutil is configured with an OAuth2 service account, but '

'you do not have cryptography installed. '

'Service account authentication requires one of these libraries; '

'please reactivate your service account via the gcloud auth '

'command and ensure any gcloud packages necessary for '

'service accounts are present.')))

else:

raise CommandException('\n'.join(

textwrap.wrap(

'Your gsutil is configured with an OAuth2 service account, but '

'you do not have cryptography installed. '

'Service account authentication requires one of these libraries; '

'please install either of them to proceed, or configure a '

'different type of credentials with "gsutil config".')))

else:

# With no boto config file the user can still access publicly readable

# buckets and objects.

from gslib import no_op_auth_plugin # pylint: disable=unused-variable

def GetCertsFile():

return configured_certs_file

def GetCleanupFiles():

"""Returns a list of temp files to delete (if possible) when program exits."""

return [temp_certs_file] if temp_certs_file else []

def GetConfigFilePaths():

"""Returns a list of the path(s) to the boto config file(s) to be loaded."""

potential_config_paths = BotoConfigLocations

# When Boto's pyami.config.Config class is initialized, it attempts to read

# this file, transform it into valid Boto config syntax, and load credentials

# from it, but it does not add this file to BotoConfigLocations.

if 'AWS_CREDENTIAL_FILE' in os.environ:

potential_config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])

# Provider-specific config files, like the one below, are checked for and

# loaded when Boto's Provider class is initialized. These aren't listed in

# BotoConfigLocations, but can still affect what credentials are loaded, so

# we display them in our config list to make auth'n debugging easier.

aws_cred_file = os.path.join(os.path.expanduser('~'), '.aws', 'credentials')

if os.path.isfile(aws_cred_file):

potential_config_paths.append(aws_cred_file)

# Only return credential files which we have permission to read (and thus can

# actually load credentials from).

readable_config_paths = []

for path in potential_config_paths:

try:

with open(path, 'r'):

readable_config_paths.append(path)

except IOError:

pass

return readable_config_paths

def GetFriendlyConfigFilePaths():

"""Like GetConfigFilePaths but returns a not-found message if paths empty."""

readable_config_paths = GetConfigFilePaths()

if len(readable_config_paths) == 0:

readable_config_paths.append('No config found')

return readable_config_paths

def GetGsutilStateDir():

"""Returns the location of the directory for gsutil state files.

Certain operations, such as cross-process credential sharing and

resumable transfer tracking, need a known location for state files which

are created by gsutil as-needed.

This location should only be used for storing data that is required to be in

a static location.

Returns:

Path to directory for gsutil static state files.

"""

config_file_dir = config.get('GSUtil', 'state_dir', DEFAULT_GSUTIL_STATE_DIR)

system_util.CreateDirIfNeeded(config_file_dir)

return config_file_dir

def GetCredentialStoreFilename():

# As of gsutil 4.29, this changed from 'credstore' to 'credstore2' because

# of a change to the underlying credential storage format.

return os.path.join(GetGsutilStateDir(), 'credstore2')

def GetGceCredentialCacheFilename():

return os.path.join(GetGsutilStateDir(), 'gcecredcache')

def GetGcsJsonApiVersion():

return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION)

# Resumable downloads and uploads make one HTTP call per chunk (and must be

# in multiples of 256KiB). Overridable for testing.

def GetJsonResumableChunkSize():

chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size',

long(1024 * 1024 * 100))

if chunk_size == 0:

chunk_size = long(1024 * 256)

elif chunk_size % long(1024 * 256) != 0:

chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256))))

return chunk_size

def GetLastCheckedForGsutilUpdateTimestampFile():

return os.path.join(GetGsutilStateDir(), '.last_software_update_check')

def GetMaxConcurrentCompressedUploads():

"""Gets the max concurrent transport compressed uploads allowed in parallel.

Returns:

The max number of concurrent transport compressed uploads allowed in

parallel without exceeding the max_upload_compression_buffer_size.

"""

upload_chunk_size = GetJsonResumableChunkSize()

# From apitools compression.py.

compression_chunk_size = 16 * ONE_MIB

total_upload_size = (upload_chunk_size + compression_chunk_size + 17 + 5 *

(((compression_chunk_size - 1) / 16383) + 1))

max_concurrent_uploads = (GetMaxUploadCompressionBufferSize() /

total_upload_size)

if max_concurrent_uploads <= 0:

max_concurrent_uploads = 1

return max_concurrent_uploads

def GetMaxRetryDelay():

return config.getint('Boto', 'max_retry_delay', 32)

def GetMaxUploadCompressionBufferSize():

"""Get the max amount of memory compressed transport uploads may buffer."""

return HumanReadableToBytes(

config.get('GSUtil', 'max_upload_compression_buffer_size', '2GiB'))

def GetNewHttp(http_class=httplib2.Http, **kwargs):

"""Creates and returns a new httplib2.Http instance.

Args:

http_class: Optional custom Http class to use.

**kwargs: Arguments to pass to http_class constructor.

Returns:

An initialized httplib2.Http instance.

"""

##Get Proxy configuration from boto file, defaults are None, 0 and False

proxy_host = config.get('Boto', 'proxy', None) #needed for if clause below

boto_proxy_config = {

'proxy_host':

proxy_host,

'proxy_type':

config.get('Boto', 'proxy_type', 'http'),

'proxy_port':

config.getint('Boto', 'proxy_port'),

'proxy_user':

config.get('Boto', 'proxy_user', None),

'proxy_pass':

config.get('Boto', 'proxy_pass', None),

'proxy_rdns':

config.get('Boto', 'proxy_rdns', True if proxy_host else None)

}

#Use SetProxyInfo to convert boto config to httplib2.proxyinfo object

proxy_info = SetProxyInfo(boto_proxy_config)

# Some installers don't package a certs file with httplib2, so use the

# one included with gsutil.

kwargs['ca_certs'] = GetCertsFile()

# Use a non-infinite SSL timeout to avoid hangs during network flakiness.

kwargs['timeout'] = SSL_TIMEOUT_SEC

http = http_class(proxy_info=proxy_info, **kwargs)

http.disable_ssl_certificate_validation = (not config.getbool(

'Boto', 'https_validate_certificates'))

global_context_config = context_config.get_context_config()

if global_context_config and global_context_config.use_client_certificate:

http.add_certificate(key=global_context_config.client_cert_path,

cert=global_context_config.client_cert_path,

domain='',

password=global_context_config.client_cert_password)

return http

# Retry for 10 minutes with exponential backoff, which corresponds to

# the maximum Downtime Period specified in the GCS SLA

# (https://cloud.google.com/storage/sla)

def GetNumRetries():

return config.getint('Boto', 'num_retries', 23)

def GetTabCompletionLogFilename():

return os.path.join(GetGsutilStateDir(), 'tab-completion-logs')

def GetTabCompletionCacheFilename():

tab_completion_dir = os.path.join(GetGsutilStateDir(), 'tab-completion')

# Limit read permissions on the directory to owner for privacy.

system_util.CreateDirIfNeeded(tab_completion_dir, mode=0o700)

return os.path.join(tab_completion_dir, 'cache')

def HasConfiguredCredentials():

"""Determines if boto credential/config file exists."""

has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and

config.has_option('Credentials', 'gs_secret_access_key'))

has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and

config.has_option('Credentials', 'aws_secret_access_key'))

has_oauth_creds = (config.has_option('Credentials',

'gs_oauth2_refresh_token'))

has_external_creds = (config.has_option('Credentials',

'gs_external_account_file'))

has_external_account_authorized_user_creds = (config.has_option(

'Credentials', 'gs_external_account_authorized_user_file'))

has_service_account_creds = (

HAS_CRYPTO and

config.has_option('Credentials', 'gs_service_client_id') and

config.has_option('Credentials', 'gs_service_key_file'))

if (has_goog_creds or has_amzn_creds or has_oauth_creds or

has_service_account_creds or has_external_creds or

has_external_account_authorized_user_creds):

return True

valid_auth_handler = None

try:

valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost,

config,

Provider('google'),

requested_capability=['s3'])

# Exclude the no-op auth handler as indicating credentials are configured.

# Note we can't use isinstance() here because the no-op module may not be

# imported so we can't get a reference to the class type.

if 'NoOpAuth' == getattr(

getattr(valid_auth_handler, '__class__', None),

'__name__',

None): # yapf: disable

valid_auth_handler = None

except NoAuthHandlerFound:

pass

return valid_auth_handler

def JsonResumableChunkSizeDefined():

chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None)

return chunk_size_defined is not None

def MonkeyPatchBoto():

"""Apply gsutil-specific patches to Boto.

Here be dragons. Sorry.

Note that this method should not be used as a replacement for contributing

fixes to the upstream Boto library. However, the Boto library has historically

not been consistent about release cadence, so upstream fixes may not be

available immediately in a version which we can pin to. Also, some fixes may

only be applicable to gsutil. In such cases, patches should be applied to the

Boto library here (and removed if/when they are included in the upstream

repository and included in an official new release that we pull in). This

method should be invoked after all other Boto-related initialization has been

completed.

"""

# We have to do all sorts of gross things here (dynamic imports, invalid names

# to resolve symbols in copy/pasted methods, invalid spacing from copy/pasted

# methods, etc.), so we just disable pylint warnings for this whole method.

# yapf: disable

# pylint: disable=all

# This should have already been imported if this method was called in the

# correct place, but we need to perform the import within this method for

# this module to recognize it. We don't import this at module level because

# its import timing is important and is managed elsewhere.

import gcs_oauth2_boto_plugin

# TODO(https://github.com/boto/boto/issues/3831): Remove this if the GitHub

# issue is ever addressed.

orig_get_plugin_method = boto.plugin.get_plugin

def _PatchedGetPluginMethod(cls, requested_capability=None):

handler_subclasses = orig_get_plugin_method(

cls, requested_capability=requested_capability)

# In Boto's logic, higher precedence handlers should be closer to the end

# of the list. We always want to prefer OAuth2 credentials over HMAC

# credentials if both are present, so we shuffle OAuth2 handlers to the end

# of the handler list.

xml_oauth2_handlers = (

gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2ServiceAccountAuth,

gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2Auth)

new_result = (

# We need to sort each of these to avoid inconsistent handler selection

# when multiple credential types are configured in Python 3.5. See

# https://issuetracker.google.com/issues/135709541 for specific logs.

sorted(

[r for r in handler_subclasses if r not in xml_oauth2_handlers],

# Types aren't sortable, so we use their names:

key=(lambda handler_t: handler_t.__name__),

) + # Now append XML handlers to the end (highest precedence)

sorted(

[r for r in handler_subclasses if r in xml_oauth2_handlers],

key=(lambda handler_t: handler_t.__name__),

))

return new_result

boto.plugin.get_plugin = _PatchedGetPluginMethod

#########################################################################

# TODO(boto>2.49.0): Remove this.

# Fixes SSL issue where SNI was not being used for OpenSSL 1.1.1+

# https://github.com/boto/boto/pull/3843/files

# Import modules and resolve symbols used by our patch method.

import socket

import ssl

InvalidCertificateException = (

boto.https_connection.InvalidCertificateException)

ValidateCertificateHostname = (

boto.https_connection.ValidateCertificateHostname)

def _PatchedConnectMethod(self):

# The lines below were copied directly from the Boto file, so we don't lint

# or otherwise alter them.

if hasattr(self, "timeout"):

sock = socket.create_connection((self.host, self.port), self.timeout)

else:

sock = socket.create_connection((self.host, self.port))

msg = "wrapping ssl socket; "

if self.ca_certs:

msg += "CA certificate file=%s" % self.ca_certs

else:

msg += "using system provided SSL certs"

boto.log.debug(msg)

if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False):

# Use SSLContext so we can specify server_hostname for SNI

# (Required for connections to storage.googleapis.com)

context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)

context.verify_mode = ssl.CERT_REQUIRED

if self.ca_certs:

context.load_verify_locations(self.ca_certs)

if self.cert_file:

context.load_cert_chain(self.cert_file, self.key_file)

self.sock = context.wrap_socket(sock, server_hostname=self.host)

# Add attributes only set in SSLSocket constructor without context:

self.sock.keyfile = self.key_file

self.sock.certfile = self.cert_file

self.sock.cert_reqs = context.verify_mode

self.sock.ssl_version = ssl.PROTOCOL_SSLv23

self.sock.ca_certs = self.ca_certs

self.sock.ciphers = None

else:

self.sock = ssl.wrap_socket(sock,

keyfile=self.key_file,

certfile=self.cert_file,

cert_reqs=ssl.CERT_REQUIRED,

ca_certs=self.ca_certs)

cert = self.sock.getpeercert()

hostname = self.host.split(':', 0)[0]

if not ValidateCertificateHostname(cert, hostname):

raise InvalidCertificateException(

hostname, cert, 'remote hostname "%s" does not match '

'certificate' % hostname)

# End `_PatchedConnectMethod` declaration.

boto.https_connection.CertValidatingHTTPSConnection.connect = (

_PatchedConnectMethod)

def ProxyInfoFromEnvironmentVar(proxy_env_var):

"""Reads proxy info from the environment and converts to httplib2.ProxyInfo.

Args:

proxy_env_var: Environment variable string to read, such as http_proxy or

https_proxy.

Returns:

httplib2.ProxyInfo constructed from the environment string.

"""

proxy_url = os.environ.get(proxy_env_var)

if not proxy_url or not proxy_env_var.lower().startswith('http'):

return httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0)

proxy_protocol = proxy_env_var.lower().split('_')[0]

if not proxy_url.lower().startswith('http'):

# proxy_info_from_url requires a protocol, which is always http or https.

proxy_url = proxy_protocol + '://' + proxy_url

return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol)

def ResumableThreshold():

return config.getint('GSUtil', 'resumable_threshold', 8 * ONE_MIB)

def SetProxyInfo(boto_proxy_config):

"""Sets proxy info from boto and environment and converts to httplib2.ProxyInfo.

Args:

dict: Values read from the .boto file

Returns:

httplib2.ProxyInfo constructed from boto or environment variable string.

"""

#Defining proxy_type based on httplib2 library, accounting for None entry too.

proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3}

#proxy_type defaults to 'http (3)' for backwards compatibility

proxy_type = proxy_type_spec.get(

boto_proxy_config.get('proxy_type').lower(), proxy_type_spec['http'])

proxy_host = boto_proxy_config.get('proxy_host')

proxy_port = boto_proxy_config.get('proxy_port')

proxy_user = boto_proxy_config.get('proxy_user')

proxy_pass = boto_proxy_config.get('proxy_pass')

proxy_rdns = bool(boto_proxy_config.get('proxy_rdns'))

#For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use

#to http only

proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host,

proxy_type=proxy_type,

proxy_port=proxy_port,

proxy_user=proxy_user,

proxy_pass=proxy_pass,

proxy_rdns=proxy_rdns)

#Added to force socks proxies not to use rdns

if not (proxy_info.proxy_type == proxy_type_spec['http']):

proxy_info.proxy_rdns = False

if not (proxy_info.proxy_host and proxy_info.proxy_port):

# Fall back to using the environment variable. Use only http proxies.

for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:

if proxy_env_var in os.environ and os.environ[proxy_env_var]:

proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)

# Assume proxy_rnds is True if a proxy environment variable exists.

if boto_proxy_config.get('proxy_rdns') == None:

proxy_info.proxy_rdns = True

break

return proxy_info

def UsingCrcmodExtension():

boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None)

if boto_opt is not None:

return boto_opt

# Python 3 makes this attribute tough to access due to the way the top-level

# crcmod package imports and (identically) names its crcmod module. The only

# way to get it is "from crcmod.crcmod import _usingExtension". This is the

# alternative form of that statement, but doesn't pollute this module's

# namespace with a "_usingExtension" attribute. This also works in both Python

# 2.7 and 3.5+.

nested_crcmod = __import__(

'crcmod.crcmod',

globals(),

locals(),

['_usingExtension'],

0,

)

return getattr(nested_crcmod, '_usingExtension', False)

# TODO(boto-2.49.0): Remove when we pull in the next version of Boto.

def _PatchedShouldRetryMethod(self, response, chunked_transfer=False):

"""Replaces boto.s3.key's should_retry() to handle KMS-encrypted objects."""

# We copy/pasted this from boto and slightly modified it; keep the old

# formatting style.

# yapf: disable

provider = self.bucket.connection.provider

if not chunked_transfer:

if response.status in [500, 503]:

# 500 & 503 can be plain retries.

return True

if response.getheader('location'):

# If there's a redirect, plain retry.

return True

if 200 <= response.status <= 299:

self.etag = response.getheader('etag')

md5 = self.md5

if isinstance(md5, bytes):

md5 = md5.decode(UTF8)

# If you use customer-provided encryption keys, the ETag value that

# Amazon S3 returns in the response will not be the MD5 of the

# object.

amz_server_side_encryption_customer_algorithm = response.getheader(

'x-amz-server-side-encryption-customer-algorithm', None)

# The same is applicable for KMS-encrypted objects in gs buckets.

goog_customer_managed_encryption = response.getheader(

'x-goog-encryption-kms-key-name', None)

if (amz_server_side_encryption_customer_algorithm is None and

goog_customer_managed_encryption is None):

if self.etag != '"%s"' % md5:

raise provider.storage_data_error(

'ETag from S3 did not match computed MD5. '

'%s vs. %s' % (self.etag, self.md5))

return True

if response.status == 400:

# The 400 must be trapped so the retry handler can check to

# see if it was a timeout.

# If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb

# out.

body = response.read()

err = provider.storage_response_error(

response.status,

response.reason,

body

)

if err.error_code in ['RequestTimeout']:

raise boto.exception.PleaseRetryException(

"Saw %s, retrying" % err.error_code,

response=response

)

return False

def HasUserSpecifiedGsHost():

gs_host = boto.config.get('Credentials', 'gs_host', None)

default_host = six.ensure_str(boto.gs.connection.GSConnection.DefaultHost)

if gs_host is not None:

return default_host == six.ensure_str(gs_host)

return False

def UsingGsHmac():

# NOTE: External credentials are omitted intentionally as HMAC takes priority

# over the external credentials types.

config = boto.config

has_refresh_token = config.has_option('Credentials', 'gs_oauth2_refresh_token')

has_service_account_credentials = (

config.has_option('Credentials', 'gs_service_client_id')

and config.has_option('Credentials', 'gs_service_key_file'))

has_hmac_credentials = (

config.has_option('Credentials', 'gs_access_key_id')

and config.has_option('Credentials', 'gs_secret_access_key'))

return (not has_refresh_token and not has_service_account_credentials

and has_hmac_credentials)