Make storage upload have no chunk size by default.

dhermes · dhermes · commit afc9994bdf82 · 2015-02-09T21:22:20.000-08:00
Fixes #546.
diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py
@@ -271,7 +271,8 @@ def download_as_string(self):
         return string_buffer.getvalue()
 
     def upload_from_file(self, file_obj, rewind=False, size=None,
-                         content_type=None, num_retries=6):
+                         content_type=None, num_retries=6,
+                         upload_chunk_size=None):
         """Upload the contents of this blob from a file-like object.
 
         .. note::
@@ -285,6 +286,11 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
            `lifecycle <https://cloud.google.com/storage/docs/lifecycle>`_
            API documents for details.
 
+        The current ``Blob``'s chunk size is not used by default. Default
+        behavior is instead to attempt uploading the entire object. See
+        https://github.com/GoogleCloudPlatform/gcloud-python/issues/546
+        for more details.
+
         :type file_obj: file
         :param file_obj: A file handle open for reading.
 
@@ -296,6 +302,15 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
         :param size: The number of bytes to read from the file handle.
                      If not provided, we'll try to guess the size using
                      :func:`os.fstat`
+
+        :type content_type: string or ``NoneType``
+        :param content_type: Optional content type of uploaded content.
+
+        :type num_retries: int
+        :param num_retries: Optional number of retries. Defaults to 6.
+
+        :type upload_chunk_size: int or ``NoneType``
+        :param upload_chunk_size: Optional size of chunks to upload with.
         """
         # Rewind the file if desired.
         if rewind:
@@ -313,7 +328,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
         upload = transfer.Upload(file_obj,
                                  content_type or 'application/unknown',
                                  total_bytes, auto_transfer=False,
-                                 chunksize=self.CHUNK_SIZE)
+                                 chunksize=upload_chunk_size)
 
         url_builder = _UrlBuilder(bucket_name=self.bucket.name,
                                   object_name=self.name)
@@ -611,7 +626,7 @@ def updated(self):
 
 
 class _UploadConfig(object):
-    """ Faux message FBO apitools' 'ConfigureRequest'.
+    """Faux message for benefit of apitools' 'ConfigureRequest'.
 
     Values extracted from apitools
     'samples/storage_sample/storage/storage_v1_client.py'
@@ -625,7 +640,7 @@ class _UploadConfig(object):
 
 
 class _UrlBuilder(object):
-    """Faux builder FBO apitools' 'ConfigureRequest'"""
+    """Faux builder for benefit of apitools' 'ConfigureRequest'"""
     def __init__(self, bucket_name, object_name):
         self.query_params = {'name': object_name}
         self._bucket_name = bucket_name
diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py
@@ -382,13 +382,14 @@ def test_upload_from_file_resumable(self):
         )
         bucket = _Bucket(connection)
         blob = self._makeOne(BLOB_NAME, bucket=bucket)
-        blob.CHUNK_SIZE = 5
+        UPLOAD_CHUNK_SIZE = 5
         # Set the threshhold low enough that we force a resumable uploada.
         with _Monkey(transfer, _RESUMABLE_UPLOAD_THRESHOLD=5):
             with NamedTemporaryFile() as fh:
                 fh.write(DATA)
                 fh.flush()
-                blob.upload_from_file(fh, rewind=True)
+                blob.upload_from_file(fh, rewind=True,
+                                      upload_chunk_size=UPLOAD_CHUNK_SIZE)
         rq = connection.http._requested
         self.assertEqual(len(rq), 3)
         self.assertEqual(rq[0]['method'], 'POST')
@@ -408,16 +409,18 @@ def test_upload_from_file_resumable(self):
         self.assertEqual(rq[1]['uri'], UPLOAD_URL)
         headers = dict(
             [(x.title(), str(y)) for x, y in rq[1]['headers'].items()])
-        self.assertEqual(rq[1]['body'], DATA[:5])
+        self.assertEqual(rq[1]['body'], DATA[:UPLOAD_CHUNK_SIZE])
         headers = dict(
             [(x.title(), str(y)) for x, y in rq[1]['headers'].items()])
         self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
         self.assertEqual(rq[2]['method'], 'PUT')
         self.assertEqual(rq[2]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[2]['body'], DATA[5:])
+        self.assertEqual(rq[2]['body'], DATA[UPLOAD_CHUNK_SIZE:])
         headers = dict(
             [(x.title(), str(y)) for x, y in rq[2]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
+        content_range = 'bytes %d-%d/%d' % (
+            UPLOAD_CHUNK_SIZE, UPLOAD_CHUNK_SIZE, len(DATA))
+        self.assertEqual(headers['Content-Range'], content_range)
 
     def test_upload_from_file_w_slash_in_name(self):
         from six.moves.http_client import OK
