GoogleCloud: Add GS_BLOB_CHUNK_SIZE setting

docs/backends/gcloud.rst

@@ -138,6 +138,15 @@ By default files with the same name will overwrite each other. Set this to ``Fal
 The maximum amount of memory a returned file can take up (in bytes) before being
 rolled over into a temporary file on disk. Default is 0: Do not roll over.
 
+``GS_BLOB_CHUNK_SIZE`` (optional: default is ``None``)
+
+The size of blob chunks that are sent via resumable upload. If this is not set then the generated request
+must fit in memory. Recommended if you are going to be uploading large files.
+
+.. note::
+
+ This must be a multiple of 256K (1024 * 256)
+
 ``GS_CACHE_CONTROL`` (optional: default is ``None``)
 
 Sets Cache-Control HTTP header for the file, more about HTTP caching can be found `here <https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching#cache-control>`_

storages/backends/gcloud.py

@@ -30,7 +30,9 @@ def __init__(self, name, mode, storage):
  self._storage = storage
  self.blob = storage.bucket.get_blob(name)
  if not self.blob and 'w' in mode:
- self.blob = Blob(self.name, storage.bucket)
+ self.blob = Blob(
+ self.name, storage.bucket,
+ chunk_size=setting('GS_BLOB_CHUNK_SIZE'))
  self._file = None
  self._is_dirty = False
 

tests/test_gcloud.py

@@ -86,7 +86,7 @@ def test_open_write(self, MockBlob):
  self.storage.default_acl = 'projectPrivate'
 
  f = self.storage.open(self.filename, 'wb')
- MockBlob.assert_called_with(self.filename, self.storage._bucket)
+ MockBlob.assert_called_with(self.filename, self.storage._bucket, chunk_size=None)
 
  f.write(data)
  tmpfile = f._file