Upload delete of files larger than 5 Gig to Swift #35

As coded, the low level Swift Connection is used to upload and delete
files. However, this class does not handle the upload and deletion of
files over 5 Gig in size that need to be segmented.

This commit replaces the use of the Swift Connection class with the
SwiftService class in these two cases. Files that are larger than
5 Gig will now be broken into segments on the 5 Gig boundary, and the
delete call will be able to delete the segmented files.

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,5 +1,7 @@
 """Provider implementation based on OpenStack Python clients for OpenStack."""
 
+import inspect
+
 import os
 
 from cinderclient import client as cinder_client
@@ -236,19 +238,58 @@ class OpenStackCloudProvider(BaseCloudProvider):
 #         return glance_client.Client(version=api_version,
 #                                     session=self.keystone.session)
 
-    def _connect_swift(self):
+    @staticmethod
+    def _clean_options(options, method_to_match):
+        """
+        Returns a copy of the source options with all keys that are not in the
+        ``method_to_match`` parameter list removed.
+
+        :param options: The source options.
+        :type options: ``dict``
+        :param method_to_match: The method whose signature is to be matched
+        :type method_to_match: A callable
+        :return: A copy of the source options with all keys that are not in the
+            ``method_to_match`` parameter list removed. If options is ``None``
+            then this will be an empty dictionary
+        :rtype: ``dict``
+        """
+        result = dict(options or {})
+        if len(result):
+            # Don't allow the options to override our authentication
+            result['os_options'] = None
+            passed_in_options = set(result.keys())
+            try:
+                method_signature = inspect.signature(method_to_match)
+                parameters = set(method_signature.parameters.keys())
+            except AttributeError:
+                parameters = set(inspect.getargspec(method_to_match)[0])
+            difference = passed_in_options - parameters
+            for name in difference:
+                del result[name]
+        return result
+
+    def _connect_swift(self, options=None):
+        """
+        Get an OpenStack Swift (object store) client connection.
+
+        :param options: A dictionary of options from which values will be
+            passed to the connection.
+        :return: A Swift client connection using the auth credentials held by
+            the OpenStackCloudProvider instance
+        """
+        clean_options = self._clean_options(options,
+                                            swift_client.Connection.__init__)
         storage_url = self._get_config_value(
             'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
         auth_token = self._get_config_value(
             'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
-
-        """Get an OpenStack Swift (object store) client object cloud."""
         if storage_url and auth_token:
-            return swift_client.Connection(preauthurl=storage_url,
-                                           preauthtoken=auth_token)
+            clean_options['preauthurl'] = storage_url
+            clean_options['preauthtoken'] = auth_token
         else:
-            return swift_client.Connection(authurl=self.auth_url,
-                                           session=self._keystone_session)
+            clean_options['authurl'] = self.auth_url
+            clean_options['session'] = self._keystone_session
+        return swift_client.Connection(**clean_options)
 
     def _connect_neutron(self):
         """Get an OpenStack Neutron (networking) client object cloud."""

cloudbridge/cloud/providers/openstack/resources.py

@@ -5,6 +5,8 @@ import inspect
 import ipaddress
 import json
 
+import os
+
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
 from cloudbridge.cloud.base.resources import BaseBucket
 from cloudbridge.cloud.base.resources import BaseBucketObject
@@ -35,7 +37,12 @@ from keystoneclient.v3.regions import Region
 
 import novaclient.exceptions as novaex
 
-import swiftclient.exceptions as swiftex
+import swiftclient
+
+from swiftclient.service import SwiftService, SwiftUploadObject
+
+ONE_GIG = 1048576000  # in bytes
+FIVE_GIG = ONE_GIG * 5  # in bytes
 
 
 class OpenStackMachineImage(BaseMachineImage):
@@ -1076,16 +1083,47 @@ class OpenStackBucketObject(BaseBucketObject):
         """
         Set the contents of this object to the data read from the source
         string.
+
+        .. warning:: Will fail if the data is larger than 5 Gig.
         """
         self._provider.swift.put_object(self.cbcontainer.name, self.name,
                                         data)
 
     def upload_from_file(self, path):
         """
-        Stores the contents of the file pointed by the "path" variable.
-        """
-        with open(path, 'rb') as f:
-            self.upload(f)
+        Stores the contents of the file pointed by the ``path`` variable.
+        If the file is bigger than 5 Gig, it will be broken into segments.
+
+        :type path: ``str``
+        :param path: Absolute path to the file to be uploaded to Swift.
+        :rtype: ``bool``
+        :return: ``True`` if successful, ``False`` if not.
+
+        .. note::
+            * The size of the segments chosen (or any of the other upload
+              options) is not under user control.
+            * If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
+
+        .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
+        """
+        upload_options = {}
+        if 'segment_size' not in upload_options:
+            if os.path.getsize(path) >= FIVE_GIG:
+                upload_options['segment_size'] = FIVE_GIG
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            upload_object = SwiftUploadObject(path, object_name=self.name)
+            for up_res in swift.upload(self.cbcontainer.name,
+                                       [upload_object, ],
+                                       options=upload_options):
+                result = result and up_res['success']
+        return result
 
     def delete(self):
         """
@@ -1093,14 +1131,20 @@ class OpenStackBucketObject(BaseBucketObject):
 
         :rtype: ``bool``
         :return: True if successful
+
+        .. note:: If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
         """
-        try:
-            self._provider.swift.delete_object(self.cbcontainer.name,
-                                               self.name)
-        except swiftex.ClientException as err:
-            if err.http_status == 404:
-                return True
-        return False
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            for del_res in swift.delete(self.cbcontainer.name, [self.name, ]):
+                result = result and del_res['success']
+        return result
 
     def generate_url(self, expires_in=0):
         """

test/test_object_store_service.py

@@ -1,4 +1,6 @@
+import filecmp
 import os
+import tempfile
 import uuid
 
 from datetime import datetime
@@ -207,3 +209,33 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                 obj.save_content(target_stream)
                 with open(test_file, 'rb') as f:
                     self.assertEqual(target_stream.getvalue(), f.read())
+
+    @skip("Skip unless you want to test swift objects bigger than 5 Gig")
+    @helpers.skipIfNoService(['object_store'])
+    def test_upload_download_bucket_content_with_large_file(self):
+        """
+        Creates a 6 Gig file in the temp directory, then uploads it to
+        Swift. Once uploaded, then downloads to a new file in the temp
+        directory and compares the two files to see if they match.
+        """
+        temp_dir = tempfile.gettempdir()
+        file_name = '6GigTest.tmp'
+        six_gig_file = os.path.join(temp_dir, file_name)
+        with open(six_gig_file, "wb") as out:
+            out.truncate(6 * 1024 * 1024 * 1024)  # 6 Gig...
+        with helpers.cleanup_action(lambda: os.remove(six_gig_file)):
+            download_file = "{0}/cbtestfile-{1}".format(temp_dir, file_name)
+            bucket_name = "cbtestbucketlargeobjs-{0}".format(uuid.uuid4())
+            test_bucket = self.provider.object_store.create(bucket_name)
+            with helpers.cleanup_action(lambda: test_bucket.delete()):
+                test_obj = test_bucket.create_object(file_name)
+                with helpers.cleanup_action(lambda: test_obj.delete()):
+                    file_uploaded = test_obj.upload_from_file(six_gig_file)
+                    self.assertTrue(file_uploaded, "Could not upload object?")
+                    with helpers.cleanup_action(
+                            lambda: os.remove(download_file)):
+                        with open(download_file, 'wb') as f:
+                            test_obj.save_content(f)
+                            self.assertTrue(
+                                filecmp.cmp(six_gig_file, download_file),
+                                "Uploaded file != downloaded")