Merge pull request #41 from MartinPaulo/issue_35

MartinPaulo Upload delete of files larger than 5 Gig to Swift #35

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,5 +1,7 @@
 """Provider implementation based on OpenStack Python clients for OpenStack."""
 
+import inspect
+
 import os
 
 from cinderclient import client as cinder_client
@@ -236,19 +238,66 @@ class OpenStackCloudProvider(BaseCloudProvider):
 #         return glance_client.Client(version=api_version,
 #                                     session=self.keystone.session)
 
-    def _connect_swift(self):
+    @staticmethod
+    def _clean_options(options, method_to_match):
+        """
+        Returns a **copy** of the source options with all keys that are not in
+        the ``method_to_match`` parameter list removed.
+
+        .. note:: If ``options`` has the ``os_options`` key it will have
+            both the key and its value removed. This is because any entries
+            in this dictionary value will override our settings. This
+            situation is only going to happen when the `_connect_swift`
+            method is called by the SwiftService to manufacture new
+            connections.
+
+        .. seealso::
+            https://docs.openstack.org/developer/python-swiftclient/swiftclient.html#module-swiftclient.client
+
+        :param options: The source options.
+        :type options: ``dict``
+        :param method_to_match: The method whose signature is to be matched
+        :type method_to_match: A callable
+        :return: A copy of the source options with all keys that are not in the
+            ``method_to_match`` parameter list removed. If options is ``None``
+            then this will be an empty dictionary
+        :rtype: ``dict``
+        """
+        result = {}
+        if options:
+            try:
+                method_signature = inspect.signature(method_to_match)
+                parameters = set(method_signature.parameters.keys())
+            except AttributeError:
+                parameters = set(inspect.getargspec(method_to_match).args)
+            result = {key: val for key, val in options.items() if
+                      key in parameters}
+            # Don't allow the options to override our authentication
+            result.pop('os_options', None)
+        return result
+
+    def _connect_swift(self, options=None):
+        """
+        Get an OpenStack Swift (object store) client connection.
+
+        :param options: A dictionary of options from which values will be
+            passed to the connection.
+        :return: A Swift client connection using the auth credentials held by
+            the OpenStackCloudProvider instance
+        """
+        clean_options = self._clean_options(options,
+                                            swift_client.Connection.__init__)
         storage_url = self._get_config_value(
             'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
         auth_token = self._get_config_value(
             'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
-
-        """Get an OpenStack Swift (object store) client object cloud."""
         if storage_url and auth_token:
-            return swift_client.Connection(preauthurl=storage_url,
-                                           preauthtoken=auth_token)
+            clean_options['preauthurl'] = storage_url
+            clean_options['preauthtoken'] = auth_token
         else:
-            return swift_client.Connection(authurl=self.auth_url,
-                                           session=self._keystone_session)
+            clean_options['authurl'] = self.auth_url
+            clean_options['session'] = self._keystone_session
+        return swift_client.Connection(**clean_options)
 
     def _connect_neutron(self):
         """Get an OpenStack Neutron (networking) client object cloud."""

cloudbridge/cloud/providers/openstack/resources.py

@@ -5,6 +5,8 @@ import inspect
 import ipaddress
 import json
 
+import os
+
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
 from cloudbridge.cloud.base.resources import BaseBucket
 from cloudbridge.cloud.base.resources import BaseBucketObject
@@ -35,7 +37,12 @@ from keystoneclient.v3.regions import Region
 
 import novaclient.exceptions as novaex
 
-import swiftclient.exceptions as swiftex
+import swiftclient
+
+from swiftclient.service import SwiftService, SwiftUploadObject
+
+ONE_GIG = 1048576000  # in bytes
+FIVE_GIG = ONE_GIG * 5  # in bytes
 
 
 class OpenStackMachineImage(BaseMachineImage):
@@ -1076,16 +1083,47 @@ class OpenStackBucketObject(BaseBucketObject):
         """
         Set the contents of this object to the data read from the source
         string.
+
+        .. warning:: Will fail if the data is larger than 5 Gig.
         """
         self._provider.swift.put_object(self.cbcontainer.name, self.name,
                                         data)
 
     def upload_from_file(self, path):
         """
-        Stores the contents of the file pointed by the "path" variable.
-        """
-        with open(path, 'rb') as f:
-            self.upload(f)
+        Stores the contents of the file pointed by the ``path`` variable.
+        If the file is bigger than 5 Gig, it will be broken into segments.
+
+        :type path: ``str``
+        :param path: Absolute path to the file to be uploaded to Swift.
+        :rtype: ``bool``
+        :return: ``True`` if successful, ``False`` if not.
+
+        .. note::
+            * The size of the segments chosen (or any of the other upload
+              options) is not under user control.
+            * If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
+
+        .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
+        """
+        upload_options = {}
+        if 'segment_size' not in upload_options:
+            if os.path.getsize(path) >= FIVE_GIG:
+                upload_options['segment_size'] = FIVE_GIG
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            upload_object = SwiftUploadObject(path, object_name=self.name)
+            for up_res in swift.upload(self.cbcontainer.name,
+                                       [upload_object, ],
+                                       options=upload_options):
+                result = result and up_res['success']
+        return result
 
     def delete(self):
         """
@@ -1093,14 +1131,20 @@ class OpenStackBucketObject(BaseBucketObject):
 
         :rtype: ``bool``
         :return: True if successful
+
+        .. note:: If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
         """
-        try:
-            self._provider.swift.delete_object(self.cbcontainer.name,
-                                               self.name)
-        except swiftex.ClientException as err:
-            if err.http_status == 404:
-                return True
-        return False
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            for del_res in swift.delete(self.cbcontainer.name, [self.name, ]):
+                result = result and del_res['success']
+        return result
 
     def generate_url(self, expires_in=0):
         """

test/test_object_store_service.py

@@ -1,4 +1,6 @@
+import filecmp
 import os
+import tempfile
 import uuid
 
 from datetime import datetime
@@ -207,3 +209,33 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                 obj.save_content(target_stream)
                 with open(test_file, 'rb') as f:
                     self.assertEqual(target_stream.getvalue(), f.read())
+
+    @skip("Skip unless you want to test swift objects bigger than 5 Gig")
+    @helpers.skipIfNoService(['object_store'])
+    def test_upload_download_bucket_content_with_large_file(self):
+        """
+        Creates a 6 Gig file in the temp directory, then uploads it to
+        Swift. Once uploaded, then downloads to a new file in the temp
+        directory and compares the two files to see if they match.
+        """
+        temp_dir = tempfile.gettempdir()
+        file_name = '6GigTest.tmp'
+        six_gig_file = os.path.join(temp_dir, file_name)
+        with open(six_gig_file, "wb") as out:
+            out.truncate(6 * 1024 * 1024 * 1024)  # 6 Gig...
+        with helpers.cleanup_action(lambda: os.remove(six_gig_file)):
+            download_file = "{0}/cbtestfile-{1}".format(temp_dir, file_name)
+            bucket_name = "cbtestbucketlargeobjs-{0}".format(uuid.uuid4())
+            test_bucket = self.provider.object_store.create(bucket_name)
+            with helpers.cleanup_action(lambda: test_bucket.delete()):
+                test_obj = test_bucket.create_object(file_name)
+                with helpers.cleanup_action(lambda: test_obj.delete()):
+                    file_uploaded = test_obj.upload_from_file(six_gig_file)
+                    self.assertTrue(file_uploaded, "Could not upload object?")
+                    with helpers.cleanup_action(
+                            lambda: os.remove(download_file)):
+                        with open(download_file, 'wb') as f:
+                            test_obj.save_content(f)
+                            self.assertTrue(
+                                filecmp.cmp(six_gig_file, download_file),
+                                "Uploaded file != downloaded")