Implemented integration test for azure object store

cloudbridge/cloud/providers/azure/azure_client.py

@@ -41,10 +41,6 @@ class AzureClient(object):
     def storage_account_name(self):
         return self._config.get('azure_storage_account_name')
 
-    @property
-    def block_blob_service(self):
-        return self._storage_client
-
     @property
     def storage_client(self):
         return self._storage_client
@@ -112,7 +108,10 @@ class AzureClient(object):
         return self.blob_service.get_container_properties(container_name)
 
     def get_container(self, container_name):
-        return self.blob_service.get_container_properties(container_name)
+        try:
+            return self.blob_service.get_container_properties(container_name)
+        except AzureMissingResourceHttpError:
+            return None
 
     def delete_container(self, container_name):
         self.blob_service.delete_container(container_name)

cloudbridge/cloud/providers/azure/services.py

@@ -93,8 +93,11 @@ class AzureObjectStoreService(BaseObjectStoreService):
                                      limit=limit, marker=marker)
 
     def list(self, limit=None, marker=None):
-        raise NotImplementedError(
-            "AzureObjectStoreService does not implement this method")
+        object_stores = [AzureBucket(self.provider, object_store)
+                   for object_store in
+                   self.provider.azure_client.list_containers()]
+        return ClientPagedResultList(self.provider, object_stores,
+                                     limit=limit, marker=marker)
 
     def create(self, name, location=None):
         object_store = self.provider.azure_client.create_container(name)

integration_test/__init__.py

@@ -0,0 +1,43 @@
+"""
+Tests the functionality of each provider implementation against registered
+test cases. These tests require that provider credentials are set as
+environment variables, as required for each provider (see `tox.ini` for a list
+of env variables).
+
+Since the tests exercise the ``cloudbridge`` interfaces, and there are multiple
+implementations of these interfaces, for m interfaces and n implementation,
+exercising all interfaces means that m*n test case classes are needed.
+Otherwise, the standard test runners such as unittest and nose2 do not
+correctly pick up the tests.
+
+To avoid an explosion of repetitive test cases, the
+``ProviderTestCaseGenerator`` class will automatically generate a new Python
+class for each combination of test and provider. The ``load_tests`` protocol
+(https://docs.python.org/2/library/unittest.html#load-tests-protocol)
+is used to aid test discovery.
+
+Use ``python setup.py test`` to run these unit tests (alternatively, use
+``python -m unittest test``).
+
+All test cases need to be registered below, and available providers will be
+discovered through the ``ProviderFactory``. Test Cases must not inherit from
+``unittest.TestCase``, to avoid confusing unittest and nose2's automatic
+discovery. (The test generator will automatically add ``unittest.TestCase``
+as a base class to each combination).
+"""
+import cloudbridge
+from integration_test.test_integration_azure_object_store_service import AzureIntegrationObjectStoreServiceTestCase
+from integration_test.helpers import ProviderTestCaseGenerator
+
+PROVIDER_TESTS ={'azure': [
+     AzureIntegrationObjectStoreServiceTestCase
+]}
+
+
+def load_tests(loader=None, tests=None, pattern=None):
+    """
+    This function is required to aid the load_tests protocol
+    (https://docs.python.org/2/library/unittest.html#load-tests-protocol)
+    """
+    cloudbridge.init_logging()
+    return ProviderTestCaseGenerator(PROVIDER_TESTS).generate_tests()

integration_test/helpers.py

@@ -0,0 +1,196 @@
+import functools
+import os
+import sys
+import unittest
+from contextlib import contextmanager
+
+from six import reraise
+
+from cloudbridge.cloud.factory import CloudProviderFactory
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
+
+
+def parse_bool(val):
+    if val:
+        return str(val).upper() in ['TRUE', 'YES']
+    else:
+        return False
+
+
+@contextmanager
+def cleanup_action(cleanup_func):
+    """n csdmmnd
+    Context manager to carry out a given
+    cleanup action after carrying out a set
+    of tasks, or when an exception occurs.
+    If any errors occur during the cleanup
+    action, those are ignored, and the original
+    traceback is preserved.
+
+    :params func: This function is called if
+    an exception occurs or at the end of the
+    context block. If any exceptions raised
+        by func are ignored.
+    Usage:
+        with cleanup_action(lambda e: print("Oops!")):
+            do_something()
+    """
+    try:
+        yield
+    except Exception:
+        ex_class, ex_val, ex_traceback = sys.exc_info()
+        try:
+            cleanup_func()
+        except Exception as e:
+            print("Error during exception cleanup: {0}".format(e))
+        reraise(ex_class, ex_val, ex_traceback)
+    try:
+        cleanup_func()
+    except Exception as e:
+        print("Error during cleanup: {0}".format(e))
+
+
+def skipIfNoService(services):
+    """
+    A decorator for skipping tests if the provider
+    does not implement a given service.
+    """
+    def wrap(func):
+        """
+        The actual wrapper
+        """
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            provider = getattr(self, 'provider')
+            if provider:
+                for service in services:
+                    if not provider.has_service(service):
+                        self.skipTest("Skipping test because '%s' service is"
+                                      " not implemented" % (service,))
+            func(self, *args, **kwargs)
+        return wrapper
+    return wrap
+
+
+class ProviderTestBase(object):
+
+    """
+    A dummy base class for Test Cases. Does not inherit from unittest.TestCase
+    to avoid confusing test discovery by unittest and nose2. unittest.TestCase
+    is injected as a base class by the generator, so calling the unittest
+    constructor works correctly.
+    """
+
+    def __init__(self, methodName, provider):
+        unittest.TestCase.__init__(self, methodName=methodName)
+        self.provider = provider
+
+    def setUp(self):
+        if isinstance(self.provider, TestMockHelperMixin):
+            self.provider.setUpMock()
+
+    def tearDown(self):
+        if isinstance(self.provider, TestMockHelperMixin):
+            self.provider.tearDownMock()
+
+
+class ProviderTestCaseGenerator():
+
+    """
+    Generates test cases for all provider - testcase combinations.
+    Detailed docs at test/__init__.py
+    """
+
+    def __init__(self, test_classes):
+        self.all_test_classes = test_classes
+        self.provider_name = os.environ.get("CB_TEST_PROVIDER", "azure")
+        self.use_mock_providers = os.environ.get("CB_USE_MOCK_PROVIDERS", False)
+
+    def get_provider_wait_interval(self, provider_class):
+        if issubclass(provider_class, TestMockHelperMixin):
+            return 0
+        else:
+            return 1
+
+    def create_provider_instance(self, provider_class):
+        """
+        Instantiate a default provider instance. All required connection
+        settings are expected to be set as environment variables.
+        """
+        config = {'default_wait_interval':
+                  self.get_provider_wait_interval(provider_class),
+                  'azure_subscription_id': '7904d702-e01c-4826-8519-f5a25c866a96',
+                  'azure_client_Id': '69621fe1-f59f-43de-8799-269007c76b95',
+                  'azure_secret': 'Orcw9U5Kd4cUDntDABg0dygN32RQ4FGBYyLRaJ/BlrM=',
+                  'azure_tenant': '75ec242e-054d-4b22-98a9-a4602ebb6027'
+                  }
+
+        return provider_class(config)
+
+    def generate_new_test_class(self, name, testcase_class):
+        """
+        Generates a new type which inherits from the given testcase_class and
+        unittest.TestCase
+        """
+        class_name = "{0}{1}".format(name, testcase_class.__name__)
+        return type(class_name, (testcase_class, unittest.TestCase), {})
+
+    def generate_test_suite_for_provider_testcase(
+            self, provider_class, testcase_class):
+        """
+        Generate and return a suite of tests for a specific provider class and
+        testcase combination
+        """
+        testloader = unittest.TestLoader()
+        testnames = testloader.getTestCaseNames(testcase_class)
+        suite = unittest.TestSuite()
+        for name in testnames:
+            generated_cls = self.generate_new_test_class(
+                provider_class.__name__,
+                testcase_class)
+            suite.addTest(
+                generated_cls(
+                    name,
+                    self.create_provider_instance(provider_class)))
+        return suite
+
+    def generate_test_suite_for_provider(self, provider_class):
+        """
+        Generate and return a suite of all available tests for a given provider
+        class
+        """
+        suite = unittest.TestSuite()
+        suites = [
+            self.generate_test_suite_for_provider_testcase(
+                provider_class, test_class)
+            for test_class in self.all_test_classes.get(self.provider_name)]
+        for s in suites:
+            suite.addTest(s)
+        return suite
+
+    def generate_tests(self):
+        """
+        Generate and return a suite of tests for all provider and test class
+        combinations
+        """
+        factory = CloudProviderFactory()
+        use_mock_drivers = parse_bool(self.use_mock_providers)
+        provider_name = self.provider_name
+        if provider_name:
+            provider_classes = [
+                factory.get_provider_class(
+                    provider_name,
+                    get_mock=use_mock_drivers)]
+            if not provider_classes[0]:
+                raise ValueError(
+                    "Could not find specified test provider %s" %
+                    provider_name)
+        else:
+            provider_classes = factory.get_all_provider_classes(
+                get_mock=use_mock_drivers)
+        suite = unittest.TestSuite()
+        suites = [
+            self.generate_test_suite_for_provider(p) for p in provider_classes]
+        for s in suites:
+            suite.addTest(s)
+        return suite

integration_test/test_integration_azure_object_store_service.py

@@ -0,0 +1,79 @@
+import os
+import tempfile
+import uuid
+
+import integration_test.helpers as helpers
+
+
+class AzureIntegrationObjectStoreServiceTestCase(helpers.ProviderTestBase):
+
+    def __init__(self, methodName, provider):
+        super(AzureIntegrationObjectStoreServiceTestCase, self).__init__(
+            methodName=methodName, provider=provider)
+
+    @helpers.skipIfNoService(['object_store'])
+    def test_azure_bucket_service(self):
+        container_name = '{0}'.format(uuid.uuid4())
+        object_name = '{0}'.format(uuid.uuid4())
+
+        containers_count1 = len(self.provider.object_store.list())
+
+        container = self.provider.object_store.create(container_name)
+        self.assertTrue(container is not None , 'Container {0} not created'.format(container_name))
+
+        containers_count2 = len(self.provider.object_store.list())
+        self.assertTrue(containers_count2 > containers_count1, 'Container {0} not present in list'.format(container_name))
+
+        find_container = self.provider.object_store.find(container_name)
+        self.assertTrue(len(find_container) == 1, 'Container {0} not found'.format(container_name))
+
+        get_container = self.provider.object_store.get(container.id)
+        self.assertTrue(get_container is not None, 'Unable to get the container {0}'.format(container_name))
+
+        obj = container.create_object(object_name)
+        self.assertTrue(obj is not None, 'Object {0} not created'.format(container_name))
+
+        obj_count = len(container.list())
+        self.assertTrue(obj_count == 1, 'Object count should be 1')
+
+        get_obj = container.get(object_name)
+        self.assertTrue(get_obj is not None, 'Unable to get object {0} from container {1}.'.format(object_name, container_name))
+
+        exits = container.exists(object_name)
+        self.assertTrue(exits, 'Object {0} not exists in container {1}'.format(object_name, container_name))
+
+        obj_content = 'abc'
+        obj.upload(obj_content)
+
+        content = obj.iter_content()
+        self.assertTrue(content == obj_content, 'Object {0} content should be {1}'.format(object_name, obj_content))
+
+        file_name = 'mytest.txt'
+        file_content = 'defaults'
+
+        tmp = os.path.join(tempfile.gettempdir(), file_name)
+
+        try:
+            if not os.path.exists(tmp):
+                with open(tmp, "w") as file:
+                    file.write(file_content)
+
+            obj.upload_from_file(tmp)
+
+            content = obj.iter_content()
+            self.assertTrue(content == file_content, 'Object {0} content should be {1}'.format(object_name, file_content))
+
+        finally:
+            print('Deleting file')
+            os.remove(tmp)
+
+        url = obj.generate_url()
+        self.assertTrue(url is not None, 'Url should not be None')
+
+        obj.delete()
+        delete_obj = container.get(object_name)
+        self.assertTrue(delete_obj is None, 'Object {0} not deleted from container {1}'.format(object_name,container_name))
+
+        container.delete()
+        deleted_container = self.provider.object_store.get(container.id)
+        self.assertTrue(deleted_container is None, 'Container {0} not deleted'.format(container_name))