Simplified test execution by using tox features

test/__init__.py

@@ -1,65 +1,9 @@
 """
-Tests the functionality of each provider implementation against registered
-test cases. These tests require that provider credentials are set as
-environment variables, as required for each provider (see `tox.ini` for a list
-of env variables).
-
-Since the tests exercise the ``cloudbridge`` interfaces, and there are multiple
-implementations of these interfaces, for m interfaces and n implementation,
-exercising all interfaces means that m*n test case classes are needed.
-Otherwise, the standard test runners such as unittest and nose2 do not
-correctly pick up the tests.
-
-To avoid an explosion of repetitive test cases, the
-``ProviderTestCaseGenerator`` class will automatically generate a new Python
-class for each combination of test and provider. The ``load_tests`` protocol
-(https://docs.python.org/2/library/unittest.html#load-tests-protocol)
-is used to aid test discovery.
-
 Use ``python setup.py test`` to run these unit tests (alternatively, use
 ``python -m unittest test``).
 
-All test cases need to be registered below, and available providers will be
-discovered through the ``ProviderFactory``. Test Cases must not inherit from
-``unittest.TestCase``, to avoid confusing unittest and nose2's automatic
-discovery. (The test generator will automatically add ``unittest.TestCase``
-as a base class to each combination).
+You must set the CB_TEST_PROVIDER environment variable before running the
+tests. Otherwise, the test suite will default to running against the mock
+aws provider. Alternatively, use tox, which will run tests for all available
+provider combinations.
 """
-import cloudbridge
-
-from test.helpers import ProviderTestCaseGenerator
-from test.test_block_store_service import CloudBlockStoreServiceTestCase
-from test.test_cloud_helpers import CloudHelpersTestCase
-from test.test_compute_service import CloudComputeServiceTestCase
-from test.test_image_service import CloudImageServiceTestCase
-from test.test_instance_types_service import CloudInstanceTypesServiceTestCase
-from test.test_interface import CloudInterfaceTestCase
-from test.test_network_service import CloudNetworkServiceTestCase
-from test.test_object_life_cycle import CloudObjectLifeCycleTestCase
-from test.test_object_store_service import CloudObjectStoreServiceTestCase
-from test.test_region_service import CloudRegionServiceTestCase
-from test.test_security_service import CloudSecurityServiceTestCase
-
-
-PROVIDER_TESTS = [
-    CloudHelpersTestCase,
-    CloudInterfaceTestCase,
-    CloudObjectLifeCycleTestCase,
-    CloudSecurityServiceTestCase,
-    CloudNetworkServiceTestCase,
-    CloudInstanceTypesServiceTestCase,
-    CloudBlockStoreServiceTestCase,
-    CloudObjectStoreServiceTestCase,
-    CloudComputeServiceTestCase,
-    CloudRegionServiceTestCase,
-    CloudImageServiceTestCase
-]
-
-
-def load_tests(loader=None, tests=None, pattern=None):
-    """
-    This function is required to aid the load_tests protocol
-    (https://docs.python.org/2/library/unittest.html#load-tests-protocol)
-    """
-    cloudbridge.init_logging()
-    return ProviderTestCaseGenerator(PROVIDER_TESTS).generate_tests()

test/helpers.py

@@ -82,7 +82,7 @@ TEST_DATA_CONFIG = {
         "image": os.environ.get('CB_IMAGE_OS',
                                 '842b949c-ea76-48df-998d-8a41f2626243'),
         "instance_type": os.environ.get('CB_INSTANCE_TYPE_OS', 'm1.tiny'),
-        "placement": os.environ.get('CB_PLACEMENT_OS', 'zone-r6'),
+        "placement": os.environ.get('CB_PLACEMENT_OS', 'nova'),
     }
 }
 
@@ -156,18 +156,9 @@ def cleanup_test_resources(instance=None, network=None, security_group=None,
                     terminal_states=[InstanceState.ERROR])
 
 
-class ProviderTestBase(object):
+class ProviderTestBase(unittest.TestCase):
 
-    """
-    A dummy base class for Test Cases. Does not inherit from unittest.TestCase
-    to avoid confusing test discovery by unittest and nose2. unittest.TestCase
-    is injected as a base class by the generator, so calling the unittest
-    constructor works correctly.
-    """
-
-    def __init__(self, methodName, provider):
-        unittest.TestCase.__init__(self, methodName=methodName)
-        self.provider = provider
+    _provider = None
 
     def setUp(self):
         if isinstance(self.provider, TestMockHelperMixin):
@@ -176,17 +167,7 @@ class ProviderTestBase(object):
     def tearDown(self):
         if isinstance(self.provider, TestMockHelperMixin):
             self.provider.tearDownMock()
-
-
-class ProviderTestCaseGenerator():
-
-    """
-    Generates test cases for all provider - testcase combinations.
-    Detailed docs at test/__init__.py
-    """
-
-    def __init__(self, test_classes):
-        self.all_test_classes = test_classes
+        self._provider = None
 
     def get_provider_wait_interval(self, provider_class):
         if issubclass(provider_class, TestMockHelperMixin):
@@ -194,80 +175,19 @@ class ProviderTestCaseGenerator():
         else:
             return 1
 
-    def create_provider_instance(self, provider_class):
-        """
-        Instantiate a default provider instance. All required connection
-        settings are expected to be set as environment variables.
-        """
+    def create_provider_instance(self):
+        provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
+        use_mock_drivers = parse_bool(
+            os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
+        factory = CloudProviderFactory()
+        provider_class = factory.get_provider_class(provider_name,
+                                                    get_mock=use_mock_drivers)
         config = {'default_wait_interval':
                   self.get_provider_wait_interval(provider_class)}
         return provider_class(config)
 
-    def generate_new_test_class(self, name, testcase_class):
-        """
-        Generates a new type which inherits from the given testcase_class and
-        unittest.TestCase
-        """
-        class_name = "{0}{1}".format(name, testcase_class.__name__)
-        return type(class_name, (testcase_class, unittest.TestCase), {})
-
-    def generate_test_suite_for_provider_testcase(
-            self, provider_class, testcase_class):
-        """
-        Generate and return a suite of tests for a specific provider class and
-        testcase combination
-        """
-        testloader = unittest.TestLoader()
-        testnames = testloader.getTestCaseNames(testcase_class)
-        suite = unittest.TestSuite()
-        for name in testnames:
-            generated_cls = self.generate_new_test_class(
-                provider_class.__name__,
-                testcase_class)
-            suite.addTest(
-                generated_cls(
-                    name,
-                    self.create_provider_instance(provider_class)))
-        return suite
-
-    def generate_test_suite_for_provider(self, provider_class):
-        """
-        Generate and return a suite of all available tests for a given provider
-        class
-        """
-        suite = unittest.TestSuite()
-        suites = [
-            self.generate_test_suite_for_provider_testcase(
-                provider_class, test_class)
-            for test_class in self.all_test_classes]
-        for s in suites:
-            suite.addTest(s)
-        return suite
-
-    def generate_tests(self):
-        """
-        Generate and return a suite of tests for all provider and test class
-        combinations
-        """
-        factory = CloudProviderFactory()
-        use_mock_drivers = parse_bool(
-            os.environ.get("CB_USE_MOCK_PROVIDERS", True))
-        provider_name = os.environ.get("CB_TEST_PROVIDER", None)
-        if provider_name:
-            provider_classes = [
-                factory.get_provider_class(
-                    provider_name,
-                    get_mock=use_mock_drivers)]
-            if not provider_classes[0]:
-                raise ValueError(
-                    "Could not find specified test provider %s" %
-                    provider_name)
-        else:
-            provider_classes = factory.get_all_provider_classes(
-                get_mock=use_mock_drivers)
-        suite = unittest.TestSuite()
-        suites = [
-            self.generate_test_suite_for_provider(p) for p in provider_classes]
-        for s in suites:
-            suite.addTest(s)
-        return suite
+    @property
+    def provider(self):
+        if not self._provider:
+            self._provider = self.create_provider_instance()
+        return self._provider

test/test_block_store_service.py

@@ -13,10 +13,6 @@ import test.helpers as helpers
 
 class CloudBlockStoreServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudBlockStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['block_store.volumes'])
     def test_crud_volume(self):
         """

test/test_cloud_helpers.py

@@ -17,10 +17,6 @@ class DummyResult(object):
 
 class CloudHelpersTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudHelpersTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def setUp(self):
         super(CloudHelpersTestCase, self).setUp()
         self.objects = [DummyResult(1, "One"),

test/test_compute_service.py

@@ -14,10 +14,6 @@ import test.helpers as helpers
 
 class CloudComputeServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudComputeServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['compute.instances', 'network'])
     def test_crud_instance(self):
         name = "CBInstCrud-{0}-{1}".format(

test/test_image_service.py

@@ -10,10 +10,6 @@ import test.helpers as helpers
 
 class CloudImageServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudImageServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['compute.images', 'network',
                               'compute.instances'])
     def test_create_and_list_image(self):

test/test_instance_types_service.py

@@ -8,10 +8,6 @@ from test.helpers import ProviderTestBase
 
 class CloudInstanceTypesServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudInstanceTypesServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types(self):
         instance_types = self.provider.compute.instance_types.list()

test/test_interface.py

@@ -9,10 +9,6 @@ from cloudbridge.cloud.factory import CloudProviderFactory
 
 class CloudInterfaceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudInterfaceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def test_name_property(self):
         """
         Name should always return a value and should not raise an exception

test/test_network_service.py

@@ -7,10 +7,6 @@ from cloudbridge.cloud.interfaces.resources import RouterState
 
 class CloudNetworkServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudNetworkServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['network'])
     def test_crud_network_service(self):
         name = 'cbtestnetworkservice-{0}'.format(uuid.uuid4())

test/test_object_life_cycle.py

@@ -8,10 +8,6 @@ import test.helpers as helpers
 
 class CloudObjectLifeCycleTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudObjectLifeCycleTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['block_store.volumes'])
     def test_object_life_cycle(self):
         """

test/test_object_store_service.py

@@ -15,10 +15,6 @@ import test.helpers as helpers
 
 class CloudObjectStoreServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudObjectStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['object_store'])
     def test_crud_bucket(self):
         """

test/test_region_service.py

@@ -7,10 +7,6 @@ import test.helpers as helpers
 
 class CloudRegionServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudRegionServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['compute.regions'])
     def test_get_and_list_regions(self):
         """

test/test_security_service.py

@@ -11,10 +11,6 @@ import test.helpers as helpers
 
 class CloudSecurityServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudSecurityServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['security.key_pairs'])
     def test_crud_key_pair_service(self):
         name = 'cbtestkeypairA-{0}'.format(uuid.uuid4())

tox.ini

@@ -1,14 +1,28 @@
 # Tox (http://tox.testrun.org/) is a tool for running tests
 # in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
+# test suite on all supported python versions and providers.
+# To use it, "pip install tox" and then run "tox" from this directory.
+# You will have to set all required environment variables (below) before
+# running the tests.
+#
+# Alternatively, to run mock tests only, run tox as follows:
+# CB_USE_MOCK_PROVIDERS=True tox -e py27-aws
+#
+# Simply running tox -e py27-aws also works, because the default is to use
+# mock providers.
 
 [tox]
-envlist = py27, py36, pypy
+envlist = {py27,py36,pypy}-{aws,openstack}
 
 [testenv]
 commands = {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test
-passenv = AWS_ACCESS_KEY AWS_SECRET_KEY OS_AUTH_URL OS_PASSWORD OS_PROJECT_NAME OS_TENANT_NAME OS_USERNAME OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME NOVA_SERVICE_NAME CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS CB_TEST_PROVIDER CB_USE_MOCK_PROVIDERS
+setenv =
+    aws: CB_TEST_PROVIDER=aws
+    openstack: CB_TEST_PROVIDER=openstack
+passenv =
+    CB_USE_MOCK_PROVIDERS
+    aws: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY
+    openstack:  CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS OS_AUTH_URL OS_PASSWORD OS_PROJECT_NAME OS_TENANT_NAME OS_USERNAME OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME NOVA_SERVICE_NAME
 deps =
     -rrequirements.txt
     coverage