Merged azuedev branch to azure_dev_v2 branch

azure_integration_test/helpers.py

@@ -0,0 +1,207 @@
+import functools
+import os
+import sys
+import unittest
+
+from contextlib import contextmanager
+
+from cloudbridge.cloud.factory import CloudProviderFactory
+from cloudbridge.cloud.interfaces import InstanceState
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
+
+from six import reraise
+
+
+def parse_bool(val):
+    if val:
+        return str(val).upper() in ['TRUE', 'YES']
+    else:
+        return False
+
+
+@contextmanager
+def cleanup_action(cleanup_func):
+    """
+    Context manager to carry out a given
+    cleanup action after carrying out a set
+    of tasks, or when an exception occurs.
+    If any errors occur during the cleanup
+    action, those are ignored, and the original
+    traceback is preserved.
+
+    :params func: This function is called if
+    an exception occurs or at the end of the
+    context block. If any exceptions raised
+        by func are ignored.
+    Usage:
+        with cleanup_action(lambda e: print("Oops!")):
+            do_something()
+    """
+    try:
+        yield
+    except Exception:
+        ex_class, ex_val, ex_traceback = sys.exc_info()
+        try:
+            cleanup_func()
+        except Exception as e:
+            print("Error during exception cleanup: {0}".format(e))
+        reraise(ex_class, ex_val, ex_traceback)
+    try:
+        cleanup_func()
+    except Exception as e:
+        print("Error during cleanup: {0}".format(e))
+
+
+def skipIfNoService(services):
+    """
+    A decorator for skipping tests if the provider
+    does not implement a given service.
+    """
+
+    def wrap(func):
+        """
+        The actual wrapper
+        """
+
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            provider = getattr(self, 'provider')
+            if provider:
+                for service in services:
+                    if not provider.has_service(service):
+                        self.skipTest("Skipping test because '%s' service is"
+                                      " not implemented" % (service,))
+            func(self, *args, **kwargs)
+
+        return wrapper
+
+    return wrap
+
+
+TEST_DATA_CONFIG = {
+    "AWSCloudProvider": {
+        "image": os.environ.get('CB_IMAGE_AWS', 'ami-5ac2cd4d'),
+        "instance_type": os.environ.get('CB_INSTANCE_TYPE_AWS', 't2.nano'),
+        "placement": os.environ.get('CB_PLACEMENT_AWS', 'us-east-1a'),
+    },
+    "OpenStackCloudProvider": {
+        "image": os.environ.get('CB_IMAGE_OS',
+                                '842b949c-ea76-48df-998d-8a41f2626243'),
+        "instance_type": os.environ.get('CB_INSTANCE_TYPE_OS', 'm1.tiny'),
+        "placement": os.environ.get('CB_PLACEMENT_OS', 'nova'),
+    }
+}
+
+
+def get_provider_test_data(provider, key):
+    if "AWSCloudProvider" in provider.name:
+        return TEST_DATA_CONFIG.get("AWSCloudProvider").get(key)
+    elif "OpenStackCloudProvider" in provider.name:
+        return TEST_DATA_CONFIG.get("OpenStackCloudProvider").get(key)
+    return None
+
+
+def create_test_network(provider, name):
+    """
+    Create a network with one subnet, returning the network and subnet objects.
+    """
+    net = provider.network.create(name=name)
+    cidr_block = (net.cidr_block).split('/')[0] or '10.0.0.1'
+    sn = net.create_subnet(cidr_block='{0}/28'.format(cidr_block), name=name,
+                           zone=get_provider_test_data(provider, 'placement'))
+    return net, sn
+
+
+def delete_test_network(network):
+    """
+    Delete the supplied network, first deleting any contained subnets.
+    """
+    with cleanup_action(lambda: network.delete()):
+        for sn in network.subnets():
+            sn.delete()
+
+
+def create_test_instance(
+        provider, instance_name, subnet, zone=None, launch_config=None,
+        key_pair=None, security_groups=None):
+    return provider.compute.instances.create(
+        instance_name,
+        get_provider_test_data(provider, 'image'),
+        get_provider_test_data(provider, 'instance_type'),
+        subnet=subnet,
+        zone=zone,
+        key_pair=key_pair,
+        security_groups=security_groups,
+        launch_config=launch_config)
+
+
+def get_test_instance(provider, name, key_pair=None, security_groups=None,
+                      subnet=None):
+    launch_config = None
+    instance = create_test_instance(
+        provider,
+        name,
+        subnet=subnet,
+        key_pair=key_pair,
+        security_groups=security_groups,
+        launch_config=launch_config)
+    instance.wait_till_ready()
+    return instance
+
+
+def get_test_fixtures_folder():
+    return os.path.join(os.path.dirname(__file__), 'fixtures/')
+
+
+def delete_test_instance(instance):
+    if instance:
+        instance.terminate()
+        instance.wait_for([InstanceState.TERMINATED, InstanceState.UNKNOWN],
+                          terminal_states=[InstanceState.ERROR])
+
+
+def cleanup_test_resources(instance=None, network=None, security_group=None,
+                           key_pair=None):
+    """Clean up any combination of supplied resources."""
+    with cleanup_action(lambda: delete_test_network(network)
+    if network else None):
+        with cleanup_action(lambda: key_pair.delete() if key_pair else None):
+            with cleanup_action(lambda: security_group.delete()
+            if security_group else None):
+                delete_test_instance(instance)
+
+
+class ProviderTestBase(unittest.TestCase):
+    _provider = None
+
+    def setUp(self):
+        if isinstance(self.provider, TestMockHelperMixin):
+            self.provider.setUpMock()
+
+    def tearDown(self):
+        if isinstance(self.provider, TestMockHelperMixin):
+            self.provider.tearDownMock()
+        self._provider = None
+
+    def get_provider_wait_interval(self, provider_class):
+        if issubclass(provider_class, TestMockHelperMixin):
+            return 0
+        else:
+            return 1
+
+    def create_provider_instance(self):
+        provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
+        use_mock_drivers = parse_bool(
+            os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
+        factory = CloudProviderFactory()
+        provider_class = factory.get_provider_class(provider_name,
+                                                    get_mock=use_mock_drivers)
+        config = {'default_wait_interval':
+                      self.get_provider_wait_interval(provider_class)}
+        return provider_class(config)
+
+    @property
+    def provider(self):
+        if not self._provider:
+            self._provider = self.create_provider_instance()
+        return self._provider

cloudbridge/cloud/providers/azure/provider.py

@@ -110,4 +110,3 @@ class MockAzureCloudProvider(AzureCloudProvider, TestMockHelperMixin):
 
     def tearDownMock(self):
         pass
-

codecov.yml

@@ -1,5 +1,5 @@
 codecov:
-  branch: azuedev
+  branch: azure_dev_v2
   notify:
     require_ci_to_pass: yes
 

integration_test/test_integration_azure_object_store_service.py

@@ -2,15 +2,10 @@ import os
 import tempfile
 import uuid
 
-import integration_test.helpers as helpers
+import azure_integration_test.helpers as helpers
 
 
 class AzureIntegrationObjectStoreServiceTestCase(helpers.ProviderTestBase):
-
-    def __init__(self, methodName, provider):
-        super(AzureIntegrationObjectStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['object_store'])
     def test_azure_bucket_service(self):
         container_name = '{0}'.format(uuid.uuid4())

integration_test/test_integration_azure_security_group.py

@@ -4,15 +4,11 @@ import uuid
 
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+from azure_integration_test.helpers import ProviderTestBase
+import azure_integration_test.helpers as helpers
 
 
 class AzureIntegrationSecurityServiceTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureIntegrationSecurityServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group(self):
         sg_name = '{0}'.format(uuid.uuid4())
@@ -56,7 +52,7 @@ class AzureIntegrationSecurityServiceTestCase(ProviderTestBase):
         lenBeforeCreateRule = len(cb.rules)
         cb.add_rule('tcp', '1111', '2222', '0.0.0.0/0')
         lenAfterCreateRule = len(cb.rules)
-        self.assertEqual(lenAfterCreateRule, lenBeforeCreateRule+1)
+        self.assertEqual(lenAfterCreateRule, lenBeforeCreateRule + 1)
 
         print(str(cb.rules))
         get_rule = cb.get_rule('tcp', '1111', '2222', '0.0.0.0/0')
@@ -84,7 +80,7 @@ class AzureIntegrationSecurityServiceTestCase(ProviderTestBase):
         listAfterDeleteFound = self.provider.security.security_groups.list()
         print("Length before delete - " + str(len(listBeforeDeleteFound)))
         print("Length after delete - " + str(len(listAfterDeleteFound)))
-        self.assertEqual(len(listAfterDeleteFound), len(listBeforeDeleteFound)-1)
+        self.assertEqual(len(listAfterDeleteFound), len(listBeforeDeleteFound) - 1)
         sg_id = "/subscriptions/7904d702-e01c-4826-8519-f5a25c866a96/resourceGroups/cloudbridge-azure/providers'\
         '/Microsoft.Network/networkSecurityGroups/sg5"
         sg_del_notfound = self.provider.security.security_groups.delete(sg_id)

integration_test/test_integration_azure_volume_service.py

@@ -2,15 +2,10 @@ import os
 import tempfile
 import uuid
 
-import integration_test.helpers as helpers
+import azure_integration_test.helpers as helpers
 
 
 class AzureIntegrationVolumeServiceTestCase(helpers.ProviderTestBase):
-
-    def __init__(self, methodName, provider):
-        super(AzureIntegrationVolumeServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['block_store'])
     def test_azure_volume_service(self):
         volume_name = '{0}'.format(uuid.uuid4())

setup.py

@@ -25,11 +25,11 @@ REQS_BASE = [
 ]
 REQS_AWS = ['boto3']
 REQS_AZURE = ['azure-common==1.1.5',
-'azure-mgmt-resource==1.0.0rc1',
-'azure-mgmt-compute==1.0.0rc1',
-'azure-mgmt-network==1.0.0rc1',
-'azure-mgmt-storage==1.0.0rc1',
-'azure-storage==0.34.0']
+              'azure-mgmt-resource==1.0.0rc1',
+              'azure-mgmt-compute==1.0.0rc1',
+              'azure-mgmt-network==1.0.0rc1',
+              'azure-mgmt-storage==1.0.0rc1',
+              'azure-storage==0.34.0']
 REQS_OPENSTACK = [
     'openstacksdk',
     'python-novaclient>=7.0.0',

test/test_azure_helpers.py

@@ -1,12 +1,8 @@
-from test.helpers import ProviderTestBase
+from azure_test.helpers import ProviderTestBase
 from cloudbridge.cloud.providers.azure import helpers as azure_helpers
 
 
 class AzureHelpersTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureHelpersTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def test_parse_url_valid(self):
         params = azure_helpers.parse_url('/subscriptionId/{subscriptionId}', '/subscriptionId/123-1345')
         self.assertTrue(len(params) == 1, 'Parameter count should be 1')

test/test_azure_object_store_service.py

@@ -1,12 +1,8 @@
-import test.helpers as helpers
-from test.helpers import ProviderTestBase
+import azure_test.helpers as helpers
+from azure_test.helpers import ProviderTestBase
 
 
 class AzureObjectStoreServiceTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureObjectStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['object_store'])
     def test_azure_bucket_create(self):
         container = self.provider.object_store.create("container3")
@@ -165,4 +161,3 @@ class AzureObjectStoreServiceTestCase(ProviderTestBase):
         print(str(url))
         self.assertEqual(
             str(url), 'https://cloudbridgeazure.blob.core.windows.net/vhds/block1')
-

test/test_azure_provider.py

@@ -1,11 +1,7 @@
-from test.helpers import ProviderTestBase
+from azure_test.helpers import ProviderTestBase
 
 
 class AzureProviderTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureProviderTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def test_azure_provider(self):
         with self.assertRaises(Exception):
             compute = self.provider.compute
@@ -26,4 +22,3 @@ class AzureProviderTestCase(ProviderTestBase):
 
         security = self.provider.security
         self.assertTrue(security is not None, 'Security should not be None')
-

test/test_azure_resource_group.py

@@ -1,18 +1,7 @@
-import json
-import unittest
-import uuid
-
-from cloudbridge.cloud.interfaces import TestMockHelperMixin
-
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+from azure_test.helpers import ProviderTestBase
 
 
 class AzureResourceGroupTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureResourceGroupTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def test_resource_group_create(self):
         resource_group_params = {'location': self.provider.region_name}
         rg = self.provider._azure_client.create_resource_group(self.provider.resource_group, resource_group_params)

test/test_azure_security_service.py

@@ -1,20 +1,17 @@
-import test.helpers as helpers
-from test.helpers import ProviderTestBase
+import azure_test.helpers as helpers
+from azure_test.helpers import ProviderTestBase
 
 
 class AzureSecurityServiceTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureSecurityServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+    @helpers.skipIfNoService(['security.security_groups'])
+    def test_azure_security_key_paires(self):
         with self.assertRaises(NotImplementedError):
             self.key_pairs = self.provider.security.key_pairs
 
-        self.security_groups = self.provider.security.security_groups
-
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_create(self):
         name = "testCreateSecGroup3"
-        sg = self.security_groups.create(
+        sg = self.provider.security.security_groups.create(
             name=name, description=name, network_id="")
         print("Create( " + "Name-" + sg.name + "  Id-" + sg.id + " Rules - " + str(sg.rules) + " )")
         self.assertEqual(name, sg.name)
@@ -33,7 +30,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_list(self):
-        sgl = self.security_groups.list()
+        sgl = self.provider.security.security_groups.list()
         found_sg = [g.name for g in sgl]
         for group in sgl:
             print("List( " + "Name-" + group.name + "  Id-" + group.id + " Rules - " + " )")
@@ -43,7 +40,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_get_found(self):
-        sgl = self.security_groups.get(
+        sgl = self.provider.security.security_groups.get(
             "/subscriptions/7904d702-e01c-4826-8519-f5a25c866a96/resourceGroups/CloudBridge-Azure'\
             '/providers/Microsoft.Network/networkSecurityGroups/sg3")
         print("Get ( " + "Name - " + sgl.name + "  Id - " + sgl.id + " )")
@@ -53,7 +50,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_get_not_found(self):
-        sgl = self.security_groups.get(
+        sgl = self.provider.security.security_groups.get(
             "/subscriptions/7904d702-e01c-4826-8519-f5a25c866a96/resourceGroups/CloudBridge-Azure'\
             '/providers/Microsoft.Network/networkSecurityGroups/sg4")
         print(str(sgl))
@@ -63,7 +60,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_delete_IdExists(self):
-        sg = self.security_groups.delete(
+        sg = self.provider.security.security_groups.delete(
             "/subscriptions/7904d702-e01c-4826-8519-f5a25c866a96/resourceGroups/CloudBridge-Azure'\
             '/providers/Microsoft.Network/networkSecurityGroups/sg2")
         print("Delete - ")
@@ -71,14 +68,14 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_delete_IdNotExist(self):
-        sg = self.security_groups.delete(
+        sg = self.provider.security.security_groups.delete(
             "/subscriptions/7904d702-e01c-4826-8519-f5a25c866a96/resourceGroups/CloudBridge-Azure'\
             '/providers/Microsoft.Network/networkSecurityGroups/sg5")
         self.assertEqual(sg, False)
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_rule_create(self):
-        list = self.security_groups.list()
+        list = self.provider.security.security_groups.list()
         cb = list.data[0]
         rules = cb.rules
         for rule in rules:
@@ -91,7 +88,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_rule_delete(self):
-        list = self.security_groups.list()
+        list = self.provider.security.security_groups.list()
         cb = list.data[0]
         rules = cb.rules
         print("Before deleting Rule -  " + str(rules[0]) + " length - " + str(len(rules)))
@@ -102,7 +99,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_rule_get_exist(self):
-        list = self.security_groups.list()
+        list = self.provider.security.security_groups.list()
         cb = list.data[0]
         rule = cb.get_rule('*', '25', '1', '100')
         print("Get Rule -  " + str(rule))
@@ -110,7 +107,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_rule_get_notExist(self):
-        list = self.security_groups.list()
+        list = self.provider.security.security_groups.list()
         cb = list.data[0]
         rule = cb.get_rule('*', '25', '1', '1')
         print("Get Rule -  " + str(rule))
@@ -118,7 +115,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_to_json(self):
-        list = self.security_groups.list()
+        list = self.provider.security.security_groups.list()
         cb = list.data[0]
         rule = cb.to_json()
         print("Get Rule -  " + str(rule))
@@ -126,7 +123,7 @@ class AzureSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_azure_security_group_rule_to_json(self):
-        list = self.security_groups.list()
+        list = self.provider.security.security_groups.list()
         cb = list.data[0]
         rules = cb.rules
         rule = rules[0]

test/test_azure_volume_service.py

@@ -1,12 +1,8 @@
-import test.helpers as helpers
-from test.helpers import ProviderTestBase
+import azure_test.helpers as helpers
+from azure_test.helpers import ProviderTestBase
 
 
 class AzureVolumeServiceTestCase(ProviderTestBase):
-    def __init__(self, methodName, provider):
-        super(AzureVolumeServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     @helpers.skipIfNoService(['block_store.volumes'])
     def test_azure_volume_create_and_get(self):
         volume = self.provider.block_store.volumes.create("MyVolume", 1, description='My volume')

tox.ini

@@ -12,17 +12,19 @@
 # mock providers.
 
 [tox]
-envlist = {py27,py36,pypy}-{aws,openstack}
+envlist = {py27,py36,pypy}-{aws,azure,openstack}
 
 [testenv]
 commands = flake8 cloudbridge test setup.py 
     {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test {posargs}
 setenv =
     aws: CB_TEST_PROVIDER=aws
+    azure: CB_TEST_PROVIDER=azure
     openstack: CB_TEST_PROVIDER=openstack
 passenv =
     CB_USE_MOCK_PROVIDERS
     aws: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY
+    azure: AZURE_SUBSCRIPTION_ID AZURE_CLIENT_ID AZURE_SECRET AZURE_TENANT
     openstack:  CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS OS_AUTH_URL OS_PASSWORD OS_PROJECT_NAME OS_TENANT_NAME OS_USERNAME OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME NOVA_SERVICE_NAME
 deps =
     -rrequirements.txt