Merge remote-tracking branch 'origin/master' into pr101

.gitignore

@@ -63,3 +63,5 @@ bootstrap.py
 ISB-*
 launch.json
 settings.json
+run_nose.py
+*ipynb*

.travis.yml

@@ -58,12 +58,13 @@ before_install:
            ;;
       esac
 install:
+    - pip install -U pip
     - pip install -U setuptools
     - pip install tox
     - pip install coveralls
     - pip install codecov
 script:
-    - tox -e $TOX_ENV
+    - travis_wait 30 tox -r -e $TOX_ENV
 after_script:
     - |
       case "$TRAVIS_EVENT_TYPE" in
@@ -84,4 +85,3 @@ after_script:
            coveralls & codecov & wait
            ;;
       esac
-

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 gvlproject
+Copyright (c) 2015 CloudVE
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -2,16 +2,16 @@ CloudBridge aims to provide a simple layer of abstraction over
 different cloud providers, reducing or eliminating the need to write
 conditional code for each cloud.
 
-.. image:: https://landscape.io/github/gvlproject/cloudbridge/master/landscape.svg?style=flat
-   :target: https://landscape.io/github/gvlproject/cloudbridge/master
+.. image:: https://landscape.io/github/CloudVE/cloudbridge/master/landscape.svg?style=flat
+   :target: https://landscape.io/github/CloudVE/cloudbridge/master
    :alt: Landscape Code Health
 
-.. image:: https://coveralls.io/repos/gvlproject/cloudbridge/badge.svg?branch=master&service=github
-   :target: https://coveralls.io/github/gvlproject/cloudbridge?branch=master
+.. image:: https://coveralls.io/repos/CloudVE/cloudbridge/badge.svg?branch=master&service=github
+   :target: https://coveralls.io/github/CloudVE/cloudbridge?branch=master
    :alt: Code Coverage
 
-.. image:: https://codeclimate.com/github/gvlproject/cloudbridge/badges/gpa.svg
-   :target: https://codeclimate.com/github/gvlproject/cloudbridge
+.. image:: https://codeclimate.com/github/CloudVE/cloudbridge/badges/gpa.svg
+   :target: https://codeclimate.com/github/CloudVE/cloudbridge
    :alt: Code Climate
 
 .. image:: https://img.shields.io/pypi/v/cloudbridge.svg
@@ -22,37 +22,37 @@ conditional code for each cloud.
    :target: http://cloudbridge.readthedocs.org/en/latest/?badge=latest
    :alt: Documentation Status
 
-.. image:: https://badge.waffle.io/gvlproject/cloudbridge.png?label=in%20progress&title=In%20Progress 
-   :target: https://waffle.io/gvlproject/cloudbridge?utm_source=badge
+.. image:: https://badge.waffle.io/CloudVE/cloudbridge.png?label=in%20progress&title=In%20Progress 
+   :target: https://waffle.io/CloudVE/cloudbridge?utm_source=badge
    :alt: 'Waffle.io - Issues in progress'
 
-.. |aws-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/1
-              :target: https://travis-ci.org/gvlproject/cloudbridge
-.. |aws-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/4
-              :target: https://travis-ci.org/gvlproject/cloudbridge
-.. |aws-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/7
-              :target: https://travis-ci.org/gvlproject/cloudbridge
-
-.. |os-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/3
-             :target: https://travis-ci.org/gvlproject/cloudbridge
-.. |os-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/6
-             :target: https://travis-ci.org/gvlproject/cloudbridge
-.. |os-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/9
-             :target: https://travis-ci.org/gvlproject/cloudbridge
-
-.. |azure-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/2
-                :target: https://travis-ci.org/gvlproject/cloudbridge/branches
-.. |azure-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/5
-                :target: https://travis-ci.org/gvlproject/cloudbridge/branches
-.. |azure-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/master/8
-                :target: https://travis-ci.org/gvlproject/cloudbridge/branches
-
-.. |gce-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/gce/3
-              :target: https://travis-ci.org/gvlproject/cloudbridge/branches
-.. |gce-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/gce/6
-              :target: https://travis-ci.org/gvlproject/cloudbridge/branches
-.. |gce-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/gvlproject/cloudbridge/branches/gce/9
-              :target: https://travis-ci.org/gvlproject/cloudbridge/branches
+.. |aws-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/1
+              :target: https://travis-ci.org/CloudVE/cloudbridge
+.. |aws-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/4
+              :target: https://travis-ci.org/CloudVE/cloudbridge
+.. |aws-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/7
+              :target: https://travis-ci.org/CloudVE/cloudbridge
+
+.. |os-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/3
+             :target: https://travis-ci.org/CloudVE/cloudbridge
+.. |os-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/6
+             :target: https://travis-ci.org/CloudVE/cloudbridge
+.. |os-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/9
+             :target: https://travis-ci.org/CloudVE/cloudbridge
+
+.. |azure-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/2
+                :target: https://travis-ci.org/CloudVE/cloudbridge/branches
+.. |azure-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/5
+                :target: https://travis-ci.org/CloudVE/cloudbridge/branches
+.. |azure-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/master/8
+                :target: https://travis-ci.org/CloudVE/cloudbridge/branches
+
+.. |gce-py27| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/gce/3
+              :target: https://travis-ci.org/CloudVE/cloudbridge/branches
+.. |gce-py36| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/gce/6
+              :target: https://travis-ci.org/CloudVE/cloudbridge/branches
+.. |gce-pypy| image:: https://travis-matrix-badges.herokuapp.com/repos/CloudVE/cloudbridge/branches/gce/9
+              :target: https://travis-ci.org/CloudVE/cloudbridge/branches
 
 
 Build Status
@@ -154,7 +154,7 @@ Community contributions for any part of the project are welcome. If you have
 a completely new idea or would like to bounce your idea before moving forward
 with the implementation, feel free to create an issue to start a discussion.
 
-Contributions should come in the form or a pull request. We strive for 100% test
+Contributions should come in the form of a pull request. We strive for 100% test
 coverage so code will only be accepted if it comes with appropriate tests and it
 does not break existing functionality. Further, the code needs to be well
 documented and all methods have docstrings. We are largely adhering to the

cloudbridge/cloud/base/resources.py

@@ -62,7 +62,7 @@ class BaseCloudResource(CloudResource):
 
     # Regular expression for valid cloudbridge resource names.
     # They, must match the same criteria as GCE labels.
-    # as discussed here: https://github.com/gvlproject/cloudbridge/issues/55
+    # as discussed here: https://github.com/CloudVE/cloudbridge/issues/55
     #
     # NOTE: The following regex is based on GCEs internal validation logic,
     # and is significantly complex to allow for international characters.
@@ -202,7 +202,7 @@ class BaseCloudResource(CloudResource):
     @staticmethod
     def assert_valid_resource_name(name):
         if not BaseCloudResource.is_valid_resource_name(name):
-            log.debug("InvalidNameException raised on %s", name, exc_info=True)
+            log.debug("InvalidNameException raised on %s", name)
             raise InvalidNameException(
                 u"Invalid name: %s. Name must be at most 63 characters "
                 "long and consist of lowercase letters, numbers, "

cloudbridge/cloud/interfaces/resources.py

@@ -188,7 +188,7 @@ class ObjectLifeCycleMixin(object):
     @abstractmethod
     def refresh(self):
         """
-        Refreshs this object's state and synchronize it with the underlying
+        Refresh this object's state and synchronize it with the underlying
         service provider.
         """
         pass
@@ -2187,7 +2187,7 @@ class BucketObject(CloudResource):
         pass
 
     @abstractmethod
-    def generate_url(self, expires_in=0):
+    def generate_url(self, expires_in):
         """
         Generate a URL to this object.
 
@@ -2203,6 +2203,14 @@ class BucketObject(CloudResource):
         """
         pass
 
+    @abstractmethod
+    def refresh(self):
+        """
+        Refresh this object's state and synchronize it with the underlying
+        service provider.
+        """
+        pass
+
 
 class Bucket(CloudResource):
 

cloudbridge/cloud/interfaces/services.py

@@ -55,7 +55,7 @@ class ComputeService(CloudService):
                 print(image.id, image.name)
 
             # find image by name
-            image = provider.compute.images.find(name='Ubuntu 14.04')
+            image = provider.compute.images.find(name='Ubuntu 16.04')
             print(image.id, image.name)
 
         :rtype: :class:`.ImageService`
@@ -95,7 +95,7 @@ class ComputeService(CloudService):
         .. code-block:: python
 
             # launch a new instance
-            image = provider.compute.images.find(name='Ubuntu 14.04')[0]
+            image = provider.compute.images.find(name='Ubuntu 16.04')[0]
             size = provider.compute.vm_types.find(name='m1.small')
             instance = provider.compute.instances.create('Hello', image, size)
             print(instance.id, instance.name)

cloudbridge/cloud/providers/aws/provider.py

@@ -5,7 +5,7 @@ import os
 import boto3
 try:
     # These are installed only for the case of a dev instance
-    from moto.packages.responses import responses
+    import responses
     from moto import mock_ec2
     from moto import mock_s3
 except ImportError:

cloudbridge/cloud/providers/aws/resources.py

@@ -78,8 +78,8 @@ class AWSMachineImage(BaseMachineImage):
     def name(self):
         try:
             return self._ec2_image.name
-        except AttributeError:
-            return None
+        except (AttributeError, ClientError) as e:
+            log.warn("Cannot get name for image {0}: {1}".format(self.id, e))
 
     @property
     def description(self):
@@ -387,7 +387,10 @@ class AWSVolume(BaseVolume):
     @property
     # pylint:disable=arguments-differ
     def name(self):
-        return find_tag_value(self._volume.tags, 'Name')
+        try:
+            return find_tag_value(self._volume.tags, 'Name')
+        except ClientError as e:
+            log.warn("Cannot get name for volume {0}: {1}".format(self.id, e))
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -497,7 +500,10 @@ class AWSSnapshot(BaseSnapshot):
     @property
     # pylint:disable=arguments-differ
     def name(self):
-        return find_tag_value(self._snapshot.tags, 'Name')
+        try:
+            return find_tag_value(self._snapshot.tags, 'Name')
+        except ClientError as e:
+            log.warn("Cannot get name for snap {0}: {1}".format(self.id, e))
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -764,7 +770,10 @@ class AWSBucketObject(BaseBucketObject):
 
     @property
     def size(self):
-        return self._obj.content_length
+        try:
+            return self._obj.content_length
+        except AttributeError:  # we're dealing with s3.ObjectSummary
+            return self._obj.size
 
     @property
     def last_modified(self):
@@ -782,12 +791,15 @@ class AWSBucketObject(BaseBucketObject):
     def delete(self):
         self._obj.delete()
 
-    def generate_url(self, expires_in=0):
+    def generate_url(self, expires_in):
         return self._provider.s3_conn.meta.client.generate_presigned_url(
             'get_object',
             Params={'Bucket': self._obj.bucket_name, 'Key': self.id},
             ExpiresIn=expires_in)
 
+    def refresh(self):
+        self._obj.load()
+
 
 class AWSBucket(BaseBucket):
 
@@ -834,7 +846,7 @@ class AWSBucketContainer(BaseBucketContainer):
         else:
             # pylint:disable=protected-access
             boto_objs = self.bucket._bucket.objects.all()
-        objects = [self.get(obj.key) for obj in boto_objs]
+        objects = [AWSBucketObject(self._provider, obj) for obj in boto_objs]
         return ClientPagedResultList(self._provider, objects,
                                      limit=limit, marker=marker)
 
@@ -1223,9 +1235,12 @@ class AWSInternetGateway(BaseInternetGateway):
         return None
 
     def delete(self):
-        if self.network_id:
-            self._gateway.detach_from_vpc(VpcId=self.network_id)
-        self._gateway.delete()
+        try:
+            if self.network_id:
+                self._gateway.detach_from_vpc(VpcId=self.network_id)
+            self._gateway.delete()
+        except ClientError as e:
+            log.warn("Error deleting gateway {0}: {1}".format(self.id, e))
 
     @property
     def floating_ips(self):

cloudbridge/cloud/providers/azure/azure_client.py

@@ -3,56 +3,74 @@ import logging
 from io import BytesIO
 
 from azure.common.credentials import ServicePrincipalCredentials
+from azure.cosmosdb.table.tableservice import TableService
 from azure.mgmt.compute import ComputeManagementClient
+from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.network import NetworkManagementClient
 from azure.mgmt.resource import ResourceManagementClient
 from azure.mgmt.resource.subscriptions import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
 from azure.storage.blob import BlobPermissions
 from azure.storage.blob import BlockBlobService
-from azure.storage.table import TableService
+
+from cloudbridge.cloud.interfaces.exceptions import WaitStateException
+
+from msrestazure.azure_exceptions import CloudError
+
+import tenacity
 
 from . import helpers as azure_helpers
 
 log = logging.getLogger(__name__)
 
-IMAGE_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups/' \
-                    '{resourceGroupName}/providers/Microsoft.Compute/' \
-                    'images/{imageName}'
-NETWORK_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups/' \
-                     '{resourceGroupName}/providers/Microsoft.Network' \
-                     '/virtualNetworks/{virtualNetworkName}'
-NETWORK_INTERFACE_RESOURCE_ID = '/subscriptions/{subscriptionId}/' \
-                                'resourceGroups/{resourceGroupName}' \
-                                '/providers/Microsoft.Network/' \
-                                'networkInterfaces/{networkInterfaceName}'
-PUBLIC_IP_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups' \
-                        '/{resourceGroupName}/providers/Microsoft.Network' \
-                        '/publicIPAddresses/{publicIpAddressName}'
-SNAPSHOT_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups/' \
-                       '{resourceGroupName}/providers/Microsoft.Compute/' \
-                       'snapshots/{snapshotName}'
-SUBNET_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups/' \
-                     '{resourceGroupName}/providers/Microsoft.Network' \
-                     '/virtualNetworks/{virtualNetworkName}/subnets' \
-                     '/{subnetName}'
-VM_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups/' \
-                       '{resourceGroupName}/providers/Microsoft.Compute/' \
-                       'virtualMachines/{vmName}'
-VM_FIREWALL_RESOURCE_ID = '/subscriptions/{subscriptionId}/' \
-                             'resourceGroups/{resourceGroupName}/' \
-                             'providers/Microsoft.Network/' \
-                             'networkSecurityGroups/' \
-                             '{networkSecurityGroupName}'
-VM_FIREWALL_RULE_RESOURCE_ID = '/subscriptions/{subscriptionId}/' \
-                             'resourceGroups/{resourceGroupName}/' \
-                             'providers/Microsoft.Network/' \
-                             'networkSecurityGroups/' \
-                             '{networkSecurityGroupName}/' \
-                             'securityRules/{securityRuleName}'
-VOLUME_RESOURCE_ID = '/subscriptions/{subscriptionId}/resourceGroups/' \
-                     '{resourceGroupName}/providers/Microsoft.Compute/' \
-                     'disks/{diskName}'
+IMAGE_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
+                     '{resourceGroupName}/providers/Microsoft.Compute/'
+                     'images/{imageName}',
+                     '{imageName}',
+                     '{publisher}:{offer}:{sku}:{version}']
+NETWORK_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
+                       '{resourceGroupName}/providers/Microsoft.Network'
+                       '/virtualNetworks/{virtualNetworkName}',
+                       '{virtualNetworkName}']
+NETWORK_INTERFACE_RESOURCE_ID = ['/subscriptions/{subscriptionId}/'
+                                 'resourceGroups/{resourceGroupName}'
+                                 '/providers/Microsoft.Network/'
+                                 'networkInterfaces/{networkInterfaceName}',
+                                 '{networkInterfaceName}']
+PUBLIC_IP_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups'
+                         '/{resourceGroupName}/providers/Microsoft.Network'
+                         '/publicIPAddresses/{publicIpAddressName}',
+                         '{publicIpAddressName}']
+SNAPSHOT_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
+                        '{resourceGroupName}/providers/Microsoft.Compute/'
+                        'snapshots/{snapshotName}',
+                        '{snapshotName}']
+SUBNET_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
+                      '{resourceGroupName}/providers/Microsoft.Network'
+                      '/virtualNetworks/{virtualNetworkName}/subnets'
+                      '/{subnetName}',
+                      '{virtualNetworkName}/{subnetName}']
+VM_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
+                  '{resourceGroupName}/providers/Microsoft.Compute/'
+                  'virtualMachines/{vmName}',
+                  '{vmName}']
+VM_FIREWALL_RESOURCE_ID = ['/subscriptions/{subscriptionId}/'
+                           'resourceGroups/{resourceGroupName}/'
+                           'providers/Microsoft.Network/'
+                           'networkSecurityGroups/'
+                           '{networkSecurityGroupName}',
+                           '{networkSecurityGroupName}']
+VM_FIREWALL_RULE_RESOURCE_ID = ['/subscriptions/{subscriptionId}/'
+                                'resourceGroups/{resourceGroupName}/'
+                                'providers/Microsoft.Network/'
+                                'networkSecurityGroups/'
+                                '{networkSecurityGroupName}/'
+                                'securityRules/{securityRuleName}',
+                                '{securityRuleName}']
+VOLUME_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
+                      '{resourceGroupName}/providers/Microsoft.Compute/'
+                      'disks/{diskName}',
+                      '{diskName}']
 
 IMAGE_NAME = 'imageName'
 NETWORK_NAME = 'virtualNetworkName'
@@ -65,6 +83,66 @@ VM_FIREWALL_NAME = 'networkSecurityGroupName'
 VM_FIREWALL_RULE_NAME = 'securityRuleName'
 VOLUME_NAME = 'diskName'
 
+# Listing possible somewhat through:
+# azure.mgmt.devtestlabs.operations.GalleryImageOperations
+gallery_image_references = \
+    [GalleryImageReference(publisher='Canonical',
+                           offer='UbuntuServer',
+                           sku='16.04.0-LTS',
+                           version='latest'),
+     GalleryImageReference(publisher='Canonical',
+                           offer='UbuntuServer',
+                           sku='14.04.5-LTS',
+                           version='latest'),
+     GalleryImageReference(publisher='OpenLogic',
+                           offer='CentOS',
+                           sku='7.5',
+                           version='latest'),
+     GalleryImageReference(publisher='OpenLogic',
+                           offer='CentOS',
+                           sku='6.9',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftWindowsServer',
+                           offer='WindowsServer',
+                           sku='2016-Nano-Server',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftWindowsServer',
+                           offer='WindowsServer',
+                           sku='2016-Datacenter',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftWindowsDesktop',
+                           offer='Windows-10',
+                           sku='rs4-pron',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftVisualStudio',
+                           offer='Windows',
+                           sku='Windows-10-N-x64',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftVisualStudio',
+                           offer='VisualStudio',
+                           sku='VS-2017-Ent-WS2016',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftSQLServer',
+                           offer='SQL2017-WS2016',
+                           sku='Web',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftSQLServer',
+                           offer='SQL2017-WS2016',
+                           sku='Standard',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftSQLServer',
+                           offer='SQL2017-WS2016',
+                           sku='SQLDEV',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftSQLServer',
+                           offer='SQL2017-WS2016',
+                           sku='Express',
+                           version='latest'),
+     GalleryImageReference(publisher='MicrosoftSQLServer',
+                           offer='SQL2017-WS2016',
+                           sku='Enterprise',
+                           version='latest')]
+
 
 class AzureClient(object):
     """
@@ -72,7 +150,7 @@ class AzureClient(object):
     """
     def __init__(self, config):
         self._config = config
-        self.subscription_id = config.get('azure_subscription_id')
+        self.subscription_id = str(config.get('azure_subscription_id'))
         self._credentials = ServicePrincipalCredentials(
             client_id=config.get('azure_client_id'),
             secret=config.get('azure_secret'),
@@ -91,10 +169,25 @@ class AzureClient(object):
         log.debug("azure subscription : %s", self.subscription_id)
 
     @property
+    @tenacity.retry(stop=tenacity.stop_after_attempt(5), reraise=True)
     def access_key_result(self):
         if not self._access_key_result:
+            storage_account = self.storage_account
+
+            if self.get_storage_account(storage_account).\
+                    provisioning_state.value != 'Succeeded':
+                log.debug(
+                    "Storage account %s is not in Succeeded state yet. ",
+                    storage_account)
+                raise WaitStateException(
+                    "Waited too long for storage account: {0} to "
+                    "become ready.".format(
+                        storage_account,
+                        self.get_storage_account(storage_account).
+                        provisioning_state))
+
             self._access_key_result = self.storage_client.storage_accounts. \
-                list_keys(self.resource_group, self.storage_account)
+                list_keys(self.resource_group, storage_account)
         return self._access_key_result
 
     @property
@@ -268,7 +361,7 @@ class AzureClient(object):
         self.blob_service.delete_blob(container_name, blob_name)
 
     def get_blob_url(self, container_name, blob_name, expiry_time):
-        expiry_date = datetime.datetime.now() + datetime.timedelta(
+        expiry_date = datetime.datetime.utcnow() + datetime.timedelta(
             seconds=expiry_time)
         sas = self.blob_service.generate_blob_shared_access_signature(
             container_name, blob_name, permission=BlobPermissions.READ,
@@ -359,6 +452,12 @@ class AzureClient(object):
             raw=True
         )
 
+    def is_gallery_image(self, image_id):
+        url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
+                                             image_id)
+        # If it is a gallery image, it will always have an offer
+        return 'offer' in url_params
+
     def create_image(self, name, params):
         return self.compute_client.images. \
             create_or_update(self.resource_group, name,
@@ -367,29 +466,43 @@ class AzureClient(object):
     def delete_image(self, image_id):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
                                              image_id)
-        name = url_params.get(IMAGE_NAME)
-        self.compute_client.images.delete(self.resource_group, name).wait()
+        if not self.is_gallery_image(image_id):
+            name = url_params.get(IMAGE_NAME)
+            self.compute_client.images.delete(self.resource_group, name).wait()
 
     def list_images(self):
-        return self.compute_client.images. \
-            list_by_resource_group(self.resource_group)
+        azure_images = list(self.compute_client.images.
+                            list_by_resource_group(self.resource_group))
+        return azure_images
+
+    def list_gallery_refs(self):
+        return gallery_image_references
 
     def get_image(self, image_id):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
                                              image_id)
-        name = url_params.get(IMAGE_NAME)
-        return self.compute_client.images.get(self.resource_group, name)
+        if self.is_gallery_image(image_id):
+            return GalleryImageReference(publisher=url_params['publisher'],
+                                         offer=url_params['offer'],
+                                         sku=url_params['sku'],
+                                         version=url_params['version'])
+        else:
+            name = url_params.get(IMAGE_NAME)
+            return self.compute_client.images.get(self.resource_group, name)
 
     def update_image_tags(self, image_id, tags):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
                                              image_id)
-        name = url_params.get(IMAGE_NAME)
-        return self.compute_client.images. \
-            create_or_update(self.resource_group, name,
-                             {
-                                 'tags': tags,
-                                 'location': self.region_name
-                             }).result()
+        if self.is_gallery_image(image_id):
+            return True
+        else:
+            name = url_params.get(IMAGE_NAME)
+            return self.compute_client.images. \
+                create_or_update(self.resource_group, name,
+                                 {
+                                     'tags': tags,
+                                     'location': self.region_name
+                                 }).result()
 
     def list_vm_types(self):
         return self.compute_client.virtual_machine_sizes. \
@@ -427,7 +540,7 @@ class AzureClient(object):
 
     def get_network_id_for_subnet(self, subnet_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID, subnet_id)
-        network_id = NETWORK_RESOURCE_ID
+        network_id = NETWORK_RESOURCE_ID[0]
         for key, val in url_params.items():
             network_id = network_id.replace("{" + key + "}", val)
         return network_id
@@ -460,18 +573,35 @@ class AzureClient(object):
 
         return subnet_info
 
+    def __if_subnet_in_use(e):
+        # return True if the CloudError exception is due to subnet being in use
+        if isinstance(e, CloudError):
+            error_message = e.message
+            if "Subnet" in error_message \
+                    and 'in use' in error_message:
+                return True
+        return False
+
+    @tenacity.retry(stop=tenacity.stop_after_attempt(5),
+                    retry=tenacity.retry_if_exception(__if_subnet_in_use),
+                    reraise=True)
     def delete_subnet(self, subnet_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
                                              subnet_id)
         network_name = url_params.get(NETWORK_NAME)
         subnet_name = url_params.get(SUBNET_NAME)
-        result_delete = self.network_management_client \
-            .subnets.delete(
-                self.resource_group,
-                network_name,
-                subnet_name
-            )
-        result_delete.wait()
+
+        try:
+            result_delete = self.network_management_client \
+                .subnets.delete(
+                    self.resource_group,
+                    network_name,
+                    subnet_name
+                )
+            result_delete.wait()
+        except CloudError as cloud_error:
+            log.exception(cloud_error.message)
+            raise cloud_error
 
     def create_floating_ip(self, public_ip_name, public_ip_parameters):
         return self.network_management_client.public_ip_addresses. \

cloudbridge/cloud/providers/azure/helpers.py

@@ -20,22 +20,47 @@ def filter_by_tag(list_items, filters):
         return list_items
 
 
-def parse_url(template_url, original_url):
+def parse_url(template_urls, original_url):
     """
     In Azure all the resource IDs are returned as URIs.
     ex: '/subscriptions/{subscriptionId}/resourceGroups/' \
        '{resourceGroupName}/providers/Microsoft.Compute/' \
        'virtualMachines/{vmName}'
-    This function splits the resource ID based on the template url passed
+    This function splits the resource ID based on the template urls passed
     and returning the dictionary.
+
+    The only exception to that format are image URN's which are used for
+    public gallery references:
+    https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cli-ps-findimage
     """
-    template_url_parts = template_url.split('/')
+    if not original_url:
+        raise InvalidValueException(template_urls, original_url)
     original_url_parts = original_url.split('/')
+    if len(original_url_parts) == 1:
+        original_url_parts = original_url.split(':')
+    for each_template in template_urls:
+        template_url_parts = each_template.split('/')
+        if len(template_url_parts) == 1:
+            template_url_parts = each_template.split(':')
+        if len(template_url_parts) == len(original_url_parts):
+            break
     if len(template_url_parts) != len(original_url_parts):
-        raise InvalidValueException(template_url, original_url)
+        raise InvalidValueException(template_urls, original_url)
     resource_param = {}
     for key, value in zip(template_url_parts, original_url_parts):
         if key.startswith('{') and key.endswith('}'):
             resource_param.update({key[1:-1]: value})
-
     return resource_param
+
+
+def generate_urn(gallery_image):
+    """
+    This function takes an azure gallery image and outputs a corresponding URN
+    :param gallery_image: a GalleryImageReference object
+    :return: URN as string
+    """
+    reference_dict = gallery_image.as_dict()
+    return ':'.join([reference_dict['publisher'],
+                     reference_dict['offer'],
+                     reference_dict['sku'],
+                     reference_dict['version']])

cloudbridge/cloud/providers/azure/provider.py

@@ -9,6 +9,8 @@ from cloudbridge.cloud.providers.azure.services \
 
 from msrestazure.azure_exceptions import CloudError
 
+import tenacity
+
 log = logging.getLogger(__name__)
 
 
@@ -36,12 +38,17 @@ class AzureCloudProvider(BaseCloudProvider):
         self.resource_group = self._get_config_value(
             'azure_resource_group', os.environ.get('AZURE_RESOURCE_GROUP',
                                                    'cloudbridge'))
-        # Storage account name is limited to a max length of 24 characters
-        # so take part of the client id to keep it unique
+        # Storage account name is limited to a max length of 24 alphanum chars
+        # and unique across an account. Yet, all our operations are tied to a
+        # resource group, making it impossible to use a storage account
+        # defined in a different resource group from the one used by the
+        # current session. With that, base the name of the storage account on
+        # the current resource group, up to 24 chars in length.
         self.storage_account = self._get_config_value(
             'azure_storage_account',
-            os.environ.get('AZURE_STORAGE_ACCOUNT',
-                           'storageacc' + self.client_id[-12:]))
+            os.environ.get(
+                'AZURE_STORAGE_ACCOUNT', 'storageacc' + ''.join(
+                    ch for ch in self.resource_group if ch.isalnum())[-12:]))
 
         self.vm_default_user_name = self._get_config_value(
             'azure_vm_default_user_name', os.environ.get
@@ -110,9 +117,14 @@ class AzureCloudProvider(BaseCloudProvider):
             resource_group_params = {'location': self.region_name}
             self._azure_client.create_resource_group(self.resource_group,
                                                      resource_group_params)
+        # Create a storage account. To prevent a race condition, try
+        # to get or create at least twice
+        self._get_or_create_storage_account()
 
+    @tenacity.retry(stop=tenacity.stop_after_attempt(2), reraise=True)
+    def _get_or_create_storage_account(self):
         try:
-            self._azure_client.get_storage_account(self.storage_account)
+            return self._azure_client.get_storage_account(self.storage_account)
         except CloudError:
             storage_account_params = {
                 'sku': {
@@ -121,6 +133,5 @@ class AzureCloudProvider(BaseCloudProvider):
                 'kind': 'storage',
                 'location': self.region_name,
             }
-            self._azure_client. \
-                create_storage_account(self.storage_account,
-                                       storage_account_params)
+            self._azure_client.create_storage_account(self.storage_account,
+                                                      storage_account_params)

cloudbridge/cloud/providers/azure/resources.py

@@ -6,6 +6,7 @@ import logging
 import uuid
 
 from azure.common import AzureException
+from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.network.models import NetworkSecurityGroup
 
 import cloudbridge.cloud.base.helpers as cb_helpers
@@ -25,6 +26,8 @@ from msrestazure.azure_exceptions import CloudError
 
 import pysftp
 
+from . import helpers as azure_helpers
+
 log = logging.getLogger(__name__)
 
 
@@ -85,8 +88,8 @@ class AzureVMFirewall(BaseVMFirewall):
                 get_vm_firewall(self.id)
             if not self._vm_firewall.tags:
                 self._vm_firewall.tags = {}
-        except (CloudError, ValueError) as cloudError:
-            log.exception(cloudError.message)
+        except (CloudError, ValueError) as cloud_error:
+            log.exception(cloud_error.message)
             # The security group no longer exists and cannot be refreshed.
 
     def to_json(self):
@@ -304,13 +307,17 @@ class AzureBucketObject(BaseBucketObject):
         self._provider.azure_client.delete_blob(self._container.name,
                                                 self.name)
 
-    def generate_url(self, expires_in=0):
+    def generate_url(self, expires_in):
         """
         Generate a URL to this object.
         """
         return self._provider.azure_client.get_blob_url(
             self._container.name, self.name, expires_in)
 
+    def refresh(self):
+        self._key = self._provider.azure_client.get_blob(
+            self._container.name, self._key.name)
+
 
 class AzureBucket(BaseBucket):
     def __init__(self, provider, bucket):
@@ -520,7 +527,7 @@ class AzureVolume(BaseVolume):
         for vm in self._provider.azure_client.list_vm():
             for item in vm.storage_profile.data_disks:
                 if item.managed_disk and \
-                                item.managed_disk.id == self.resource_id:
+                        item.managed_disk.id == self.resource_id:
                     vm.storage_profile.data_disks.remove(item)
                     self._provider.azure_client.update_vm(vm.id, vm)
 
@@ -550,8 +557,8 @@ class AzureVolume(BaseVolume):
             self._volume = self._provider.azure_client. \
                 get_disk(self.id)
             self._update_state()
-        except (CloudError, ValueError) as cloudError:
-            log.exception(cloudError.message)
+        except (CloudError, ValueError) as cloud_error:
+            log.exception(cloud_error.message)
             # The volume no longer exists and cannot be refreshed.
             # set the state to unknown
             self._state = 'unknown'
@@ -642,8 +649,8 @@ class AzureSnapshot(BaseSnapshot):
             self._snapshot = self._provider.azure_client. \
                 get_snapshot(self.id)
             self._state = self._snapshot.provisioning_state
-        except (CloudError, ValueError) as cloudError:
-            log.exception(cloudError.message)
+        except (CloudError, ValueError) as cloud_error:
+            log.exception(cloud_error.message)
             # The snapshot no longer exists and cannot be refreshed.
             # set the state to unknown
             self._state = 'unknown'
@@ -673,11 +680,15 @@ class AzureMachineImage(BaseMachineImage):
 
     def __init__(self, provider, image):
         super(AzureMachineImage, self).__init__(provider)
+        # Image can be either a dict for public image reference
+        # or the Azure iamge object
         self._image = image
-        self._state = self._image.provisioning_state
-
-        if not self._image.tags:
-            self._image.tags = {}
+        if isinstance(self._image, GalleryImageReference):
+            self._state = 'Succeeded'
+        else:
+            self._state = self._image.provisioning_state
+            if not self._image.tags:
+                self._image.tags = {}
 
     @property
     def id(self):
@@ -687,11 +698,17 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``str``
         :return: ID for this instance as returned by the cloud middleware.
         """
-        return self._image.id
+        if isinstance(self._image, GalleryImageReference):
+            return azure_helpers.generate_urn(self._image)
+        else:
+            return self._image.id
 
     @property
     def resource_id(self):
-        return self._image.id
+        if isinstance(self._image, GalleryImageReference):
+            return azure_helpers.generate_urn(self._image)
+        else:
+            return self._image.id
 
     @property
     def name(self):
@@ -701,17 +718,21 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``str``
         :return: Name for this image as returned by the cloud middleware.
         """
-        return self._image.tags.get('Name', self._image.name)
+        if isinstance(self._image, GalleryImageReference):
+            return azure_helpers.generate_urn(self._image)
+        else:
+            return self._image.tags.get('Name', self._image.name)
 
     @name.setter
     def name(self, value):
         """
         Set the image name.
         """
-        self.assert_valid_resource_name(value)
-        self._image.tags.update(Name=value)
-        self._provider.azure_client. \
-            update_image_tags(self.id, self._image.tags)
+        if not isinstance(self._image, GalleryImageReference):
+            self.assert_valid_resource_name(value)
+            self._image.tags.update(Name=value)
+            self._provider.azure_client. \
+                update_image_tags(self.id, self._image.tags)
 
     @property
     def description(self):
@@ -721,16 +742,21 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``str``
         :return: Description for this image as returned by the cloud middleware
         """
-        return self._image.tags.get('Description', None)
+        if isinstance(self._image, GalleryImageReference):
+            return 'Public gallery image from the Azure Marketplace: '\
+                    + self.name
+        else:
+            return self._image.tags.get('Description', None)
 
     @description.setter
     def description(self, value):
         """
-        Set the image name.
+        Set the image description.
         """
-        self._image.tags.update(Description=value)
-        self._provider.azure_client. \
-            update_image_tags(self.id, self._image.tags)
+        if not isinstance(self._image, GalleryImageReference):
+            self._image.tags.update(Description=value)
+            self._provider.azure_client. \
+                update_image_tags(self.id, self._image.tags)
 
     @property
     def min_disk(self):
@@ -743,31 +769,47 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``int``
         :return: The minimum disk size needed by this image
         """
-        return self._image.storage_profile.os_disk.disk_size_gb or 0
+        if isinstance(self._image, GalleryImageReference):
+            return 0
+        else:
+            return self._image.storage_profile.os_disk.disk_size_gb or 0
 
     def delete(self):
         """
         Delete this image
         """
-        self._provider.azure_client.delete_image(self.id)
+        if not isinstance(self._image, GalleryImageReference):
+            self._provider.azure_client.delete_image(self.id)
 
     @property
     def state(self):
-        return AzureMachineImage.IMAGE_STATE_MAP.get(
-            self._state, MachineImageState.UNKNOWN)
+        if isinstance(self._image, GalleryImageReference):
+            return MachineImageState.AVAILABLE
+        else:
+            return AzureMachineImage.IMAGE_STATE_MAP.get(
+                self._state, MachineImageState.UNKNOWN)
+
+    @property
+    def is_gallery_image(self):
+        """
+        Returns true if the image is a public reference and false if it
+        is a private image in the resource group.
+        """
+        return isinstance(self._image, GalleryImageReference)
 
     def refresh(self):
         """
         Refreshes the state of this instance by re-querying the cloud provider
         for its latest state.
         """
-        try:
-            self._image = self._provider.azure_client.get_image(self.id)
-            self._state = self._image.provisioning_state
-        except CloudError as cloudError:
-            log.exception(cloudError.message)
-            # image no longer exists
-            self._state = "unknown"
+        if not isinstance(self._image, dict):
+            try:
+                self._image = self._provider.azure_client.get_image(self.id)
+                self._state = self._image.provisioning_state
+            except CloudError as cloud_error:
+                log.exception(cloud_error.message)
+                # image no longer exists
+                self._state = "unknown"
 
 
 class AzureGatewayContainer(BaseGatewayContainer):
@@ -860,8 +902,8 @@ class AzureNetwork(BaseNetwork):
             self._network = self._provider.azure_client.\
                 get_network(self.id)
             self._state = self._network.provisioning_state
-        except (CloudError, ValueError) as cloudError:
-            log.exception(cloudError.message)
+        except (CloudError, ValueError) as cloud_error:
+            log.exception(cloud_error.message)
             # The network no longer exists and cannot be refreshed.
             # set the state to unknown
             self._state = 'unknown'
@@ -1104,8 +1146,8 @@ class AzureSubnet(BaseSubnet):
             self._subnet = self._provider.azure_client. \
                 get_subnet(self.id)
             self._state = self._subnet.provisioning_state
-        except (CloudError, ValueError) as cloudError:
-            log.exception(cloudError.message)
+        except (CloudError, ValueError) as cloud_error:
+            log.exception(cloud_error.message)
             # The subnet no longer exists and cannot be refreshed.
             # set the state to unknown
             self._state = 'unknown'
@@ -1240,11 +1282,8 @@ class AzureInstance(BaseInstance):
             self._provider.azure_client.delete_nic(nic_id)
         for data_disk in self._vm.storage_profile.data_disks:
             if data_disk.managed_disk:
-                disk = self._provider.azure_client.\
-                    get_disk(data_disk.managed_disk.id)
-                if disk and disk.tags \
-                        and disk.tags.get('delete_on_terminate',
-                                          'False') == 'True':
+                if self._vm.tags.get('delete_on_terminate',
+                                     'False') == 'True':
                     self._provider.azure_client.\
                         delete_disk(data_disk.managed_disk.id)
         if self._vm.storage_profile.os_disk.managed_disk:
@@ -1256,7 +1295,12 @@ class AzureInstance(BaseInstance):
         """
         Get the image ID for this instance.
         """
-        return self._vm.storage_profile.image_reference.id
+        # Not tested for resource group images
+        reference_dict = self._vm.storage_profile.image_reference.as_dict()
+        return ':'.join([reference_dict['publisher'],
+                         reference_dict['offer'],
+                         reference_dict['sku'],
+                         reference_dict['version']])
 
     @property
     def zone_id(self):
@@ -1454,8 +1498,8 @@ class AzureInstance(BaseInstance):
             if not self._vm.tags:
                 self._vm.tags = {}
             self._update_state()
-        except (CloudError, ValueError) as cloudError:
-            log.exception(cloudError.message)
+        except (CloudError, ValueError) as cloud_error:
+            log.exception(cloud_error.message)
             # The volume no longer exists and cannot be refreshed.
             # set the state to unknown
             self._state = 'unknown'

cloudbridge/cloud/providers/azure/services.py

@@ -57,9 +57,9 @@ class AzureVMFirewallService(BaseVMFirewallService):
         try:
             fws = self.provider.azure_client.get_vm_firewall(fw_id)
             return AzureVMFirewall(self.provider, fws)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def list(self, limit=None, marker=None):
@@ -78,7 +78,7 @@ class AzureVMFirewallService(BaseVMFirewallService):
         fw = self.provider.azure_client.create_vm_firewall(name, parameters)
 
         # Add default rules to negate azure default rules.
-        # See: https://github.com/gvlproject/cloudbridge/issues/106
+        # See: https://github.com/CloudVE/cloudbridge/issues/106
         # pylint:disable=protected-access
         for rule in fw.default_security_rules:
             rule_name = "cb-override-" + rule.name
@@ -116,7 +116,7 @@ class AzureVMFirewallService(BaseVMFirewallService):
         filters = {'Name': name}
         fws = [AzureVMFirewall(self.provider, vm_firewall)
                for vm_firewall in azure_helpers.filter_by_tag(
-                self.provider.azure_client.list_vm_firewall(), filters)]
+               self.provider.azure_client.list_vm_firewall(), filters)]
         return ClientPagedResultList(self.provider, fws)
 
     def delete(self, group_id):
@@ -143,7 +143,7 @@ class AzureKeyPairService(BaseKeyPairService):
 
     def list(self, limit=None, marker=None):
         key_pairs, resume_marker = self.provider.azure_client.list_public_keys(
-            AzureKeyPairService.PARTITION_KEY,  marker=marker,
+            AzureKeyPairService.PARTITION_KEY, marker=marker,
             limit=limit or self.provider.config.default_result_limit)
         results = [AzureKeyPair(self.provider, key_pair)
                    for key_pair in key_pairs]
@@ -178,11 +178,11 @@ class AzureKeyPairService(BaseKeyPairService):
             public_key_material, private_key = cb_helpers.generate_key_pair()
 
         entity = {
-                  'PartitionKey': AzureKeyPairService.PARTITION_KEY,
-                  'RowKey': str(uuid.uuid4()),
-                  'Name': name,
-                  'Key': public_key_material
-                 }
+            'PartitionKey': AzureKeyPairService.PARTITION_KEY,
+            'RowKey': str(uuid.uuid4()),
+            'Name': name,
+            'Key': public_key_material
+        }
 
         self.provider.azure_client.create_public_key(entity)
         key_pair = self.get(name)
@@ -270,9 +270,9 @@ class AzureVolumeService(BaseVolumeService):
         try:
             volume = self.provider.azure_client.get_disk(volume_id)
             return AzureVolume(self.provider, volume)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def find(self, **kwargs):
@@ -286,7 +286,7 @@ class AzureVolumeService(BaseVolumeService):
         filters = {'Name': name}
         cb_vols = [AzureVolume(self.provider, volume)
                    for volume in azure_helpers.filter_by_tag(
-                self.provider.azure_client.list_disks(), filters)]
+                   self.provider.azure_client.list_disks(), filters)]
         return ClientPagedResultList(self.provider, cb_vols)
 
     def list(self, limit=None, marker=None):
@@ -354,9 +354,9 @@ class AzureSnapshotService(BaseSnapshotService):
         try:
             snapshot = self.provider.azure_client.get_snapshot(ss_id)
             return AzureSnapshot(self.provider, snapshot)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def find(self, **kwargs):
@@ -370,7 +370,7 @@ class AzureSnapshotService(BaseSnapshotService):
         filters = {'Name': name}
         cb_snapshots = [AzureSnapshot(self.provider, snapshot)
                         for snapshot in azure_helpers.filter_by_tag(
-                self.provider.azure_client.list_snapshots(), filters)]
+                        self.provider.azure_client.list_snapshots(), filters)]
         return ClientPagedResultList(self.provider, cb_snapshots)
 
     def list(self, limit=None, marker=None):
@@ -495,15 +495,15 @@ class AzureInstanceService(BaseInstanceService):
                                                        instance_name, zone_id)
 
         nic_params = {
-                'location': self._provider.region_name,
-                'ip_configurations': [{
-                    'name': instance_name + '_ip_config',
-                    'private_ip_allocation_method': 'Dynamic',
-                    'subnet': {
-                        'id': subnet_id
-                    }
-                }]
-            }
+            'location': self._provider.region_name,
+            'ip_configurations': [{
+                'name': instance_name + '_ip_config',
+                'private_ip_allocation_method': 'Dynamic',
+                'subnet': {
+                    'id': subnet_id
+                }
+            }]
+        }
 
         if vm_firewall_id:
             nic_params['network_security_group'] = {
@@ -524,16 +524,16 @@ class AzureInstanceService(BaseInstanceService):
                 'admin_username': self.provider.vm_default_user_name,
                 'computer_name': instance_name,
                 'linux_configuration': {
-                             "disable_password_authentication": True,
-                             "ssh": {
-                                 "public_keys": [{
-                                      "path":
-                                      "/home/{}/.ssh/authorized_keys".format(
-                                          self.provider.vm_default_user_name),
-                                      "key_data": key_pair._key_pair.Key
-                                     }]
-                                   }
-                           }
+                    "disable_password_authentication": True,
+                    "ssh": {
+                        "public_keys": [{
+                            "path":
+                                "/home/{}/.ssh/authorized_keys".format(
+                                        self.provider.vm_default_user_name),
+                                "key_data": key_pair._key_pair.Key
+                        }]
+                    }
+                }
             },
             'hardware_profile': {
                 'vm_size': instance_size
@@ -550,6 +550,9 @@ class AzureInstanceService(BaseInstanceService):
         if key_pair:
             params['tags'].update(Key_Pair=key_pair.name)
 
+        for disk_def in storage_profile.get('data_disks', []):
+            params['tags'] = dict(disk_def.get('tags', {}), **params['tags'])
+
         if user_data:
             custom_data = base64.b64encode(bytes(ud, 'utf-8'))
             params['os_profile']['custom_data'] = str(custom_data, 'utf-8')
@@ -563,7 +566,8 @@ class AzureInstanceService(BaseInstanceService):
                 if disk_def.get('tags', {}).get('delete_on_terminate'):
                     disk_id = disk_def.get('managed_disk', {}).get('id')
                     if disk_id:
-                        self.provider.storage.volumes.delete(disk_id)
+                        vol = self.provider.storage.volumes.get(disk_id)
+                        vol.delete()
             raise e
         finally:
             if temp_key_pair:
@@ -603,10 +607,21 @@ class AzureInstanceService(BaseInstanceService):
     def _create_storage_profile(self, image, launch_config, instance_name,
                                 zone_id):
 
-        storage_profile = {
-            'image_reference': {
+        if image.is_gallery_image:
+            reference = image._image.as_dict()
+            image_ref = {
+                'publisher': reference['publisher'],
+                'offer': reference['offer'],
+                'sku': reference['sku'],
+                'version': reference['version']
+            }
+        else:
+            image_ref = {
                 'id': image.resource_id
-            },
+            }
+
+        storage_profile = {
+            'image_reference': image_ref,
             "os_disk": {
                 "name": instance_name + '_os_disk',
                 "create_option": DiskCreateOption.from_image
@@ -713,9 +728,9 @@ class AzureInstanceService(BaseInstanceService):
         try:
             vm = self.provider.azure_client.get_vm(instance_id)
             return AzureInstance(self.provider, vm)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def find(self, **kwargs):
@@ -729,7 +744,7 @@ class AzureInstanceService(BaseInstanceService):
         filtr = {'Name': name}
         instances = [AzureInstance(self.provider, inst)
                      for inst in azure_helpers.filter_by_tag(
-                self.provider.azure_client.list_vm(), filtr)]
+                     self.provider.azure_client.list_vm(), filtr)]
         return ClientPagedResultList(self.provider, instances)
 
 
@@ -744,9 +759,9 @@ class AzureImageService(BaseImageService):
         try:
             image = self.provider.azure_client.get_image(image_id)
             return AzureMachineImage(self.provider, image)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def find(self, **kwargs):
@@ -760,7 +775,12 @@ class AzureImageService(BaseImageService):
         filters = {'Name': name}
         cb_images = [AzureMachineImage(self.provider, image)
                      for image in azure_helpers.filter_by_tag(
-                self.provider.azure_client.list_images(), filters)]
+                     self.provider.azure_client.list_images(), filters)]
+        # All gallery image properties (id, resource_id, name) are the URN
+        # Improvement: wrap the filters by publisher, offer, etc...
+        cb_images.extend([AzureMachineImage(self.provider, image) for image
+                          in self.provider.azure_client.list_gallery_refs()
+                          if azure_helpers.generate_urn(image) == name])
         return ClientPagedResultList(self.provider, cb_images)
 
     def list(self, limit=None, marker=None):
@@ -768,8 +788,9 @@ class AzureImageService(BaseImageService):
         List all images.
         """
         azure_images = self.provider.azure_client.list_images()
+        azure_gallery_refs = self.provider.azure_client.list_gallery_refs()
         cb_images = [AzureMachineImage(self.provider, img)
-                     for img in azure_images]
+                     for img in azure_images + azure_gallery_refs]
         return ClientPagedResultList(self.provider, cb_images,
                                      limit=limit, marker=marker)
 
@@ -822,9 +843,9 @@ class AzureNetworkService(BaseNetworkService):
         try:
             network = self.provider.azure_client.get_network(network_id)
             return AzureNetwork(self.provider, network)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def list(self, limit=None, marker=None):
@@ -845,8 +866,9 @@ class AzureNetworkService(BaseNetworkService):
                             " Supported attributes: %s" % (kwargs, 'name'))
 
         filters = {'Name': name}
-        networks = [AzureNetwork(self.provider, network)
-                    for network in azure_helpers.filter_by_tag(
+        networks = [
+            AzureNetwork(self.provider, network) for network
+            in azure_helpers.filter_by_tag(
                 self.provider.azure_client.list_networks(), filters)]
         return ClientPagedResultList(self.provider, networks)
 
@@ -920,9 +942,9 @@ class AzureSubnetService(BaseSubnetService):
             azure_subnet = self.provider.azure_client.get_subnet(subnet_id)
             return AzureSubnet(self.provider,
                                azure_subnet) if azure_subnet else None
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def list(self, network=None, limit=None, marker=None):
@@ -940,15 +962,28 @@ class AzureSubnetService(BaseSubnetService):
                 if isinstance(network, Network) else network
             result_list = self.provider.azure_client.list_subnets(network_id)
         else:
-            for net in self.provider.azure_client.list_networks():
-                result_list.extend(self.provider.azure_client.list_subnets(
-                    net.id
-                ))
+            for net in self.provider.networking.networks:
+                try:
+                    result_list.extend(self.provider.azure_client.list_subnets(
+                        net.id
+                    ))
+                except CloudError as cloud_error:
+                    message = cloud_error.message
+                    if "not found" in message and "virtualNetworks" in message:
+                        log.exception(cloud_error)
+                    else:
+                        raise cloud_error
         subnets = [AzureSubnet(self.provider, subnet)
                    for subnet in result_list]
 
         return subnets
 
+    def find(self, network=None, **kwargs):
+        obj_list = self._list_subnets(network)
+        filters = ['name']
+        matches = cb_helpers.generic_find(filters, kwargs, obj_list)
+        return ClientPagedResultList(self._provider, list(matches))
+
     def create(self, network, cidr_block, name=None, **kwargs):
         """
         Create subnet
@@ -964,12 +999,12 @@ class AzureSubnetService(BaseSubnetService):
 
         subnet_info = self.provider.azure_client\
             .create_subnet(
-                            network_id,
-                            subnet_name,
-                            {
-                                'address_prefix': cidr_block
-                            }
-                          )
+                network_id,
+                subnet_name,
+                {
+                    'address_prefix': cidr_block
+                }
+            )
 
         return AzureSubnet(self.provider, subnet_info)
 
@@ -1008,9 +1043,9 @@ class AzureRouterService(BaseRouterService):
         try:
             route = self.provider.azure_client.get_route_table(router_id)
             return AzureRouter(self.provider, route)
-        except (CloudError, InvalidValueException) as cloudError:
+        except (CloudError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
-            log.exception(cloudError)
+            log.exception(cloud_error)
             return None
 
     def find(self, **kwargs):
@@ -1024,7 +1059,7 @@ class AzureRouterService(BaseRouterService):
         filters = {'Name': name}
         routes = [AzureRouter(self.provider, route)
                   for route in azure_helpers.filter_by_tag(
-                self.provider.azure_client.list_route_tables(), filters)]
+                  self.provider.azure_client.list_route_tables(), filters)]
 
         return ClientPagedResultList(self.provider, routes)
 

cloudbridge/cloud/providers/openstack/resources.py

@@ -5,6 +5,12 @@ import inspect
 import ipaddress
 import logging
 import os
+try:
+    from urllib.parse import urlparse
+    from urllib.parse import urljoin
+except ImportError:  # python 2
+    from urlparse import urlparse
+    from urlparse import urljoin
 
 import cloudbridge.cloud.base.helpers as cb_helpers
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
@@ -49,11 +55,12 @@ from neutronclient.common.exceptions import PortNotFoundClient
 import novaclient.exceptions as novaex
 
 from openstack.exceptions import HttpException
+from openstack.exceptions import NotFoundException
 from openstack.exceptions import ResourceNotFound
 
 import swiftclient
 from swiftclient.service import SwiftService, SwiftUploadObject
-
+from swiftclient.utils import generate_temp_url
 
 ONE_GIG = 1048576000  # in bytes
 FIVE_GIG = ONE_GIG * 5  # in bytes
@@ -961,7 +968,8 @@ class OpenStackFloatingIPContainer(BaseFloatingIPContainer):
         try:
             return OpenStackFloatingIP(
                 self._provider, self._provider.os_conn.network.get_ip(fip_id))
-        except ResourceNotFound:
+        except (ResourceNotFound, NotFoundException):
+            log.debug("Floating IP %s not found.", fip_id)
             return None
 
     def list(self, limit=None, marker=None):
@@ -1222,7 +1230,7 @@ class OpenStackVMFirewallRuleContainer(BaseVMFirewallRuleContainer):
         except HttpException as e:
             self.firewall.refresh()
             # 409=Conflict, raised for duplicate rule
-            if e.http_status == 409:
+            if e.status_code == 409:
                 existing = self.find(direction=direction, protocol=protocol,
                                      from_port=from_port, to_port=to_port,
                                      cidr=cidr, src_dest_fw_id=src_dest_fw_id)
@@ -1342,7 +1350,7 @@ class OpenStackBucketObject(BaseBucketObject):
               ``swiftclient.service.get_conn`` factory method to
               ``self._provider._connect_swift``
 
-        .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
+        .. seealso:: https://github.com/CloudVE/cloudbridge/issues/35#issuecomment-297629661 # noqa
         """
         upload_options = {}
         if 'segment_size' not in upload_options:
@@ -1384,18 +1392,19 @@ class OpenStackBucketObject(BaseBucketObject):
                 result = result and del_res['success']
         return result
 
-    def generate_url(self, expires_in=0):
-        """
-        Generates a URL to this object.
-
-        If the object is public, `expires_in` argument is not necessary, but if
-        the object is private, the life time of URL is set using `expires_in`
-        argument.
+    def generate_url(self, expires_in):
+        # Set a temp url key on the object (http://bit.ly/2NBiXGD)
+        temp_url_key = "cloudbridge-tmp-url-key"
+        self._provider.swift.post_account(
+            headers={"x-account-meta-temp-url-key": temp_url_key})
+        base_url = urlparse(self._provider.swift.get_service_auth()[0])
+        access_point = "{0}://{1}".format(base_url.scheme, base_url.netloc)
+        url_path = "/".join([base_url.path, self.cbcontainer.name, self.name])
+        return urljoin(access_point, generate_temp_url(url_path, expires_in,
+                                                       temp_url_key, 'GET'))
 
-        See here for implementation details:
-        http://stackoverflow.com/a/37057172
-        """
-        raise NotImplementedError("This functionality is not implemented yet.")
+    def refresh(self):
+        self._obj = self.cbcontainer.objects.get(self.id)._obj
 
 
 class OpenStackBucket(BaseBucket):

cloudbridge/cloud/providers/openstack/services.py

@@ -42,6 +42,7 @@ from neutronclient.common.exceptions import NeutronClientException
 
 from novaclient.exceptions import NotFound as NovaNotFound
 
+from openstack.exceptions import NotFoundException
 from openstack.exceptions import ResourceNotFound
 
 from .resources import OpenStackBucket
@@ -198,7 +199,7 @@ class OpenStackVMFirewallService(BaseVMFirewallService):
             return OpenStackVMFirewall(
                 self.provider,
                 self.provider.os_conn.network.get_security_group(firewall_id))
-        except ResourceNotFound:
+        except (ResourceNotFound, NotFoundException):
             log.debug("Firewall %s not found.", firewall_id)
             return None
 
@@ -256,9 +257,8 @@ class OpenStackImageService(BaseImageService):
         try:
             return OpenStackMachineImage(
                 self.provider, self.provider.os_conn.image.get_image(image_id))
-        except ResourceNotFound:
-            log.debug("ResourceNotFound exception raised, %s not found",
-                      image_id)
+        except (NotFoundException, ResourceNotFound):
+            log.debug("Image %s not found", image_id)
             return None
 
     def find(self, **kwargs):

docs/getting_started.rst

@@ -33,7 +33,7 @@ AWS:
     config = {'aws_access_key': 'AKIAJW2XCYO4AF55XFEQ',
               'aws_secret_key': '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'}
     provider = CloudProviderFactory().create_provider(ProviderList.AWS, config)
-    image_id = 'ami-2d39803a'  # Ubuntu 14.04 (HVM)
+    image_id = 'ami-aa2ea6d0'  # Ubuntu 16.04 (HVM)
 
 OpenStack (with Keystone authentication v2):
 
@@ -64,7 +64,7 @@ OpenStack (with Keystone authentication v3):
               'os_user_domain_name': 'domain name'}
     provider = CloudProviderFactory().create_provider(ProviderList.OPENSTACK,
                                                       config)
-    image_id = '97755049-ee4f-4515-b92f-ca00991ee99a'  # Ubuntu 14.04 @ Jetstream
+    image_id = 'acb53109-941f-4593-9bf8-4a53cb9e0739'  # Ubuntu 16.04 @ Jetstream
 
 Azure:
 
@@ -77,7 +77,7 @@ Azure:
               'azure_secret': 'REPLACE WITH ACTUAL VALUE',
               'azure_tenant': ' REPLACE WITH ACTUAL VALUE'}
     provider = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
-    image_id = 'ami-2d39803a'  # Ubuntu 14.04 (HVM)
+    image_id = 'Canonical:UbuntuServer:16.04.0-LTS:latest'  # Ubuntu 16.04
 
 
 List some resources

docs/topics/design-decisions.rst

@@ -17,4 +17,4 @@ It is intended as a reference.
   lead to a miss match. (Related to 63_.)
 
 
-  .. _63: https://github.com/gvlproject/cloudbridge/issues/63
+  .. _63: https://github.com/CloudVE/cloudbridge/issues/63

docs/topics/install.rst

@@ -22,9 +22,9 @@ The latest release of cloudbridge can be installed from PyPI::
 Latest unreleased dev version
 -----------------------------
 The development version of the library can be installed from the
-`Github repo <https://github.com/gvlproject/cloudbridge>`_::
+`Github repo <https://github.com/CloudVE/cloudbridge>`_::
 
-    $ git clone https://github.com/gvlproject/cloudbridge.git
+    $ git clone https://github.com/CloudVE/cloudbridge.git
     $ cd cloudbridge
     $ python setup.py install
 

docs/topics/provider_development.rst

@@ -233,8 +233,8 @@ specific manner.
 
 
 
-.. _commit 1: https://github.com/gvlproject/cloudbridge/commit/54c67e93a3cd9d51e7d2b1195ebf4e257d165297
-.. _commit 2: https://github.com/gvlproject/cloudbridge/commit/82c0244aa4229ae0aecfe40d769eb93b06470dc7
-.. _commit 3: https://github.com/gvlproject/cloudbridge/commit/e90a7f6885814a3477cd0b38398d62af64f91093
-.. _commit 4: https://github.com/gvlproject/cloudbridge/commit/2d5c14166a538d320e54eed5bc3fa04997828715
-.. _commit 5: https://github.com/gvlproject/cloudbridge/commit/98c9cf578b672867ee503027295f9d901411e496
+.. _commit 1: https://github.com/CloudVE/cloudbridge/commit/54c67e93a3cd9d51e7d2b1195ebf4e257d165297
+.. _commit 2: https://github.com/CloudVE/cloudbridge/commit/82c0244aa4229ae0aecfe40d769eb93b06470dc7
+.. _commit 3: https://github.com/CloudVE/cloudbridge/commit/e90a7f6885814a3477cd0b38398d62af64f91093
+.. _commit 4: https://github.com/CloudVE/cloudbridge/commit/2d5c14166a538d320e54eed5bc3fa04997828715
+.. _commit 5: https://github.com/CloudVE/cloudbridge/commit/98c9cf578b672867ee503027295f9d901411e496

docs/topics/setup.rst

@@ -8,6 +8,12 @@ be provided in one of following ways:
 2. A dictionary
 3. Configuration file
 
+Procuring access credentials
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+For Azure, Create service principle credentials from the following link : 
+https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal#check-azure-subscription-permissions
+
+
 Providing access credentials through environment variables
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 The following environment variables must be set, depending on the provider in use.
@@ -77,9 +83,6 @@ will override environment values.
               'azure_resource_group': '<your resource group>'}
     provider = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
 
-For Azure, Create service principle credentials from the following link : 
-https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal#check-azure-subscription-permissions
-
 Some optional configuration values can only be provided through the config
 dictionary. These are listed below for each provider.
 

docs/topics/testing.rst

@@ -74,9 +74,9 @@ You can toggle the use of mock providers by setting an environment variable:
 ``CB_USE_MOCK_PROVIDERS`` to ``Yes`` or ``No``.
 
 
-.. _design goals: https://github.com/gvlproject/cloudbridge/
+.. _design goals: https://github.com/CloudVE/cloudbridge/
    blob/master/README.rst
 .. _tox: https://tox.readthedocs.org/en/latest/
-.. _ProviderList: https://github.com/gvlproject/cloudbridge/blob/master/
+.. _ProviderList: https://github.com/CloudVE/cloudbridge/blob/master/
    cloudbridge/cloud/factory.py#L15
 .. _moto: https://github.com/spulec/moto

setup.cfg

@@ -9,10 +9,13 @@ with-coverage=1
 cover-branches=1
 cover-package=cloudbridge
 processes=5
-process-timeout=2700
-match=^[Tt]est 
+process-timeout=3000
+match=^[Tt]est
+verbosity=2
 # Don't capture stdout - print immediately
 nocapture=1
+# When exceptions occur, filter only cloudbridge logs
+logging-filter=cloudbridge
 
 [bdist_wheel]
 universal = 1

setup.py

@@ -21,17 +21,10 @@ with open(os.path.join('cloudbridge', '__init__.py')) as f:
 REQS_BASE = [
     'bunch>=1.0.1',
     'six>=1.10.0',
-    'retrying>=1.3.3'
+    'tenacity>=4.12.0'
 ]
 REQS_AWS = ['boto3']
-REQS_AZURE = ['msrest>=0.4.7',
-              'msrestazure>=0.4.7',
-              'azure-common>=1.1.5',
-              'azure-mgmt-resource>=1.0.0rc1',
-              'azure-mgmt-compute>=1.0.0rc1',
-              'azure-mgmt-network>=1.0.0rc1',
-              'azure-mgmt-storage>=1.0.0rc1',
-              'azure-storage>=0.34.0',
+REQS_AZURE = ['azure>=3.0.0',
               'pysftp>=0.2.9']
 REQS_OPENSTACK = [
     'openstacksdk>=0.12.0',
@@ -47,7 +40,7 @@ REQS_FULL = REQS_BASE + REQS_AWS + REQS_AZURE + REQS_OPENSTACK
 REQS_DEV = ([
     'tox>=2.1.1',
     'nose',
-    # 'moto>=1.1.11',  # until https://github.com/spulec/moto/issues/1396
+    'moto>=1.3.2',
     'sphinx>=1.3.1',
     'pydevd',
     'flake8>=3.3.0',
@@ -61,6 +54,7 @@ setup(
     author='Galaxy and GVL Projects',
     author_email='help@genome.edu.au',
     url='http://cloudbridge.cloudve.org/',
+    setup_requires=['nose>=1.0'],
     install_requires=REQS_FULL,
     extras_require={
         ':python_version<"3.3"': ['ipaddress'],

test/helpers/__init__.py

@@ -86,7 +86,7 @@ TEST_DATA_CONFIG = {
     },
     "OpenStackCloudProvider": {
         "image": os.environ.get('CB_IMAGE_OS',
-                                '842b949c-ea76-48df-998d-8a41f2626243'),
+                                'acb53109-941f-4593-9bf8-4a53cb9e0739'),
         "vm_type": os.environ.get('CB_VM_TYPE_OS', 'm1.tiny'),
         "placement": os.environ.get('CB_PLACEMENT_OS', 'zone-r1'),
     },
@@ -95,9 +95,7 @@ TEST_DATA_CONFIG = {
             os.environ.get('CB_PLACEMENT_AZURE', 'eastus'),
         "image":
             os.environ.get('CB_IMAGE_AZURE',
-                           '/subscriptions/7904d702-e01c-4826-8519-f5a25c866a9'
-                           '6/resourceGroups/cloudbridge/providers/Microsoft.C'
-                           'ompute/images/cb-test-image'),
+                           'Canonical:UbuntuServer:16.04.0-LTS:latest'),
         "vm_type":
             os.environ.get('CB_VM_TYPE_AZURE', 'Basic_A2'),
     }

test/test_block_store_service.py

@@ -32,9 +32,10 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                 helpers.get_provider_test_data(self.provider, "placement"))
 
         def cleanup_vol(vol):
-            vol.delete()
-            vol.wait_for([VolumeState.DELETED, VolumeState.UNKNOWN],
-                         terminal_states=[VolumeState.ERROR])
+            if vol:
+                vol.delete()
+                vol.wait_for([VolumeState.DELETED, VolumeState.UNKNOWN],
+                             terminal_states=[VolumeState.ERROR])
 
         sit.check_crud(self, self.provider.storage.volumes, Volume,
                        "cb_createvol", create_vol, cleanup_vol)
@@ -150,10 +151,10 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                                                 description=name)
 
             def cleanup_snap(snap):
-                snap.delete()
-                snap.wait_for(
-                    [SnapshotState.UNKNOWN],
-                    terminal_states=[SnapshotState.ERROR])
+                if snap:
+                    snap.delete()
+                    snap.wait_for([SnapshotState.UNKNOWN],
+                                  terminal_states=[SnapshotState.ERROR])
 
             sit.check_crud(self, self.provider.storage.snapshots, Snapshot,
                            "cb_snap", create_snap, cleanup_snap)
@@ -186,10 +187,10 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                                                  description=snap_name)
 
             def cleanup_snap(snap):
-                snap.delete()
-                snap.wait_for(
-                    [SnapshotState.UNKNOWN],
-                    terminal_states=[SnapshotState.ERROR])
+                if snap:
+                    snap.delete()
+                    snap.wait_for([SnapshotState.UNKNOWN],
+                                  terminal_states=[SnapshotState.ERROR])
 
             with helpers.cleanup_action(lambda: cleanup_snap(test_snap)):
                 test_snap.wait_till_ready()

test/test_compute_service.py

@@ -33,8 +33,9 @@ class CloudComputeServiceTestCase(ProviderTestBase):
                                              subnet=subnet, user_data={})
 
         def cleanup_inst(inst):
-            inst.delete()
-            inst.wait_for([InstanceState.DELETED, InstanceState.UNKNOWN])
+            if inst:
+                inst.delete()
+                inst.wait_for([InstanceState.DELETED, InstanceState.UNKNOWN])
 
         def check_deleted(inst):
             deleted_inst = self.provider.compute.instances.get(
@@ -232,12 +233,12 @@ class CloudComputeServiceTestCase(ProviderTestBase):
                                                  description=name)
 
             def cleanup_snap(snap):
-                snap.delete()
-                snap.wait_for([SnapshotState.UNKNOWN],
-                              terminal_states=[SnapshotState.ERROR])
+                if snap:
+                    snap.delete()
+                    snap.wait_for([SnapshotState.UNKNOWN],
+                                  terminal_states=[SnapshotState.ERROR])
 
-            with helpers.cleanup_action(lambda:
-                                        cleanup_snap(test_snap)):
+            with helpers.cleanup_action(lambda: cleanup_snap(test_snap)):
                 test_snap.wait_till_ready()
 
                 lc = self.provider.compute.instances.create_launch_config()

test/test_image_service.py

@@ -24,14 +24,16 @@ class CloudImageServiceTestCase(ProviderTestBase):
         # the cleanup method access to the most current values
         test_instance = None
         net = None
+        subnet = None
 
         def create_img(name):
             return test_instance.create_image(name)
 
         def cleanup_img(img):
-            img.delete()
-            img.wait_for(
-                [MachineImageState.UNKNOWN, MachineImageState.ERROR])
+            if img:
+                img.delete()
+                img.wait_for(
+                    [MachineImageState.UNKNOWN, MachineImageState.ERROR])
 
         def extra_tests(img):
             # check image size
@@ -45,7 +47,6 @@ class CloudImageServiceTestCase(ProviderTestBase):
                 self.provider, instance_name)
             test_instance = helpers.get_test_instance(
                 self.provider, instance_name, subnet=subnet)
-
             sit.check_crud(self, self.provider.compute.images, MachineImage,
                            "cb_listimg", create_img, cleanup_img,
                            extra_test_func=extra_tests)

test/test_network_service.py

@@ -21,7 +21,8 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 name=name, cidr_block='10.0.0.0/16')
 
         def cleanup_net(net):
-            self.provider.networking.networks.delete(network_id=net.id)
+            if net:
+                self.provider.networking.networks.delete(network_id=net.id)
 
         sit.check_crud(self, self.provider.networking.networks, Network,
                        "cb_crudnetwork", create_net, cleanup_net)
@@ -90,7 +91,8 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                     self.provider, 'placement'))
 
         def cleanup_subnet(subnet):
-            self.provider.networking.subnets.delete(subnet=subnet)
+            if subnet:
+                self.provider.networking.subnets.delete(subnet=subnet)
 
         net_name = 'cb_crudsubnet-{0}'.format(helpers.get_uuid())
         net = self.provider.networking.networks.create(
@@ -111,7 +113,8 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
             return fip
 
         def cleanup_fip(fip):
-            gw.floating_ips.delete(fip.id)
+            if fip:
+                gw.floating_ips.delete(fip.id)
 
         with helpers.cleanup_action(
                 lambda: helpers.delete_test_gateway(net, gw)):

test/test_object_store_service.py

@@ -9,7 +9,6 @@ from test.helpers import ProviderTestBase
 from test.helpers import standard_interface_tests as sit
 from unittest import skip
 
-from cloudbridge.cloud.factory import ProviderList
 from cloudbridge.cloud.interfaces.exceptions import InvalidNameException
 from cloudbridge.cloud.interfaces.provider import TestMockHelperMixin
 from cloudbridge.cloud.interfaces.resources import Bucket
@@ -33,7 +32,8 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
             return self.provider.storage.buckets.create(name)
 
         def cleanup_bucket(bucket):
-            bucket.delete()
+            if bucket:
+                bucket.delete()
 
         with self.assertRaises(InvalidNameException):
             # underscores are not allowed in bucket names
@@ -69,7 +69,8 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
             return obj
 
         def cleanup_bucket_obj(bucket_obj):
-            bucket_obj.delete()
+            if bucket_obj:
+                bucket_obj.delete()
 
         with helpers.cleanup_action(lambda: test_bucket.delete()):
             name = "cb-crudbucketobj-{0}".format(uuid.uuid4())
@@ -110,6 +111,13 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                     isinstance(objs[0].size, int),
                     "Object size property needs to be a int, not {0}".format(
                         type(objs[0].size)))
+                # GET an object as the size property implementation differs
+                # for objects returned by LIST and GET.
+                obj = test_bucket.objects.get(objs[0].id)
+                self.assertTrue(
+                    isinstance(objs[0].size, int),
+                    "Object size property needs to be an int, not {0}".format(
+                        type(obj.size)))
                 self.assertTrue(
                     datetime.strptime(objs[0].last_modified[:23],
                                       "%Y-%m-%dT%H:%M:%S.%f"),
@@ -160,9 +168,6 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['storage.buckets'])
     def test_generate_url(self):
-        if self.provider.PROVIDER_ID == ProviderList.OPENSTACK:
-            raise self.skipTest("Skip until OpenStack impl is provided")
-
         name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
         test_bucket = self.provider.storage.buckets.create(name)
 

test/test_security_service.py

@@ -22,7 +22,8 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             return self.provider.security.key_pairs.create(name=name)
 
         def cleanup_kp(kp):
-            self.provider.security.key_pairs.delete(key_pair_id=kp.id)
+            if kp:
+                self.provider.security.key_pairs.delete(key_pair_id=kp.id)
 
         def extra_tests(kp):
             # Recreating existing keypair should raise an exception
@@ -70,7 +71,8 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 name=name, description=name, network_id=net.id)
 
         def cleanup_fw(fw):
-            fw.delete()
+            if fw:
+                fw.delete()
 
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 network=net)):
@@ -117,7 +119,8 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                         from_port=1111, to_port=1111, cidr='0.0.0.0/0')
 
                 def cleanup_fw_rule(rule):
-                    rule.delete()
+                    if rule:
+                        rule.delete()
 
                 sit.check_crud(self, fw.rules, VMFirewallRule, "cb_crudfwrule",
                                create_fw_rule, cleanup_fw_rule,

tox.ini

@@ -17,7 +17,7 @@ envlist = {py27,py36,pypy}-{aws,azure,openstack}
 [testenv]
 commands = flake8 cloudbridge test setup.py
            # see setup.cfg for options sent to nosetests and coverage
-           {envpython} setup.py nosetests {posargs}
+           nosetests --logging-format='%(asctime)s [%(levelname)s] %(name)s: %(message)s' {posargs}
 setenv =
     MOTO_AMIS_PATH=./test/fixtures/custom_amis.json
     aws: CB_TEST_PROVIDER=aws