Merge pull request #276 from CloudVE/update_azure_09_21

Update to latest azure libs

cloudbridge/__init__.py

@@ -2,7 +2,7 @@
 import logging
 
 # Current version of the library
-__version__ = '2.1.0'
+__version__ = '2.2.0'
 
 
 def get_version():

cloudbridge/base/helpers.py

@@ -1,9 +1,9 @@
 import fnmatch
 import functools
+import logging
 import os
 import re
 import sys
-import traceback
 from contextlib import contextmanager
 
 from cryptography.hazmat.backends import default_backend
@@ -18,6 +18,8 @@ import cloudbridge
 
 from ..interfaces.exceptions import InvalidParamException
 
+log = logging.getLogger(__name__)
+
 
 def generate_key_pair():
     """
@@ -102,15 +104,13 @@ def cleanup_action(cleanup_func):
         ex_class, ex_val, ex_traceback = sys.exc_info()
         try:
             cleanup_func()
-        except Exception as e:
-            print("Error during exception cleanup: {0}".format(e))
-            traceback.print_exc()
+        except Exception:
+            log.exception("Error during exception cleanup: ")
         six.reraise(ex_class, ex_val, ex_traceback)
     try:
         cleanup_func()
-    except Exception as e:
-        print("Error during cleanup: {0}".format(e))
-        traceback.print_exc()
+    except Exception:
+        log.exception("Error during exception cleanup: ")
 
 
 def get_env(varname, default_value=None):

cloudbridge/interfaces/services.py

@@ -1,9 +1,7 @@
 """
 Specifications for services available through a provider
 """
-from abc import ABCMeta
-from abc import abstractmethod
-from abc import abstractproperty
+from abc import ABCMeta, abstractmethod, abstractproperty
 
 from cloudbridge.interfaces.resources import PageableObjectMixin
 
@@ -1181,7 +1179,7 @@ class BucketObjectService(CloudService):
         pass
 
     @abstractmethod
-    def list(self, bucket, limit=None, marker=None):
+    def list(self, bucket, prefix=None, limit=None, marker=None):
         """
         List all bucket objects within a bucket.
 

cloudbridge/providers/azure/azure_client.py

@@ -1,28 +1,25 @@
 import datetime
 import logging
-from io import BytesIO
 
-from azure.common import AzureConflictHttpError
-from azure.common.credentials import ServicePrincipalCredentials
+import tenacity
+from cloudbridge.interfaces.exceptions import (DuplicateResourceException,
+                                               InvalidLabelException,
+                                               ProviderConnectionException,
+                                               WaitStateException)
+
+from azure.core.exceptions import (ClientAuthenticationError,
+                                   HttpResponseError, ResourceExistsError,
+                                   ResourceNotFoundError)
 from azure.cosmosdb.table.tableservice import TableService
+from azure.identity import ClientSecretCredential
 from azure.mgmt.compute import ComputeManagementClient
 from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.network import NetworkManagementClient
 from azure.mgmt.resource import ResourceManagementClient
 from azure.mgmt.resource.subscriptions import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
-from azure.storage.blob import BlobPermissions
-from azure.storage.blob import BlockBlobService
-from azure.storage.common import TokenCredential
-
-from msrestazure.azure_exceptions import CloudError
-
-import tenacity
-
-from cloudbridge.interfaces.exceptions import DuplicateResourceException
-from cloudbridge.interfaces.exceptions import InvalidLabelException
-from cloudbridge.interfaces.exceptions import ProviderConnectionException
-from cloudbridge.interfaces.exceptions import WaitStateException
+from azure.storage.blob import (BlobSasPermissions, BlobServiceClient,
+                                generate_blob_sas)
 
 from . import helpers as azure_helpers
 
@@ -161,10 +158,10 @@ class AzureClient(object):
     def __init__(self, config):
         self._config = config
         self.subscription_id = str(config.get('azure_subscription_id'))
-        self._credentials = ServicePrincipalCredentials(
+        self._credentials = ClientSecretCredential(
+            tenant_id=config.get('azure_tenant'),
             client_id=config.get('azure_client_id'),
-            secret=config.get('azure_secret'),
-            tenant=config.get('azure_tenant')
+            client_secret=config.get('azure_secret')
         )
 
         self._access_token = config.get('azure_access_token')
@@ -181,13 +178,13 @@ class AzureClient(object):
         log.debug("azure subscription : %s", self.subscription_id)
 
     @property
-    @tenacity.retry(stop=tenacity.stop_after_attempt(5), reraise=True)
+    @tenacity.retry(stop=tenacity.stop.stop_after_attempt(5), reraise=True)
     def access_key_result(self):
         if not self._access_key_result:
             storage_account = self.storage_account
 
             if self.get_storage_account(storage_account).\
-                    provisioning_state.value != 'Succeeded':
+                    provisioning_state != 'Succeeded':
                 log.debug(
                     "Storage account %s is not in Succeeded state yet. ",
                     storage_account)
@@ -260,14 +257,13 @@ class AzureClient(object):
         self._get_or_create_storage_account()
         if not self._block_blob_service:
             if self._access_token:
-                token_credential = TokenCredential(self._access_token)
-                self._block_blob_service = BlockBlobService(
-                    account_name=self.storage_account,
-                    token_credential=token_credential)
+                self._block_blob_service = BlobServiceClient(
+                    account_url=f"https://{self.storage_account}.blob.core.windows.net/",
+                    credential=self._access_token)
             else:
-                self._block_blob_service = BlockBlobService(
-                    account_name=self.storage_account,
-                    account_key=self.access_key_result.keys[0].value)
+                self._block_blob_service = BlobServiceClient(
+                    account_url=f"https://{self.storage_account}.blob.core.windows.net/",
+                    credential=self._credentials)
         return self._block_blob_service
 
     @property
@@ -283,6 +279,9 @@ class AzureClient(object):
                 self.public_key_storage_table_name)
         return self._table_service
 
+    def blob_client(self, container_name, blob_name):
+        return self.blob_service.get_blob_client(container=container_name, blob=blob_name)
+
     def get_resource_group(self, name):
         return self.resource_client.resource_groups.get(name)
 
@@ -296,12 +295,12 @@ class AzureClient(object):
 
     def create_storage_account(self, name, params):
         return self.storage_client.storage_accounts. \
-            create(self.resource_group, name.lower(), params).result()
+            begin_create(self.resource_group, name.lower(), params).result()
 
     # Create a storage account. To prevent a race condition, try
     # to get or create at least twice
-    @tenacity.retry(stop=tenacity.stop_after_attempt(2),
-                    retry=tenacity.retry_if_exception_type(CloudError),
+    @tenacity.retry(stop=tenacity.stop.stop_after_attempt(2),
+                    retry=tenacity.retry_if_exception_type(HttpResponseError),
                     reraise=True)
     def _get_or_create_storage_account(self):
         if self._storage_account:
@@ -310,52 +309,44 @@ class AzureClient(object):
             try:
                 self._storage_account = \
                     self.get_storage_account(self.storage_account)
-            except CloudError as cloud_error:
-                if cloud_error.error.error == "ResourceNotFound":
-                    storage_account_params = {
-                        'sku': {
-                            'name': 'Standard_LRS'
-                        },
-                        'kind': 'storage',
-                        'location': self.region_name,
-                    }
-                    try:
-                        self._storage_account = \
-                            self.create_storage_account(self.storage_account,
-                                                        storage_account_params)
-                    except CloudError as cloud_error2:  # pragma: no cover
-                        if cloud_error2.error.error == "AuthorizationFailed":
-                            mess = 'The following error was returned by ' \
-                                   'Azure:\n%s\n\nThis is likely because the' \
-                                   ' Role associated with the provided ' \
-                                   'credentials does not allow for Storage ' \
-                                   'Account creation.\nA Storage Account is ' \
-                                   'necessary in order to perform the ' \
-                                   'desired operation. You must either ' \
-                                   'provide an existing Storage Account name' \
-                                   ' as part of the configuration, or ' \
-                                   'elevate the associated Role.\nFor more ' \
-                                   'information on roles, see: https://docs.' \
-                                   'microsoft.com/en-us/azure/role-based-' \
-                                   'access-control/overview\n' % cloud_error2
-                            raise ProviderConnectionException(mess)
-
-                        elif cloud_error2.error.error == \
-                                "StorageAccountAlreadyTaken":
-                            mess = 'The following error was ' \
-                                   'returned by Azure:\n%s\n\n' \
-                                   'Note that Storage Account names must be ' \
-                                   'unique across Azure (not just in your ' \
-                                   'subscription).\nFor more information ' \
-                                   'see https://docs.microsoft.com/en-us/' \
-                                   'azure/azure-resource-manager/resource-' \
-                                   'manager-storage-account-name-errors\n' \
-                                   % cloud_error2
-                            raise InvalidLabelException(mess)
-                        else:
-                            raise cloud_error2
-                else:
-                    raise cloud_error
+            except ResourceNotFoundError:
+                storage_account_params = {
+                    'sku': {
+                        'name': 'Standard_LRS'
+                    },
+                    'kind': 'storage',
+                    'location': self.region_name,
+                }
+                try:
+                    self._storage_account = \
+                        self.create_storage_account(self.storage_account,
+                                                    storage_account_params)
+                except ClientAuthenticationError as auth_err:
+                    mess = 'The following error was returned by ' \
+                           'Azure:\n%s\n\nThis is likely because the' \
+                           ' Role associated with the provided ' \
+                           'credentials does not allow for Storage ' \
+                           'Account creation.\nA Storage Account is ' \
+                           'necessary in order to perform the ' \
+                           'desired operation. You must either ' \
+                           'provide an existing Storage Account name' \
+                           ' as part of the configuration, or ' \
+                           'elevate the associated Role.\nFor more ' \
+                           'information on roles, see: https://docs.' \
+                           'microsoft.com/en-us/azure/role-based-' \
+                           'access-control/overview\n' % auth_err
+                    raise ProviderConnectionException(mess)
+                except ResourceExistsError as exists_err:
+                    mess = 'The following error was ' \
+                           'returned by Azure:\n%s\n\n' \
+                           'Note that Storage Account names must be ' \
+                           'unique across Azure (not just in your ' \
+                           'subscription).\nFor more information ' \
+                           'see https://docs.microsoft.com/en-us/' \
+                           'azure/azure-resource-manager/resource-' \
+                           'manager-storage-account-name-errors\n' \
+                               % exists_err
+                    raise InvalidLabelException(mess)
 
     def list_locations(self):
         return self.subscription_client.subscriptions. \
@@ -367,116 +358,129 @@ class AzureClient(object):
 
     def create_vm_firewall(self, name, parameters):
         return self.network_management_client.network_security_groups. \
-            create_or_update(self.resource_group, name,
-                             parameters).result()
+            begin_create_or_update(self.resource_group, name,
+                                   parameters).result()
 
     def update_vm_firewall_tags(self, fw_id, tags):
         url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
                                              fw_id)
-        name = url_params.get(VM_FIREWALL_NAME)
+        name = url_params.get(VM_FIREWALL_NAME, "")
         return self.network_management_client.network_security_groups. \
-            create_or_update(self.resource_group, name,
-                             {'tags': tags,
-                              'location': self.region_name}).result()
+            begin_create_or_update(self.resource_group, name,
+                                   {'tags': tags,
+                                    'location': self.region_name}).result()
 
     def get_vm_firewall(self, fw_id):
         url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
                                              fw_id)
-        fw_name = url_params.get(VM_FIREWALL_NAME)
+        fw_name = url_params.get(VM_FIREWALL_NAME, "")
         return self.network_management_client.network_security_groups. \
             get(self.resource_group, fw_name)
 
     def delete_vm_firewall(self, fw_id):
         url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
                                              fw_id)
-        name = url_params.get(VM_FIREWALL_NAME)
+        name = url_params.get(VM_FIREWALL_NAME, "")
         self.network_management_client \
-            .network_security_groups.delete(self.resource_group, name).wait()
+            .network_security_groups.begin_delete(self.resource_group, name).wait()
 
     def create_vm_firewall_rule(self, fw_id,
                                 rule_name, parameters):
         url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
                                              fw_id)
-        vm_firewall_name = url_params.get(VM_FIREWALL_NAME)
+        vm_firewall_name = url_params.get(VM_FIREWALL_NAME, "")
         return self.network_management_client.security_rules. \
-            create_or_update(self.resource_group, vm_firewall_name,
-                             rule_name, parameters).result()
+            begin_create_or_update(self.resource_group, vm_firewall_name,
+                                   rule_name, parameters).result()
 
     def delete_vm_firewall_rule(self, fw_rule_id, vm_firewall):
         url_params = azure_helpers.parse_url(VM_FIREWALL_RULE_RESOURCE_ID,
                                              fw_rule_id)
-        name = url_params.get(VM_FIREWALL_RULE_NAME)
+        name = url_params.get(VM_FIREWALL_RULE_NAME, "")
         return self.network_management_client.security_rules. \
-            delete(self.resource_group, vm_firewall, name).result()
+            begin_delete(self.resource_group, vm_firewall, name).result()
 
     def list_containers(self, prefix=None, limit=None, marker=None):
-        results = self.blob_service.list_containers(prefix=prefix,
-                                                    num_results=limit,
+        results = self.blob_service.list_containers(name_starts_with=prefix,
+                                                    results_per_page=limit,
                                                     marker=marker)
-        return (results.items, results.next_marker)
+        return results
 
     def create_container(self, container_name):
         try:
-            self.blob_service.create_container(container_name,
-                                               fail_on_exist=True)
-        except AzureConflictHttpError as cloud_error:
-            if cloud_error.error_code == "ContainerAlreadyExists":
-                msg = "The given Bucket name '%s' already exists. Please " \
-                      "use the `get` or `find` method to get a reference to " \
-                      "an existing Bucket, or specify a new Bucket name to " \
-                      "create.\nNote that in Azure, Buckets are contained " \
-                      "in Storage Accounts." % container_name
-                raise DuplicateResourceException(msg)
-
-        return self.blob_service.get_container_properties(container_name)
+            return self.blob_service.create_container(container_name)
+        except ResourceExistsError:
+            msg = "The given Bucket name '%s' already exists. Please " \
+                    "use the `get` or `find` method to get a reference to " \
+                    "an existing Bucket, or specify a new Bucket name to " \
+                    "create.\nNote that in Azure, Buckets are contained " \
+                    "in Storage Accounts." % container_name
+            raise DuplicateResourceException(msg)
 
     def get_container(self, container_name):
-        return self.blob_service.get_container_properties(container_name)
+        return self.blob_service.get_container_client(container_name)
 
     def delete_container(self, container_name):
         self.blob_service.delete_container(container_name)
 
-    def list_blobs(self, container_name, prefix=None):
-        return self.blob_service.list_blobs(container_name, prefix=prefix)
+    def list_blobs(self, container_name, prefix=None, include=None):
+        container_client = self.get_container(container_name)
+        return container_client.list_blobs(name_starts_with=prefix, include=include)
+
+    def upload_blob(self, container_name, blob_name, data, length=None):
+        blob_client = self.blob_client(container_name, blob_name)
+        blob_client.upload_blob(data=data, length=length, overwrite=True)
 
     def get_blob(self, container_name, blob_name):
-        return self.blob_service.get_blob_properties(container_name, blob_name)
+        blob_client = self.blob_client(container_name, blob_name)
+        return blob_client.get_blob_properties(container_name, blob_name)
 
     def create_blob_from_text(self, container_name, blob_name, text):
-        self.blob_service.create_blob_from_text(container_name,
-                                                blob_name, text)
+        if isinstance(text, bytes):
+            length = len(text)
+        else:
+            length = len(text.encode())
+        self.upload_blob(container_name, blob_name, text, length)
 
-    def create_blob_from_file(self, container_name, blob_name, file_path):
-        self.blob_service.create_blob_from_path(container_name,
-                                                blob_name, file_path)
+    def create_blob_from_file(self, container_name, blob_name, file_path, length=None):
+        with open(file_path, 'rb') as data:
+            data = data.read()
+            self.upload_blob(container_name, blob_name, data, len(data))
 
-    def delete_blob(self, container_name, blob_name):
-        self.blob_service.delete_blob(container_name, blob_name)
+    def delete_blob(self, container_name, blob_name, delete_snapshots="include"):
+        blob_client = self.blob_client(container_name, blob_name)
+        blob_client.delete_blob(delete_snapshots)
 
     def get_blob_url(self, container_name, blob_name, expiry_time):
-        expiry_date = datetime.datetime.utcnow() + datetime.timedelta(
+        now = datetime.datetime.utcnow()
+        expiry = now + datetime.timedelta(
             seconds=expiry_time)
-        sas = self.blob_service.generate_blob_shared_access_signature(
-            container_name, blob_name, permission=BlobPermissions.READ,
-            expiry=expiry_date)
-        return self.blob_service.make_blob_url(container_name, blob_name,
-                                               sas_token=sas)
-
-    def get_blob_content(self, container_name, blob_name):
-        out_stream = BytesIO()
-        self.blob_service.get_blob_to_stream(container_name,
-                                             blob_name, out_stream)
-        return out_stream
+        blob_name = blob_name
+        container_name = container_name.name
+        delegation_key = self.blob_service.get_user_delegation_key(
+            key_start_time=now, key_expiry_time=expiry
+        )
+        sas = generate_blob_sas(
+            self.storage_account, container_name, blob_name,
+            permission=BlobSasPermissions(read=True), expiry=expiry,
+            user_delegation_key=delegation_key
+        )
+        url = (
+            f"https://{self.storage_account}.blob.core.windows.net/"
+            f"{container_name}/{blob_name}?{sas}"
+        )
+
+        return url
 
     def create_empty_disk(self, disk_name, params):
-        return self.compute_client.disks.create_or_update(
+        return self.compute_client.disks.begin_create_or_update(
             self.resource_group,
             disk_name,
             params
         ).result()
 
     def create_snapshot_disk(self, disk_name, params):
-        return self.compute_client.disks.create_or_update(
+        return self.compute_client.disks.begin_create_or_update(
             self.resource_group,
             disk_name,
             params
@@ -485,7 +489,7 @@ class AzureClient(object):
     def get_disk(self, disk_id):
         url_params = azure_helpers.parse_url(VOLUME_RESOURCE_ID,
                                              disk_id)
-        disk_name = url_params.get(VOLUME_NAME)
+        disk_name = url_params.get(VOLUME_NAME, "")
         return self.compute_client.disks.get(self.resource_group, disk_name)
 
     def list_disks(self):
@@ -495,19 +499,18 @@ class AzureClient(object):
     def delete_disk(self, disk_id):
         url_params = azure_helpers.parse_url(VOLUME_RESOURCE_ID,
                                              disk_id)
-        disk_name = url_params.get(VOLUME_NAME)
-        self.compute_client.disks.delete(self.resource_group, disk_name).wait()
+        disk_name = url_params.get(VOLUME_NAME, "")
+        self.compute_client.disks.begin_delete(self.resource_group, disk_name).wait()
 
     def update_disk_tags(self, disk_id, tags):
         url_params = azure_helpers.parse_url(VOLUME_RESOURCE_ID,
                                              disk_id)
-        disk_name = url_params.get(VOLUME_NAME)
-        return self.compute_client.disks.update(
+        disk_name = url_params.get(VOLUME_NAME, "")
+        return self.compute_client.disks.begin_update(
             self.resource_group,
             disk_name,
-            {'tags': tags},
-            raw=True
-        )
+            {'tags': tags}  # type: ignore
+        ).wait()
 
     def list_snapshots(self):
         return self.compute_client.snapshots. \
@@ -516,34 +519,43 @@ class AzureClient(object):
     def get_snapshot(self, snapshot_id):
         url_params = azure_helpers.parse_url(SNAPSHOT_RESOURCE_ID,
                                              snapshot_id)
-        snapshot_name = url_params.get(SNAPSHOT_NAME)
+        snapshot_name = url_params.get(SNAPSHOT_NAME, "")
         return self.compute_client.snapshots.get(self.resource_group,
                                                  snapshot_name)
 
-    def create_snapshot(self, snapshot_name, params):
-        return self.compute_client.snapshots.create_or_update(
+    def create_snapshot(self, snapshot_name, volume, tags):
+        snapshot = self.compute_client.snapshots.begin_create_or_update(
             self.resource_group,
             snapshot_name,
-            params
+            {
+                'location': volume.location,
+                'creation_data': {
+                    'create_option': 'Copy',
+                    'source_uri': volume.id
+                },
+                'tags': tags
+            }
         ).result()
 
+        self.update_snapshot_tags(snapshot.id, tags)
+        return snapshot
+
     def delete_snapshot(self, snapshot_id):
         url_params = azure_helpers.parse_url(SNAPSHOT_RESOURCE_ID,
                                              snapshot_id)
-        snapshot_name = url_params.get(SNAPSHOT_NAME)
-        self.compute_client.snapshots.delete(self.resource_group,
-                                             snapshot_name).wait()
+        snapshot_name = url_params.get(SNAPSHOT_NAME, "")
+        self.compute_client.snapshots.begin_delete(self.resource_group,
+                                                   snapshot_name).wait()
 
     def update_snapshot_tags(self, snapshot_id, tags):
         url_params = azure_helpers.parse_url(SNAPSHOT_RESOURCE_ID,
                                              snapshot_id)
-        snapshot_name = url_params.get(SNAPSHOT_NAME)
-        return self.compute_client.snapshots.update(
+        snapshot_name = url_params.get(SNAPSHOT_NAME, "")
+        return self.compute_client.snapshots.begin_update(
             self.resource_group,
             snapshot_name,
-            {'tags': tags},
-            raw=True
-        )
+            {'tags': tags}  # type: ignore
+        ).wait()
 
     def is_gallery_image(self, image_id):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
@@ -553,15 +565,14 @@ class AzureClient(object):
 
     def create_image(self, name, params):
         return self.compute_client.images. \
-            create_or_update(self.resource_group, name,
-                             params).result()
+            begin_create_or_update(self.resource_group, name, params).result()
 
     def delete_image(self, image_id):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
                                              image_id)
         if not self.is_gallery_image(image_id):
-            name = url_params.get(IMAGE_NAME)
-            self.compute_client.images.delete(self.resource_group, name).wait()
+            name = url_params.get(IMAGE_NAME, "")
+            self.compute_client.images.begin_delete(self.resource_group, name).wait()
 
     def list_images(self):
         azure_images = list(self.compute_client.images.
@@ -580,7 +591,7 @@ class AzureClient(object):
                                          sku=url_params['sku'],
                                          version=url_params['version'])
         else:
-            name = url_params.get(IMAGE_NAME)
+            name = url_params.get(IMAGE_NAME, "")
             return self.compute_client.images.get(self.resource_group, name)
 
     def update_image_tags(self, image_id, tags):
@@ -589,13 +600,13 @@ class AzureClient(object):
         if self.is_gallery_image(image_id):
             return True
         else:
-            name = url_params.get(IMAGE_NAME)
+            name = url_params.get(IMAGE_NAME, "")
             return self.compute_client.images. \
-                create_or_update(self.resource_group, name,
-                                 {
-                                     'tags': tags,
-                                     'location': self.region_name
-                                 }).result()
+                begin_create_or_update(self.resource_group, name,
+                                       {
+                                           'tags': tags,
+                                           'location': self.region_name
+                                       }).result()
 
     def list_vm_types(self):
         return self.compute_client.virtual_machine_sizes. \
@@ -608,28 +619,25 @@ class AzureClient(object):
     def get_network(self, network_id):
         url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID,
                                              network_id)
-        network_name = url_params.get(NETWORK_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
         return self.network_management_client.virtual_networks.get(
             self.resource_group, network_name)
 
     def create_network(self, name, params):
         return self.network_management_client.virtual_networks. \
-            create_or_update(self.resource_group,
-                             name,
-                             parameters=params).result()
+            begin_create_or_update(self.resource_group, name, parameters=params).result()
 
     def delete_network(self, network_id):
         url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
-        network_name = url_params.get(NETWORK_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
         return self.network_management_client.virtual_networks. \
-            delete(self.resource_group, network_name).wait()
+            begin_delete(self.resource_group, network_name).wait()
 
     def update_network_tags(self, network_id, tags):
         url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
-        network_name = url_params.get(NETWORK_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
         return self.network_management_client.virtual_networks. \
-            create_or_update(self.resource_group,
-                             network_name, tags).result()
+            begin_create_or_update(self.resource_group, network_name, tags).result()
 
     def get_network_id_for_subnet(self, subnet_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID, subnet_id)
@@ -640,23 +648,23 @@ class AzureClient(object):
 
     def list_subnets(self, network_id):
         url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
-        network_name = url_params.get(NETWORK_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
         return self.network_management_client.subnets. \
             list(self.resource_group, network_name)
 
     def get_subnet(self, subnet_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
                                              subnet_id)
-        network_name = url_params.get(NETWORK_NAME)
-        subnet_name = url_params.get(SUBNET_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
+        subnet_name = url_params.get(SUBNET_NAME, "")
         return self.network_management_client.subnets. \
             get(self.resource_group, network_name, subnet_name)
 
     def create_subnet(self, network_id, subnet_name, params):
         url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
-        network_name = url_params.get(NETWORK_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
         result_create = self.network_management_client \
-            .subnets.create_or_update(
+            .subnets.begin_create_or_update(
                 self.resource_group,
                 network_name,
                 subnet_name,
@@ -668,61 +676,59 @@ class AzureClient(object):
 
     def __if_subnet_in_use(e):
         # return True if the CloudError exception is due to subnet being in use
-        if isinstance(e, CloudError):
-            if e.error.error == "InUseSubnetCannotBeDeleted":
+        if isinstance(e, HttpResponseError):
+            if "InUseSubnetCannotBeDeleted" in e.message:
                 return True
         return False
 
-    @tenacity.retry(stop=tenacity.stop_after_attempt(5),
+    @tenacity.retry(stop=tenacity.stop.stop_after_attempt(5),
                     retry=tenacity.retry_if_exception(__if_subnet_in_use),
-                    wait=tenacity.wait_fixed(5),
+                    wait=tenacity.wait.wait_fixed(5),
                     reraise=True)
     def delete_subnet(self, subnet_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
                                              subnet_id)
-        network_name = url_params.get(NETWORK_NAME)
-        subnet_name = url_params.get(SUBNET_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
+        subnet_name = url_params.get(SUBNET_NAME, "")
 
         try:
             result_delete = self.network_management_client \
-                .subnets.delete(
+                .subnets.begin_delete(
                     self.resource_group,
                     network_name,
                     subnet_name
                 )
             result_delete.wait()
-        except CloudError as cloud_error:
+        except HttpResponseError as cloud_error:
             log.exception(cloud_error.message)
             raise cloud_error
 
     def create_floating_ip(self, public_ip_name, public_ip_parameters):
         return self.network_management_client.public_ip_addresses. \
-            create_or_update(self.resource_group,
-                             public_ip_name,
-                             public_ip_parameters).result()
+            begin_create_or_update(self.resource_group,
+                                   public_ip_name, public_ip_parameters).result()
 
     def get_floating_ip(self, public_ip_id):
         url_params = azure_helpers.parse_url(PUBLIC_IP_RESOURCE_ID,
                                              public_ip_id)
-        public_ip_name = url_params.get(PUBLIC_IP_NAME)
+        public_ip_name = url_params.get(PUBLIC_IP_NAME, "")
         return self.network_management_client. \
             public_ip_addresses.get(self.resource_group, public_ip_name)
 
     def delete_floating_ip(self, public_ip_id):
         url_params = azure_helpers.parse_url(PUBLIC_IP_RESOURCE_ID,
                                              public_ip_id)
-        public_ip_name = url_params.get(PUBLIC_IP_NAME)
+        public_ip_name = url_params.get(PUBLIC_IP_NAME, "")
         self.network_management_client. \
-            public_ip_addresses.delete(self.resource_group,
-                                       public_ip_name).wait()
+            public_ip_addresses.begin_delete(self.resource_group,
+                                             public_ip_name).wait()
 
     def update_fip_tags(self, fip_id, tags):
         url_params = azure_helpers.parse_url(PUBLIC_IP_RESOURCE_ID,
                                              fip_id)
-        fip_name = url_params.get(PUBLIC_IP_NAME)
+        fip_name = url_params.get(PUBLIC_IP_NAME, "")
         self.network_management_client.public_ip_addresses. \
-            create_or_update(self.resource_group,
-                             fip_name, tags).result()
+            begin_create_or_update(self.resource_group, fip_name, tags).result()
 
     def list_floating_ips(self):
         return self.network_management_client.public_ip_addresses.list(
@@ -736,21 +742,21 @@ class AzureClient(object):
     def restart_vm(self, vm_id):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
-        return self.compute_client.virtual_machines.restart(
+        vm_name = url_params.get(VM_NAME, "")
+        return self.compute_client.virtual_machines.begin_restart(
             self.resource_group, vm_name).wait()
 
     def delete_vm(self, vm_id):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
-        return self.compute_client.virtual_machines.delete(
+        vm_name = url_params.get(VM_NAME, "")
+        return self.compute_client.virtual_machines.begin_delete(
             self.resource_group, vm_name).wait()
 
     def get_vm(self, vm_id):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
+        vm_name = url_params.get(VM_NAME, "")
         return self.compute_client.virtual_machines.get(
             self.resource_group,
             vm_name,
@@ -759,69 +765,63 @@ class AzureClient(object):
 
     def create_vm(self, vm_name, params):
         return self.compute_client.virtual_machines. \
-            create_or_update(self.resource_group,
-                             vm_name, params).result()
+            begin_create_or_update(self.resource_group, vm_name, params).result()
 
     def update_vm(self, vm_id, params):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
+        vm_name = url_params.get(VM_NAME, "")
         return self.compute_client.virtual_machines. \
-            create_or_update(self.resource_group,
-                             vm_name, params, raw=True)
+            begin_create_or_update(self.resource_group, vm_name, params).wait()
 
     def deallocate_vm(self, vm_id):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
+        vm_name = url_params.get(VM_NAME, "")
         self.compute_client. \
-            virtual_machines.deallocate(self.resource_group,
-                                        vm_name).wait()
+            virtual_machines.begin_deallocate(self.resource_group, vm_name).wait()
 
     def generalize_vm(self, vm_id):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
+        vm_name = url_params.get(VM_NAME, "")
         self.compute_client.virtual_machines. \
             generalize(self.resource_group, vm_name)
 
     def start_vm(self, vm_id):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
+        vm_name = url_params.get(VM_NAME, "")
         self.compute_client.virtual_machines. \
-            start(self.resource_group,
-                  vm_name).wait()
+            begin_start(self.resource_group, vm_name).wait()
 
     def update_vm_tags(self, vm_id, tags):
         url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
                                              vm_id)
-        vm_name = url_params.get(VM_NAME)
+        vm_name = url_params.get(VM_NAME, "")
         self.compute_client.virtual_machines. \
-            create_or_update(self.resource_group,
-                             vm_name, tags).result()
+            begin_create_or_update(self.resource_group, vm_name, tags).result()
 
     def delete_nic(self, nic_id):
         nic_params = azure_helpers.\
             parse_url(NETWORK_INTERFACE_RESOURCE_ID, nic_id)
-        nic_name = nic_params.get(NETWORK_INTERFACE_NAME)
+        nic_name = nic_params.get(NETWORK_INTERFACE_NAME, "")
         self.network_management_client. \
-            network_interfaces.delete(self.resource_group,
-                                      nic_name).wait()
+            network_interfaces.begin_delete(self.resource_group, nic_name).wait()
 
     def get_nic(self, nic_id):
         nic_params = azure_helpers.\
             parse_url(NETWORK_INTERFACE_RESOURCE_ID, nic_id)
-        nic_name = nic_params.get(NETWORK_INTERFACE_NAME)
+        nic_name = nic_params.get(NETWORK_INTERFACE_NAME, "")
         return self.network_management_client. \
             network_interfaces.get(self.resource_group, nic_name)
 
     def update_nic(self, nic_id, params):
         nic_params = azure_helpers.\
             parse_url(NETWORK_INTERFACE_RESOURCE_ID, nic_id)
-        nic_name = nic_params.get(NETWORK_INTERFACE_NAME)
+        nic_name = nic_params.get(NETWORK_INTERFACE_NAME, "")
         async_nic_creation = self.network_management_client. \
-            network_interfaces.create_or_update(
+            network_interfaces.begin_create_or_update(
                 self.resource_group,
                 nic_name,
                 params
@@ -831,7 +831,7 @@ class AzureClient(object):
 
     def create_nic(self, nic_name, params):
         return self.network_management_client. \
-            network_interfaces.create_or_update(
+            network_interfaces.begin_create_or_update(
                 self.resource_group,
                 nic_name,
                 params
@@ -862,14 +862,14 @@ class AzureClient(object):
 
     def delete_route_table(self, route_table_name):
         self.network_management_client. \
-            route_tables.delete(self.resource_group, route_table_name
-                                ).wait()
+            route_tables.begin_delete(self.resource_group,
+                                      route_table_name).wait()
 
     def attach_subnet_to_route_table(self, subnet_id, route_table_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
                                              subnet_id)
-        network_name = url_params.get(NETWORK_NAME)
-        subnet_name = url_params.get(SUBNET_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
+        subnet_name = url_params.get(SUBNET_NAME, "")
 
         subnet_info = self.network_management_client.subnets.get(
             self.resource_group,
@@ -882,11 +882,11 @@ class AzureClient(object):
             }
 
             result_create = self.network_management_client. \
-                subnets.create_or_update(
+                subnets.begin_create_or_update(
                  self.resource_group,
                  network_name,
                  subnet_name,
-                 subnet_info)
+                 subnet_info)  # type: ignore
             subnet_info = result_create.result()
 
         return subnet_info
@@ -894,8 +894,8 @@ class AzureClient(object):
     def detach_subnet_to_route_table(self, subnet_id, route_table_id):
         url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
                                              subnet_id)
-        network_name = url_params.get(NETWORK_NAME)
-        subnet_name = url_params.get(SUBNET_NAME)
+        network_name = url_params.get(NETWORK_NAME, "")
+        subnet_name = url_params.get(SUBNET_NAME, "")
 
         subnet_info = self.network_management_client.subnets.get(
             self.resource_group,
@@ -907,11 +907,11 @@ class AzureClient(object):
             subnet_info.route_table = None
 
             result_create = self.network_management_client. \
-                subnets.create_or_update(
+                subnets.begin_create_or_update(
                  self.resource_group,
                  network_name,
                  subnet_name,
-                 subnet_info)
+                 subnet_info)  # type: ignore
             subnet_info = result_create.result()
 
         return subnet_info
@@ -923,17 +923,16 @@ class AzureClient(object):
     def get_route_table(self, router_id):
         url_params = azure_helpers.parse_url(ROUTER_RESOURCE_ID,
                                              router_id)
-        router_name = url_params.get(ROUTER_NAME)
+        router_name = url_params.get(ROUTER_NAME, "")
         return self.network_management_client. \
             route_tables.get(self.resource_group, router_name)
 
     def create_route_table(self, route_table_name, params):
         return self.network_management_client. \
-            route_tables.create_or_update(
+            route_tables.begin_create_or_update(
              self.resource_group,
              route_table_name, params).result()
 
     def update_route_table_tags(self, route_table_name, tags):
         self.network_management_client.route_tables. \
-            create_or_update(self.resource_group,
-                             route_table_name, tags).result()
+            begin_create_or_update(self.resource_group, route_table_name, tags).result()

cloudbridge/providers/azure/resources.py

@@ -2,51 +2,34 @@
 DataTypes used by this provider
 """
 import collections
+import io
 import logging
 
+import pysftp
+from cloudbridge.base.resources import (BaseAttachmentInfo, BaseBucket,
+                                        BaseBucketObject, BaseFloatingIP,
+                                        BaseInstance, BaseInternetGateway,
+                                        BaseKeyPair, BaseLaunchConfig,
+                                        BaseMachineImage, BaseNetwork,
+                                        BasePlacementZone, BaseRegion,
+                                        BaseRouter, BaseSnapshot, BaseSubnet,
+                                        BaseVMFirewall, BaseVMFirewallRule,
+                                        BaseVMType, BaseVolume)
+from cloudbridge.interfaces import InstanceState, VolumeState
+from cloudbridge.interfaces.resources import (Instance, MachineImageState,
+                                              NetworkState, RouterState,
+                                              SnapshotState, SubnetState,
+                                              TrafficDirection)
+
 from azure.common import AzureException
+from azure.core.exceptions import ResourceNotFoundError
 from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.network.models import NetworkSecurityGroup
 
-from msrestazure.azure_exceptions import CloudError
-
-import pysftp
-
-from cloudbridge.base.resources import BaseAttachmentInfo
-from cloudbridge.base.resources import BaseBucket
-from cloudbridge.base.resources import BaseBucketObject
-from cloudbridge.base.resources import BaseFloatingIP
-from cloudbridge.base.resources import BaseInstance
-from cloudbridge.base.resources import BaseInternetGateway
-from cloudbridge.base.resources import BaseKeyPair
-from cloudbridge.base.resources import BaseLaunchConfig
-from cloudbridge.base.resources import BaseMachineImage
-from cloudbridge.base.resources import BaseNetwork
-from cloudbridge.base.resources import BasePlacementZone
-from cloudbridge.base.resources import BaseRegion
-from cloudbridge.base.resources import BaseRouter
-from cloudbridge.base.resources import BaseSnapshot
-from cloudbridge.base.resources import BaseSubnet
-from cloudbridge.base.resources import BaseVMFirewall
-from cloudbridge.base.resources import BaseVMFirewallRule
-from cloudbridge.base.resources import BaseVMType
-from cloudbridge.base.resources import BaseVolume
-from cloudbridge.interfaces import InstanceState
-from cloudbridge.interfaces import VolumeState
-from cloudbridge.interfaces.resources import Instance
-from cloudbridge.interfaces.resources import MachineImageState
-from cloudbridge.interfaces.resources import NetworkState
-from cloudbridge.interfaces.resources import RouterState
-from cloudbridge.interfaces.resources import SnapshotState
-from cloudbridge.interfaces.resources import SubnetState
-from cloudbridge.interfaces.resources import TrafficDirection
-
 from . import helpers as azure_helpers
-from .subservices import AzureBucketObjectSubService
-from .subservices import AzureFloatingIPSubService
-from .subservices import AzureGatewaySubService
-from .subservices import AzureSubnetSubService
-from .subservices import AzureVMFirewallRuleSubService
+from .subservices import (AzureBucketObjectSubService,
+                          AzureFloatingIPSubService, AzureGatewaySubService,
+                          AzureSubnetSubService, AzureVMFirewallRuleSubService)
 
 log = logging.getLogger(__name__)
 
@@ -109,7 +92,7 @@ class AzureVMFirewall(BaseVMFirewall):
                 get_vm_firewall(self.id)
             if not self._vm_firewall.tags:
                 self._vm_firewall.tags = {}
-        except (CloudError, ValueError) as cloud_error:
+        except (ResourceNotFoundError, ValueError) as cloud_error:
             log.exception(cloud_error.message)
             # The security group no longer exists and cannot be refreshed.
 
@@ -177,25 +160,29 @@ class AzureVMFirewallRule(BaseVMFirewallRule):
 
 
 class AzureBucketObject(BaseBucketObject):
-    def __init__(self, provider, container, key):
+    def __init__(self, provider, container, blob_properties):
         super(AzureBucketObject, self).__init__(provider)
         self._container = container
-        self._key = key
+        self._blob_properties = blob_properties
+
+    @property
+    def _blob_client(self):
+        return self._container._bucket.get_blob_client(self.name)
 
     @property
     def id(self):
-        return self._key.name
+        return self._blob_properties.name
 
     @property
     def name(self):
-        return self._key.name
+        return self._blob_properties.name
 
     @property
     def size(self):
         """
         Get this object's size.
         """
-        return self._key.properties.content_length
+        return self._blob_properties.size
 
     @property
     def last_modified(self):
@@ -203,19 +190,39 @@ class AzureBucketObject(BaseBucketObject):
         """
         Get the date and time this object was last modified.
         """
-        return self._key.properties.last_modified. \
-            strftime("%Y-%m-%dT%H:%M:%S.%f")
+        return self._blob_properties.last_modified.strftime("%Y-%m-%dT%H:%M:%S.%f")
 
     def iter_content(self):
         """
         Returns this object's content as an
-        iterable.
+        iterable stream.
         """
-        content_stream = self._provider.azure_client. \
-            get_blob_content(self._container.id, self._key.name)
-        if content_stream:
-            content_stream.seek(0)
-        return content_stream
+
+        def iterable_to_stream(iterable):
+            class IterStream(io.RawIOBase):
+                def __init__(self):
+                    self.leftover = None
+
+                def readable(self):
+                    return True
+
+                def readinto(self, b):
+                    try:
+                        buffer_length = len(b)  # We're supposed to return at most this much
+                        chunk = self.leftover or next(iterable)
+                        output, self.leftover = chunk[:buffer_length], chunk[buffer_length:]
+                        b[:len(output)] = output
+                        return len(output)
+                    except StopIteration:
+                        return 0  # indicate EOF
+
+            return IterStream()
+
+        def blob_iterator():
+            for chunk in self._blob_client.download_blob().chunks():
+                yield chunk
+
+        return iterable_to_stream(blob_iterator())
 
     def upload(self, data):
         """
@@ -236,7 +243,7 @@ class AzureBucketObject(BaseBucketObject):
         """
         try:
             self._provider.azure_client.create_blob_from_file(
-                self._container.id, self.id, path)
+                self._container.name, self.name, path)
             return True
         except AzureException as azureEx:
             log.exception(azureEx)
@@ -249,19 +256,17 @@ class AzureBucketObject(BaseBucketObject):
         :rtype: bool
         :return: True if successful
         """
-        self._provider.azure_client.delete_blob(self._container.id,
-                                                self.id)
+        self._blob_client.delete_blob()
 
     def generate_url(self, expires_in):
         """
         Generate a URL to this object.
         """
         return self._provider.azure_client.get_blob_url(
-            self._container.id, self.id, expires_in)
+            self._container, self.name, expires_in)
 
     def refresh(self):
-        self._key = self._provider.azure_client.get_blob(
-            self._container.id, self._key.id)
+        pass
 
 
 class AzureBucket(BaseBucket):
@@ -272,14 +277,26 @@ class AzureBucket(BaseBucket):
 
     @property
     def id(self):
-        return self._bucket.name
+        try:
+            name = self._bucket.name
+        except AttributeError:
+            name = self._bucket.container_name
+        return name
 
     @property
     def name(self):
         """
         Get this bucket's name.
+
+        Due to changes in the Azure API, we can either received a
+        Container or a ContainerClient, Container has a name, but
+        the ContainerClient has a container_name
         """
-        return self._bucket.name
+        try:
+            name = self._bucket.name
+        except AttributeError:
+            name = self._bucket.container_name
+        return name
 
     def exists(self, name):
         """
@@ -452,7 +469,7 @@ class AzureVolume(BaseVolume):
             self._volume = self._provider.azure_client. \
                 get_disk(self.id)
             self._update_state()
-        except (CloudError, ValueError) as cloud_error:
+        except (ResourceNotFoundError, ValueError) as cloud_error:
             log.exception(cloud_error.message)
             # The volume no longer exists and cannot be refreshed.
             # set the state to unknown
@@ -548,7 +565,7 @@ class AzureSnapshot(BaseSnapshot):
             self._snapshot = self._provider.azure_client. \
                 get_snapshot(self.id)
             self._state = self._snapshot.provisioning_state
-        except (CloudError, ValueError) as cloud_error:
+        except (ResourceNotFoundError, ValueError) as cloud_error:
             log.exception(cloud_error.message)
             # The snapshot no longer exists and cannot be refreshed.
             # set the state to unknown
@@ -698,7 +715,7 @@ class AzureMachineImage(BaseMachineImage):
             try:
                 self._image = self._provider.azure_client.get_image(self.id)
                 self._state = self._image.provisioning_state
-            except CloudError as cloud_error:
+            except ResourceNotFoundError as cloud_error:
                 log.exception(cloud_error.message)
                 # image no longer exists
                 self._state = "unknown"
@@ -773,7 +790,7 @@ class AzureNetwork(BaseNetwork):
             self._network = self._provider.azure_client.\
                 get_network(self.id)
             self._state = self._network.provisioning_state
-        except (CloudError, ValueError) as cloud_error:
+        except (ResourceNotFoundError, ValueError) as cloud_error:
             log.exception(cloud_error.message)
             # The network no longer exists and cannot be refreshed.
             # set the state to unknown
@@ -987,7 +1004,7 @@ class AzureSubnet(BaseSubnet):
             self._subnet = self._provider.azure_client. \
                 get_subnet(self.id)
             self._state = self._subnet.provisioning_state
-        except (CloudError, ValueError) as cloud_error:
+        except (ResourceNotFoundError, ValueError) as cloud_error:
             log.exception(cloud_error.message)
             # The subnet no longer exists and cannot be refreshed.
             # set the state to unknown
@@ -1327,7 +1344,7 @@ class AzureInstance(BaseInstance):
             if not self._vm.tags:
                 self._vm.tags = {}
             self._update_state()
-        except (CloudError, ValueError) as cloud_error:
+        except (ResourceNotFoundError, ValueError) as cloud_error:
             log.exception(cloud_error.message)
             # The volume no longer exists and cannot be refreshed.
             # set the state to unknown

cloudbridge/providers/azure/services.py

@@ -2,63 +2,39 @@ import base64
 import logging
 import uuid
 
-from azure.common import AzureException
-from azure.mgmt.compute.models import DiskCreateOption
-
-from msrestazure.azure_exceptions import CloudError
-
 import cloudbridge.base.helpers as cb_helpers
 from cloudbridge.base.middleware import dispatch
-from cloudbridge.base.resources import ClientPagedResultList
-from cloudbridge.base.resources import ServerPagedResultList
-from cloudbridge.base.services import BaseBucketObjectService
-from cloudbridge.base.services import BaseBucketService
-from cloudbridge.base.services import BaseComputeService
-from cloudbridge.base.services import BaseFloatingIPService
-from cloudbridge.base.services import BaseGatewayService
-from cloudbridge.base.services import BaseImageService
-from cloudbridge.base.services import BaseInstanceService
-from cloudbridge.base.services import BaseKeyPairService
-from cloudbridge.base.services import BaseNetworkService
-from cloudbridge.base.services import BaseNetworkingService
-from cloudbridge.base.services import BaseRegionService
-from cloudbridge.base.services import BaseRouterService
-from cloudbridge.base.services import BaseSecurityService
-from cloudbridge.base.services import BaseSnapshotService
-from cloudbridge.base.services import BaseStorageService
-from cloudbridge.base.services import BaseSubnetService
-from cloudbridge.base.services import BaseVMFirewallRuleService
-from cloudbridge.base.services import BaseVMFirewallService
-from cloudbridge.base.services import BaseVMTypeService
-from cloudbridge.base.services import BaseVolumeService
-from cloudbridge.interfaces.exceptions import DuplicateResourceException
-from cloudbridge.interfaces.exceptions import InvalidParamException
-from cloudbridge.interfaces.exceptions import InvalidValueException
-from cloudbridge.interfaces.resources import MachineImage
-from cloudbridge.interfaces.resources import Network
-from cloudbridge.interfaces.resources import Snapshot
-from cloudbridge.interfaces.resources import TrafficDirection
-from cloudbridge.interfaces.resources import VMFirewall
-from cloudbridge.interfaces.resources import VMType
-from cloudbridge.interfaces.resources import Volume
-
-from .resources import AzureBucket
-from .resources import AzureBucketObject
-from .resources import AzureFloatingIP
-from .resources import AzureInstance
-from .resources import AzureInternetGateway
-from .resources import AzureKeyPair
-from .resources import AzureLaunchConfig
-from .resources import AzureMachineImage
-from .resources import AzureNetwork
-from .resources import AzureRegion
-from .resources import AzureRouter
-from .resources import AzureSnapshot
-from .resources import AzureSubnet
-from .resources import AzureVMFirewall
-from .resources import AzureVMFirewallRule
-from .resources import AzureVMType
-from .resources import AzureVolume
+from cloudbridge.base.resources import (ClientPagedResultList,
+                                        ServerPagedResultList)
+from cloudbridge.base.services import (BaseBucketObjectService,
+                                       BaseBucketService, BaseComputeService,
+                                       BaseFloatingIPService,
+                                       BaseGatewayService, BaseImageService,
+                                       BaseInstanceService, BaseKeyPairService,
+                                       BaseNetworkingService,
+                                       BaseNetworkService, BaseRegionService,
+                                       BaseRouterService, BaseSecurityService,
+                                       BaseSnapshotService, BaseStorageService,
+                                       BaseSubnetService,
+                                       BaseVMFirewallRuleService,
+                                       BaseVMFirewallService,
+                                       BaseVMTypeService, BaseVolumeService)
+from cloudbridge.interfaces.exceptions import (DuplicateResourceException,
+                                               InvalidParamException,
+                                               InvalidValueException)
+from cloudbridge.interfaces.resources import (MachineImage, Network, Snapshot,
+                                              TrafficDirection, VMFirewall,
+                                              VMType, Volume)
+from azure.core.exceptions import ResourceNotFoundError
+
+from azure.mgmt.compute.models import DiskCreateOption
+
+from .resources import (AzureBucket, AzureBucketObject, AzureFloatingIP,
+                        AzureInstance, AzureInternetGateway, AzureKeyPair,
+                        AzureLaunchConfig, AzureMachineImage, AzureNetwork,
+                        AzureRegion, AzureRouter, AzureSnapshot, AzureSubnet,
+                        AzureVMFirewall, AzureVMFirewallRule, AzureVMType,
+                        AzureVolume)
 
 log = logging.getLogger(__name__)
 
@@ -95,7 +71,7 @@ class AzureVMFirewallService(BaseVMFirewallService):
         try:
             fws = self.provider.azure_client.get_vm_firewall(vm_firewall_id)
             return AzureVMFirewall(self.provider, fws)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -258,7 +234,7 @@ class AzureKeyPairService(BaseKeyPairService):
             if key_pair:
                 return AzureKeyPair(self.provider, key_pair)
             return None
-        except AzureException as error:
+        except ResourceNotFoundError as error:
             log.debug("KeyPair %s was not found.", key_pair_id)
             log.debug(error)
             return None
@@ -365,7 +341,7 @@ class AzureVolumeService(BaseVolumeService):
         try:
             volume = self.provider.azure_client.get_disk(volume_id)
             return AzureVolume(self.provider, volume)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -457,7 +433,7 @@ class AzureSnapshotService(BaseSnapshotService):
         try:
             snapshot = self.provider.azure_client.get_snapshot(snapshot_id)
             return AzureSnapshot(self.provider, snapshot)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -499,18 +475,13 @@ class AzureSnapshotService(BaseSnapshotService):
         volume = (self.provider.storage.volumes.get(volume)
                   if isinstance(volume, str) else volume)
 
-        params = {
-            'location': self.provider.azure_client.region_name,
-            'creation_data': {
-                'create_option': DiskCreateOption.copy,
-                'source_uri': volume.resource_id
-            },
-            'disk_size_gb': volume.size,
-            'tags': tags
-        }
+        # We need to pass the Disk Object to create the snapshot
+        volume = volume._volume
+
+        azure_snap = self.provider.azure_client.create_snapshot(
+            snapshot_name, volume, tags
+        )
 
-        azure_snap = self.provider.azure_client.create_snapshot(snapshot_name,
-                                                                params)
         return AzureSnapshot(self.provider, azure_snap)
 
     @dispatch(event="provider.storage.snapshots.delete",
@@ -532,11 +503,10 @@ class AzureBucketService(BaseBucketService):
         Returns a bucket given its ID. Returns ``None`` if the bucket
         does not exist.
         """
-        try:
-            bucket = self.provider.azure_client.get_container(bucket_id)
+        bucket = self.provider.azure_client.get_container(bucket_id)
+        if bucket.exists():
             return AzureBucket(self.provider, bucket)
-        except AzureException as error:
-            log.exception(error)
+        else:
             return None
 
     @dispatch(event="provider.storage.buckets.list",
@@ -544,7 +514,7 @@ class AzureBucketService(BaseBucketService):
     def list(self, limit=None, marker=None):
         buckets = [AzureBucket(self.provider, bucket)
                    for bucket
-                   in self.provider.azure_client.list_containers()[0]]
+                   in self.provider.azure_client.list_containers()]
         return ClientPagedResultList(self.provider, buckets,
                                      limit=limit, marker=marker)
 
@@ -577,10 +547,10 @@ class AzureBucketObjectService(BaseBucketObjectService):
         Retrieve a given object from this bucket.
         """
         try:
-            obj = self.provider.azure_client.get_blob(bucket.name,
-                                                      object_id)
+            # pylint:disable=protected-access
+            obj = bucket._bucket.get_blob_client(object_id).get_blob_properties()
             return AzureBucketObject(self.provider, bucket, obj)
-        except AzureException as azureEx:
+        except ResourceNotFoundError as azureEx:
             log.exception(azureEx)
             return None
 
@@ -593,23 +563,22 @@ class AzureBucketObjectService(BaseBucketObjectService):
         """
         objects = [AzureBucketObject(self.provider, bucket, obj)
                    for obj in
-                   self.provider.azure_client.list_blobs(
-                       bucket.name, prefix=prefix)]
+                   bucket._bucket.list_blobs(name_starts_with=prefix)]
         return ClientPagedResultList(self.provider, objects,
                                      limit=limit, marker=marker)
 
     def find(self, bucket, **kwargs):
         obj_list = [AzureBucketObject(self.provider, bucket, obj)
                     for obj in
-                    self.provider.azure_client.list_blobs(bucket.name)]
+                    bucket._bucket.list_blobs()]
         filters = ['name']
         matches = cb_helpers.generic_find(filters, kwargs, obj_list)
         return ClientPagedResultList(self.provider, list(matches))
 
     def create(self, bucket, name):
-        self.provider.azure_client.create_blob_from_text(
-            bucket.name, name, '')
-        return self.get(bucket, name)
+        blob_client = bucket._bucket.get_blob_client(name)
+        blob_client.upload_blob('')
+        return AzureBucketObject(self.provider, bucket, blob_client.get_blob_properties())
 
 
 class AzureComputeService(BaseComputeService):
@@ -648,7 +617,7 @@ class AzureImageService(BaseImageService):
         try:
             image = self.provider.azure_client.get_image(image_id)
             return AzureMachineImage(self.provider, image)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -973,7 +942,7 @@ class AzureInstanceService(BaseInstanceService):
         try:
             vm = self.provider.azure_client.get_vm(instance_id)
             return AzureInstance(self.provider, vm)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -1120,7 +1089,7 @@ class AzureNetworkService(BaseNetworkService):
         try:
             network = self.provider.azure_client.get_network(network_id)
             return AzureNetwork(self.provider, network)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -1177,11 +1146,8 @@ class AzureSubnetService(BaseSubnetService):
                     result_list.extend(self.provider.azure_client.list_subnets(
                         net.id
                     ))
-                except CloudError as cloud_error:
-                    if "NotFound" in cloud_error.error.error:
-                        log.exception(cloud_error)
-                    else:
-                        raise cloud_error
+                except ResourceNotFoundError as not_found_error:
+                    log.exception(not_found_error)
         subnets = [AzureSubnet(self.provider, subnet)
                    for subnet in result_list]
 
@@ -1200,7 +1166,7 @@ class AzureSubnetService(BaseSubnetService):
             azure_subnet = self.provider.azure_client.get_subnet(subnet_id)
             return AzureSubnet(self.provider,
                                azure_subnet) if azure_subnet else None
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -1272,7 +1238,7 @@ class AzureRouterService(BaseRouterService):
         try:
             route = self.provider.azure_client.get_route_table(router_id)
             return AzureRouter(self.provider, route)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None
@@ -1364,7 +1330,7 @@ class AzureFloatingIPService(BaseFloatingIPService):
     def get(self, gateway, fip_id):
         try:
             az_ip = self.provider.azure_client.get_floating_ip(fip_id)
-        except (CloudError, InvalidValueException) as cloud_error:
+        except (ResourceNotFoundError, InvalidValueException) as cloud_error:
             # Azure raises the cloud error if the resource not available
             log.exception(cloud_error)
             return None

docs/topics/design_decisions.rst

@@ -55,7 +55,7 @@ Resource identification, naming, and labeling
   do (at least for some resources, such as vmfirewalls within a private
   network). Overall, consistency was challenging to achieve with resource
   naming. Therefore, it was decided that CloudBridge would continue to support
-  resource renaming to the best extent possible and balance between the
+  resource renaming to the best extent possible and strike a balance between the
   use of the resource name property and resource tags. However, because of the
   inconsistency in rename functionality across providers, using the rename
   capabilities within CloudBridge would lead to cloud-dependent code (Related to
@@ -129,7 +129,7 @@ Make providers single zone
   such as the one detailed above. Ultimately, it led to an impasse with GCP,
   which tended to require the zone for almost every operation and some of our
   methods were not geared to do so. Therefore, by making the provider zone
-  specific, we have removed a considerable amount of complexity from both the
+  specific, we have removed a considerable amount of complexity from the
   code, with no significant impact on usability, since operations generally
   tend to be confined to the same zone. Multi-zone operations now require
   multiple cloud provider instances.

setup.cfg

@@ -22,3 +22,4 @@ universal = 1
 
 [flake8]
 application_import_names = cloudbridge, tests
+max-line-length = 120

setup.py

@@ -6,8 +6,7 @@ import ast
 import os
 import re
 
-from setuptools import find_packages
-from setuptools import setup
+from setuptools import find_packages, setup
 
 # Cannot use "from cloudbridge import get_version" because that would try to
 # import the six package which may not be installed yet.
@@ -32,16 +31,16 @@ REQS_AWS = [
 # below are compatible with each other. List individual libraries instead
 # of using the azure umbrella package to speed up installation.
 REQS_AZURE = [
-    'msrest>=0.5.4,<0.6',
-    'msrestazure==0.5.0',
-    'azure-common==1.1.14',
-    'azure-mgmt-devtestlabs==2.2.0',
-    'azure-mgmt-resource==2.0.0',
-    'azure-mgmt-compute==4.0.1',
-    'azure-mgmt-network>=2.0.1,<=2.1',
-    'azure-mgmt-storage==2.0.0',
-    'azure-storage-blob==1.3.1',
-    'azure-cosmosdb-table==1.0.4',
+    'msrestazure==0.6.4',
+    'azure-identity==1.7.0',
+    'azure-common==1.1.27',
+    'azure-mgmt-devtestlabs==9.0.0',
+    'azure-mgmt-resource==19.0.0',
+    'azure-mgmt-compute==23.1.0',
+    'azure-mgmt-network==19.2.0',
+    'azure-mgmt-storage==19.0.0',
+    'azure-storage-blob==12.9.0',
+    'azure-cosmosdb-table==1.0.6',
     'pysftp==0.2.9'
 ]
 REQS_GCP = [

tests/helpers/__init__.py

@@ -105,7 +105,7 @@ TEST_DATA_CONFIG = {
         "image":
             cb_helpers.get_env('CB_IMAGE_AZURE',
                                'Canonical:UbuntuServer:16.04.0-LTS:latest'),
-        "vm_type": cb_helpers.get_env('CB_VM_TYPE_AZURE', 'Basic_A2'),
+        "vm_type": cb_helpers.get_env('CB_VM_TYPE_AZURE', 'Standard_A2_v2'),
         "placement": cb_helpers.get_env('CB_PLACEMENT_AZURE', 'eastus'),
         "placement_cfg_key": "azure_zone_name"
     }

tests/test_network_service.py

@@ -1,11 +1,8 @@
 from cloudbridge.base import helpers as cb_helpers
 from cloudbridge.base.resources import BaseNetwork
-from cloudbridge.interfaces.resources import FloatingIP
-from cloudbridge.interfaces.resources import Network
-from cloudbridge.interfaces.resources import NetworkState
-from cloudbridge.interfaces.resources import RouterState
-from cloudbridge.interfaces.resources import Subnet
-from cloudbridge.interfaces.resources import SubnetState
+from cloudbridge.interfaces.resources import (FloatingIP, Network,
+                                              NetworkState, RouterState,
+                                              Subnet, SubnetState)
 
 import tests.helpers as helpers
 from tests.helpers import ProviderTestBase

tox.ini

@@ -24,7 +24,7 @@ setenv =
 passenv =
     PYTHONUNBUFFERED
     aws: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY
-    azure: AZURE_SUBSCRIPTION_ID AZURE_CLIENT_ID AZURE_SECRET AZURE_TENANT AZURE_REGION_NAME AZURE_RESOURCE_GROUP AZURE_STORAGE_ACCOUNT AZURE_VM_DEFAULT_USER_NAME AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME
+    azure: CB_IMAGE_AZURE AZURE_SUBSCRIPTION_ID AZURE_CLIENT_ID AZURE_SECRET AZURE_TENANT AZURE_REGION_NAME AZURE_RESOURCE_GROUP AZURE_STORAGE_ACCOUNT AZURE_VM_DEFAULT_USER_NAME AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME
     gcp: CB_IMAGE_GCP CB_INSTANCE_TYPE_GCP CB_PLACEMENT_GCP GCP_DEFAULT_REGION GCP_DEFAULT_ZONE GCP_PROJECT_NAME GCP_SERVICE_CREDS_FILE GCP_SERVICE_CREDS_DICT
     openstack:  CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS OS_AUTH_URL OS_PASSWORD OS_PROJECT_NAME OS_TENANT_NAME OS_USERNAME OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME NOVA_SERVICE_NAME
     mock: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY