Change Resource Group and Storage Account Handling

Merge #136

.gitignore

@@ -65,3 +65,6 @@ launch.json
 settings.json
 run_nose.py
 *ipynb*
+
+# PyCharm
+.idea/

cloudbridge/cloud/providers/azure/azure_client.py

@@ -12,13 +12,14 @@ from azure.mgmt.resource.subscriptions import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
 from azure.storage.blob import BlobPermissions
 from azure.storage.blob import BlockBlobService
-
-from cloudbridge.cloud.interfaces.exceptions import WaitStateException
+from azure.storage.common import TokenCredential
 
 from msrestazure.azure_exceptions import CloudError
 
 import tenacity
 
+from cloudbridge.cloud.interfaces.exceptions import \
+    InvalidNameException, ProviderConnectionException, WaitStateException
 from . import helpers as azure_helpers
 
 log = logging.getLogger(__name__)
@@ -157,6 +158,7 @@ class AzureClient(object):
             tenant=config.get('azure_tenant')
         )
 
+        self._access_token = config.get('azure_access_token')
         self._resource_client = None
         self._storage_client = None
         self._network_management_client = None
@@ -165,6 +167,7 @@ class AzureClient(object):
         self._access_key_result = None
         self._block_blob_service = None
         self._table_service = None
+        self._storage_account = None
 
         log.debug("azure subscription : %s", self.subscription_id)
 
@@ -245,14 +248,22 @@ class AzureClient(object):
 
     @property
     def blob_service(self):
+        self._get_or_create_storage_account()
         if not self._block_blob_service:
-            self._block_blob_service = BlockBlobService(
-                self.storage_account,
-                self.access_key_result.keys[0].value)
+            if self._access_token:
+                token_credential = TokenCredential(self._access_token)
+                self._block_blob_service = BlockBlobService(
+                    account_name=self.storage_account,
+                    token_credential=token_credential)
+            else:
+                self._block_blob_service = BlockBlobService(
+                    account_name=self.storage_account,
+                    account_key=self.access_key_result.keys[0].value)
         return self._block_blob_service
 
     @property
     def table_service(self):
+        self._get_or_create_storage_account()
         if not self._table_service:
             self._table_service = TableService(
                 self.storage_account,
@@ -278,6 +289,65 @@ class AzureClient(object):
         return self.storage_client.storage_accounts. \
             create(self.resource_group, name.lower(), params).result()
 
+    # Create a storage account. To prevent a race condition, try
+    # to get or create at least twice
+    @tenacity.retry(stop=tenacity.stop_after_attempt(2),
+                    retry=tenacity.retry_if_exception_type(CloudError),
+                    reraise=True)
+    def _get_or_create_storage_account(self):
+        if self._storage_account:
+            return self._storage_account
+        else:
+            try:
+                self._storage_account = \
+                    self.get_storage_account(self.storage_account)
+            except CloudError as cloud_error:
+                if cloud_error.error.error == "ResourceNotFound":
+                    storage_account_params = {
+                        'sku': {
+                            'name': 'Standard_LRS'
+                        },
+                        'kind': 'storage',
+                        'location': self.region_name,
+                    }
+                    try:
+                        self._storage_account = \
+                            self.create_storage_account(self.storage_account,
+                                                        storage_account_params)
+                    except CloudError as cloud_error2:
+                        if cloud_error2.error.error == "AuthorizationFailed":
+                            mess = 'The following error was returned by ' \
+                                   'Azure:\n%s\n\nThis is likely because the' \
+                                   ' Role associated with the provided ' \
+                                   'credentials does not allow for Storage ' \
+                                   'Account creation.\nA Storage Account is ' \
+                                   'necessary in order to perform the ' \
+                                   'desired operation. You must either ' \
+                                   'provide an existing Storage Account name' \
+                                   ' as part of the configuration, or ' \
+                                   'elevate the associated Role.\nFor more ' \
+                                   'information on roles, see: https://docs.' \
+                                   'microsoft.com/en-us/azure/role-based-' \
+                                   'access-control/overview\n' % cloud_error2
+                            raise ProviderConnectionException(mess)
+
+                        elif cloud_error2.error.error == \
+                                "StorageAccountAlreadyTaken":
+                            mess = 'The following error was ' \
+                                   'returned by Azure:\n%s\n\n' \
+                                   'Note that Storage Account names must be ' \
+                                   'unique across Azure (not just in your ' \
+                                   'subscription).\nFor more information ' \
+                                   'see https://docs.microsoft.com/en-us/' \
+                                   'azure/azure-resource-manager/resource-' \
+                                   'manager-storage-account-name-errors\n' \
+                                   % cloud_error2
+                            raise InvalidNameException(mess)
+                        else:
+                            raise cloud_error2
+                else:
+                    raise cloud_error
+
     def list_locations(self):
         return self.subscription_client.subscriptions. \
             list_locations(self.subscription_id)
@@ -576,9 +646,7 @@ class AzureClient(object):
     def __if_subnet_in_use(e):
         # return True if the CloudError exception is due to subnet being in use
         if isinstance(e, CloudError):
-            error_message = e.message
-            if "Subnet" in error_message \
-                    and 'in use' in error_message:
+            if e.error.error == "InUseSubnetCannotBeDeleted":
                 return True
         return False
 

cloudbridge/cloud/providers/azure/provider.py

@@ -1,16 +1,18 @@
 import logging
 import os
+from uuid import uuid4
+
+from msrestazure.azure_exceptions import CloudError
+
+import tenacity
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 from cloudbridge.cloud.providers.azure.azure_client import AzureClient
 from cloudbridge.cloud.providers.azure.services \
     import AzureComputeService, AzureNetworkingService, \
     AzureSecurityService, AzureStorageService
 
-from msrestazure.azure_exceptions import CloudError
-
-import tenacity
-
 log = logging.getLogger(__name__)
 
 
@@ -32,6 +34,8 @@ class AzureCloudProvider(BaseCloudProvider):
             'azure_tenant', os.environ.get('AZURE_TENANT', None))
 
         # optional config values
+        self.access_token = self._get_config_value(
+            'azure_access_token', os.environ.get('AZURE_ACCESS_TOKEN', None))
         self.region_name = self._get_config_value(
             'azure_region_name', os.environ.get('AZURE_REGION_NAME',
                                                 'eastus'))
@@ -39,16 +43,12 @@ class AzureCloudProvider(BaseCloudProvider):
             'azure_resource_group', os.environ.get('AZURE_RESOURCE_GROUP',
                                                    'cloudbridge'))
         # Storage account name is limited to a max length of 24 alphanum chars
-        # and unique across an account. Yet, all our operations are tied to a
-        # resource group, making it impossible to use a storage account
-        # defined in a different resource group from the one used by the
-        # current session. With that, base the name of the storage account on
-        # the current resource group, up to 24 chars in length.
+        # and unique across all of Azure. Thus, a uuid is used to generate a
+        # unique name for the Storage Account
         self.storage_account = self._get_config_value(
             'azure_storage_account',
             os.environ.get(
-                'AZURE_STORAGE_ACCOUNT', 'storageacc' + ''.join(
-                    ch for ch in self.resource_group if ch.isalnum())[-12:]))
+                'AZURE_STORAGE_ACCOUNT', 'storageacc' + str(uuid4())[-12:]))
 
         self.vm_default_user_name = self._get_config_value(
             'azure_vm_default_user_name', os.environ.get
@@ -98,13 +98,17 @@ class AzureCloudProvider(BaseCloudProvider):
                 'azure_resource_group': self.resource_group,
                 'azure_storage_account': self.storage_account,
                 'azure_public_key_storage_table_name':
-                    self.public_key_storage_table_name
+                    self.public_key_storage_table_name,
+                'azure_access_token': self.access_token
             }
 
             self._azure_client = AzureClient(provider_config)
             self._initialize()
         return self._azure_client
 
+    @tenacity.retry(stop=tenacity.stop_after_attempt(2),
+                    retry=tenacity.retry_if_exception_type(CloudError),
+                    reraise=True)
     def _initialize(self):
         """
         Verifying that resource group and storage account exists
@@ -113,25 +117,31 @@ class AzureCloudProvider(BaseCloudProvider):
         """
         try:
             self._azure_client.get_resource_group(self.resource_group)
-        except CloudError:
-            resource_group_params = {'location': self.region_name}
-            self._azure_client.create_resource_group(self.resource_group,
-                                                     resource_group_params)
-        # Create a storage account. To prevent a race condition, try
-        # to get or create at least twice
-        self._get_or_create_storage_account()
-
-    @tenacity.retry(stop=tenacity.stop_after_attempt(2), reraise=True)
-    def _get_or_create_storage_account(self):
-        try:
-            return self._azure_client.get_storage_account(self.storage_account)
-        except CloudError:
-            storage_account_params = {
-                'sku': {
-                    'name': 'Standard_LRS'
-                },
-                'kind': 'storage',
-                'location': self.region_name,
-            }
-            self._azure_client.create_storage_account(self.storage_account,
-                                                      storage_account_params)
+
+        except CloudError as cloud_error:
+            if cloud_error.error.error == "ResourceGroupNotFound":
+                resource_group_params = {'location': self.region_name}
+                try:
+                    self._azure_client.\
+                        create_resource_group(self.resource_group,
+                                              resource_group_params)
+                except CloudError as cloud_error2:
+                    if cloud_error2.error.error == "AuthorizationFailed":
+                        mess = 'The following error was returned by Azure:\n' \
+                               '%s\n\nThis is likely because the Role' \
+                               'associated with the given credentials does ' \
+                               'not allow for Resource Group creation.\nA ' \
+                               'Resource Group is necessary to manage ' \
+                               'resources in Azure. You must either ' \
+                               'provide an existing Resource Group as part ' \
+                               'of the configuration, or elevate the ' \
+                               'associated role.\nFor more information on ' \
+                               'roles, see: https://docs.microsoft.com/' \
+                               'en-us/azure/role-based-access-control/' \
+                               'overview\n' % cloud_error2
+                        raise ProviderConnectionException(mess)
+                    else:
+                        raise cloud_error2
+
+            else:
+                raise cloud_error

cloudbridge/cloud/providers/azure/services.py

@@ -5,6 +5,8 @@ import uuid
 from azure.common import AzureException
 from azure.mgmt.compute.models import DiskCreateOption
 
+from msrestazure.azure_exceptions import CloudError
+
 import cloudbridge.cloud.base.helpers as cb_helpers
 from cloudbridge.cloud.base.resources import ClientPagedResultList, \
     ServerPagedResultList
@@ -19,9 +21,6 @@ from cloudbridge.cloud.interfaces.exceptions import \
     DuplicateResourceException, InvalidValueException
 from cloudbridge.cloud.interfaces.resources import MachineImage, \
     Network, PlacementZone, Snapshot, Subnet, VMFirewall, VMType, Volume
-
-from msrestazure.azure_exceptions import CloudError
-
 from . import helpers as azure_helpers
 from .resources import AzureBucket, \
     AzureInstance, AzureKeyPair, \
@@ -972,8 +971,7 @@ class AzureSubnetService(BaseSubnetService):
                         net.id
                     ))
                 except CloudError as cloud_error:
-                    message = cloud_error.message
-                    if "not found" in message and "virtualNetworks" in message:
+                    if cloud_error.error.error == "ResourceNotFound":
                         log.exception(cloud_error)
                     else:
                         raise cloud_error