Upgrade Azure/OpenStack SDKs and replace abandoned deps

Bump version bounds for azure-mgmt-* (compute<39, network<31,
resource<26, storage<25), openstacksdk<5, neutronclient<13,
novaclient<20, keystoneclient<7.

Replace abandoned libraries that block modern Python/SDK installs:
- msrestazure.CloudError -> azure.core.exceptions (HttpResponseError,
  ResourceNotFoundError).
- azure-cosmosdb-table -> azure-data-tables (TableServiceClient with
  AzureNamedKeyCredential; rewrite public-key CRUD for new dict-style
  entities and ItemPaged pagination).
- pysftp -> paramiko (SSHClient directly, preserving existing trust
  behavior for one-shot VM deprovision).

Move SubscriptionClient import to azure-mgmt-subscription;
azure.mgmt.resource.subscriptions was removed in resource SDK 25.x.

Bump tox env from py3.10 to py3.13

cloudbridge/providers/azure/azure_client.py

@@ -7,17 +7,18 @@ from cloudbridge.interfaces.exceptions import (DuplicateResourceException,
                                                ProviderConnectionException,
                                                WaitStateException)
 
+from azure.core.credentials import AzureNamedKeyCredential
 from azure.core.exceptions import (ClientAuthenticationError,
                                    HttpResponseError, ResourceExistsError,
                                    ResourceNotFoundError)
-from azure.cosmosdb.table.tableservice import TableService
+from azure.data.tables import TableServiceClient
 from azure.identity import ClientSecretCredential
 from azure.mgmt.compute import ComputeManagementClient
 from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.network import NetworkManagementClient
 from azure.mgmt.resource import ResourceManagementClient
-from azure.mgmt.resource.subscriptions import SubscriptionClient
 from azure.mgmt.storage import StorageManagementClient
+from azure.mgmt.subscription import SubscriptionClient
 from azure.storage.blob import (BlobSasPermissions, BlobServiceClient,
                                 generate_blob_sas)
 
@@ -172,7 +173,8 @@ class AzureClient(object):
         self._compute_client = None
         self._access_key_result = None
         self._block_blob_service = None
-        self._table_service = None
+        self._table_service_client = None
+        self._public_key_table_client = None
         self._storage_account = None
 
         log.debug("azure subscription : %s", self.subscription_id)
@@ -273,15 +275,18 @@ class AzureClient(object):
     @property
     def table_service(self):
         self._get_or_create_storage_account()
-        if not self._table_service:
-            self._table_service = TableService(
+        if not self._table_service_client:
+            credential = AzureNamedKeyCredential(
                 self.storage_account,
                 self.access_key_result.keys[0].value)
-        if not self._table_service. \
-                exists(table_name=self.public_key_storage_table_name):
-            self._table_service.create_table(
-                self.public_key_storage_table_name)
-        return self._table_service
+            self._table_service_client = TableServiceClient(
+                endpoint=f"https://{self.storage_account}.table.core.windows.net/",
+                credential=credential)
+        if not self._public_key_table_client:
+            self._public_key_table_client = \
+                self._table_service_client.create_table_if_not_exists(
+                    table_name=self.public_key_storage_table_name)
+        return self._public_key_table_client
 
     def blob_client(self, container_name, blob_name):
         return self.blob_service.get_blob_client(container=container_name, blob=blob_name)
@@ -845,27 +850,29 @@ class AzureClient(object):
             ).result()
 
     def create_public_key(self, entity):
-        return self.table_service. \
-            insert_or_replace_entity(self.public_key_storage_table_name,
-                                     entity)
+        return self.table_service.upsert_entity(entity)
 
     def get_public_key(self, name):
-        entities = self.table_service. \
-            query_entities(self.public_key_storage_table_name,
-                           "Name eq '{0}'".format(name), num_results=1)
-
-        return entities.items[0] if len(entities.items) > 0 else None
+        entities = list(self.table_service.query_entities(
+            query_filter="Name eq '{0}'".format(name),
+            results_per_page=1))
+        return entities[0] if entities else None
 
     def delete_public_key(self, entity):
-        self.table_service.delete_entity(self.public_key_storage_table_name,
-                                         entity.PartitionKey, entity.RowKey)
+        self.table_service.delete_entity(
+            partition_key=entity['PartitionKey'],
+            row_key=entity['RowKey'])
 
     def list_public_keys(self, partition_key, limit=None, marker=None):
-        entities = self.table_service. \
-            query_entities(self.public_key_storage_table_name,
-                           "PartitionKey eq '{0}'".format(partition_key),
-                           marker=marker, num_results=limit)
-        return (entities.items, entities.next_marker)
+        pager = self.table_service.query_entities(
+            query_filter="PartitionKey eq '{0}'".format(partition_key),
+            results_per_page=limit).by_page(continuation_token=marker)
+        try:
+            page = next(pager)
+        except StopIteration:
+            return ([], None)
+        items = list(page)
+        return (items, pager.continuation_token)
 
     def delete_route_table(self, route_table_name):
         self.network_management_client. \

cloudbridge/providers/azure/provider.py

@@ -1,9 +1,10 @@
 import logging
 import uuid
 
-from deprecation import deprecated
+from azure.core.exceptions import HttpResponseError
+from azure.core.exceptions import ResourceNotFoundError
 
-from msrestazure.azure_exceptions import CloudError
+from deprecation import deprecated
 
 import tenacity
 
@@ -143,7 +144,7 @@ class AzureCloudProvider(BaseCloudProvider):
         return self._azure_client
 
     @tenacity.retry(stop=tenacity.stop_after_attempt(2),
-                    retry=tenacity.retry_if_exception_type(CloudError),
+                    retry=tenacity.retry_if_exception_type(HttpResponseError),
                     reraise=True)
     def _initialize(self):
         """
@@ -154,33 +155,30 @@ class AzureCloudProvider(BaseCloudProvider):
         try:
             self._azure_client.get_resource_group(self.resource_group)
 
-        except CloudError as cloud_error:
-            if cloud_error.error.error == "ResourceGroupNotFound":
-                resource_group_params = {'location': self.region_name}
-                try:
-                    self._azure_client.\
-                        create_resource_group(self.resource_group,
-                                              resource_group_params)
-                except CloudError as cloud_error2:  # pragma: no cover
-                    if cloud_error2.error.error == "AuthorizationFailed":
-                        mess = 'The following error was returned by Azure:\n' \
-                               '%s\n\nThis is likely because the Role' \
-                               'associated with the given credentials does ' \
-                               'not allow for Resource Group creation.\nA ' \
-                               'Resource Group is necessary to manage ' \
-                               'resources in Azure. You must either ' \
-                               'provide an existing Resource Group as part ' \
-                               'of the configuration, or elevate the ' \
-                               'associated role.\nFor more information on ' \
-                               'roles, see: https://docs.microsoft.com/' \
-                               'en-us/azure/role-based-access-control/' \
-                               'overview\n' % cloud_error2
-                        raise ProviderConnectionException(mess)
-                    else:
-                        raise cloud_error2
-
-            else:
-                raise cloud_error
+        except ResourceNotFoundError:
+            resource_group_params = {'location': self.region_name}
+            try:
+                self._azure_client.\
+                    create_resource_group(self.resource_group,
+                                          resource_group_params)
+            except HttpResponseError as cloud_error2:  # pragma: no cover
+                if getattr(cloud_error2, 'error', None) and \
+                        cloud_error2.error.code == "AuthorizationFailed":
+                    mess = 'The following error was returned by Azure:\n' \
+                           '%s\n\nThis is likely because the Role' \
+                           'associated with the given credentials does ' \
+                           'not allow for Resource Group creation.\nA ' \
+                           'Resource Group is necessary to manage ' \
+                           'resources in Azure. You must either ' \
+                           'provide an existing Resource Group as part ' \
+                           'of the configuration, or elevate the ' \
+                           'associated role.\nFor more information on ' \
+                           'roles, see: https://docs.microsoft.com/' \
+                           'en-us/azure/role-based-access-control/' \
+                           'overview\n' % cloud_error2
+                    raise ProviderConnectionException(mess)
+                else:
+                    raise cloud_error2
 
         """
         Verify that resource group used for network exists,
@@ -188,8 +186,5 @@ class AzureCloudProvider(BaseCloudProvider):
         """
         try:
             self._azure_client.get_resource_group(self.networking_resource_group)
-        except CloudError as cloud_error:
-            if cloud_error.error.error == "ResourceGroupNotFound":
-                self.networking_resource_group = self.resource_group
-            else:
-                raise cloud_error
+        except ResourceNotFoundError:
+            self.networking_resource_group = self.resource_group

cloudbridge/providers/azure/resources.py

@@ -5,7 +5,7 @@ import collections
 import io
 import logging
 
-import pysftp
+import paramiko
 from cloudbridge.base.resources import (BaseAttachmentInfo, BaseBucket,
                                         BaseBucketObject, BaseFloatingIP,
                                         BaseInstance, BaseInternetGateway,
@@ -1205,7 +1205,7 @@ class AzureInstance(BaseInstance):
         chines/linux/capture-image. In azure, we need to deprovision the VM
         before capturing.
         To deprovision, login to the VM and execute the `waagent deprovision`
-        command. To do this programmatically, use pysftp to ssh into the VM
+        command. To do this programmatically, use paramiko to ssh into the VM
         and executing deprovision command. To SSH into the VM programmatically
         however, we need to pass private key file path, so we have modified the
         CloudBridge interface to pass the private key file path
@@ -1236,16 +1236,18 @@ class AzureInstance(BaseInstance):
         return AzureMachineImage(self._provider, image)
 
     def _deprovision(self, private_key_path):
-        cnopts = pysftp.CnOpts()
-        cnopts.hostkeys = None
-        if private_key_path:
-            with pysftp.\
-                    Connection(self.public_ips[0],
-                               username=self._provider.vm_default_user_name,
-                               cnopts=cnopts,
-                               private_key=private_key_path) as sftp:
-                sftp.execute('sudo waagent -deprovision -force')
-                sftp.close()
+        if not private_key_path:
+            return
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            client.connect(
+                hostname=self.public_ips[0],
+                username=self._provider.vm_default_user_name,
+                key_filename=private_key_path)
+            client.exec_command('sudo waagent -deprovision -force')
+        finally:
+            client.close()
 
     def add_floating_ip(self, floating_ip):
         """
@@ -1432,11 +1434,11 @@ class AzureKeyPair(BaseKeyPair):
 
     @property
     def id(self):
-        return self._key_pair.Name
+        return self._key_pair['Name']
 
     @property
     def name(self):
-        return self._key_pair.Name
+        return self._key_pair['Name']
 
 
 class AzureRouter(BaseRouter):

cloudbridge/providers/azure/services.py

@@ -877,7 +877,7 @@ class AzureInstanceService(BaseInstanceService):
                             "path":
                                 "/home/{}/.ssh/authorized_keys".format(
                                         self.provider.vm_default_user_name),
-                                "key_data": key_pair._key_pair.Key
+                                "key_data": key_pair._key_pair['Key']
                         }]
                     }
                 }

setup.py

@@ -31,27 +31,28 @@ REQS_AWS = [
 # below are compatible with each other. List individual libraries instead
 # of using the azure umbrella package to speed up installation.
 REQS_AZURE = [
-    'msrestazure<1.0.0',
     'azure-identity<2.0.0',
     'azure-common<2.0.0',
+    'azure-core<2.0.0',
     'azure-mgmt-devtestlabs<10.0.0',
-    'azure-mgmt-resource<24.0.0',
-    'azure-mgmt-compute>=27.2.0,<31.0.0',
-    'azure-mgmt-network<26.0.0',
-    'azure-mgmt-storage<22.0.0',
+    'azure-mgmt-resource<26.0.0',
+    'azure-mgmt-subscription<4.0.0',
+    'azure-mgmt-compute>=27.2.0,<39.0.0',
+    'azure-mgmt-network<31.0.0',
+    'azure-mgmt-storage<25.0.0',
     'azure-storage-blob<13.0.0',
-    'azure-cosmosdb-table<2.0.0',
-    'pysftp<1.0.0'
+    'azure-data-tables<13.0.0',
+    'paramiko<6.0.0'
 ]
 REQS_GCP = [
     'google-api-python-client>=2.0,<3.0.0'
 ]
 REQS_OPENSTACK = [
-    'openstacksdk>=0.12.0,<4.0.0',
-    'python-novaclient>=7.0.0,<19.0',
+    'openstacksdk>=0.12.0,<5.0.0',
+    'python-novaclient>=7.0.0,<20.0',
     'python-swiftclient>=3.2.0,<5.0',
-    'python-neutronclient>=6.0.0,<12.0',
-    'python-keystoneclient>=3.13.0,<6.0'
+    'python-neutronclient>=6.0.0,<13.0',
+    'python-keystoneclient>=3.13.0,<7.0'
 ]
 REQS_FULL = REQS_AWS + REQS_GCP + REQS_OPENSTACK + REQS_AZURE
 # httpretty is required with/for moto 1.0.0 or AWS tests fail

tox.ini

@@ -6,7 +6,7 @@
 # running the tests.
 
 [tox]
-envlist = {py3.10,pypy}-{aws,azure,gcp,openstack,mock},lint
+envlist = {py3.13,pypy}-{aws,azure,gcp,openstack,mock},lint
 
 [testenv]
 commands = # see setup.cfg for options sent to pytest and coverage