Introducing Azure Marketplace Images with ID, replacing Gallery Image References

cloudbridge/cloud/providers/azure/azure_client.py

@@ -6,7 +6,6 @@ from azure.common import AzureConflictHttpError
 from azure.common.credentials import ServicePrincipalCredentials
 from azure.cosmosdb.table.tableservice import TableService
 from azure.mgmt.compute import ComputeManagementClient
-from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.marketplaceordering import MarketplaceOrderingAgreements
 from azure.mgmt.network import NetworkManagementClient
 from azure.mgmt.resource import ResourceManagementClient
@@ -18,6 +17,8 @@ from azure.storage.common import TokenCredential
 
 from msrestazure.azure_exceptions import CloudError
 
+from packaging import version as ver_pckg
+
 import tenacity
 
 from cloudbridge.cloud.interfaces.exceptions import \
@@ -32,6 +33,11 @@ IMAGE_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
                      '{resourceGroupName}/providers/Microsoft.Compute/'
                      'images/{imageName}',
                      '{imageName}',
+                     '/Subscriptions/{subscriptionId}/Providers/'
+                     'Microsoft.Compute/Locations/{locationName}'
+                     '/Publishers/{publisherName}/ArtifactTypes/VMImage/'
+                     'Offers/{offerName}/Skus/{skuName}/'
+                     'Versions/{versionName}',
                      '{publisher}:{offer}:{sku}:{version}']
 NETWORK_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
                        '{resourceGroupName}/providers/Microsoft.Network'
@@ -81,7 +87,12 @@ VOLUME_RESOURCE_ID = ['/subscriptions/{subscriptionId}/resourceGroups/'
                       'disks/{diskName}',
                       '{diskName}']
 
+RES_GROUP_NAME = 'resourceGroupName'
 IMAGE_NAME = 'imageName'
+IMAGE_PUBLISHER_NAME = 'publisherName'
+IMAGE_OFFER_NAME = 'offerName'
+IMAGE_SKU_NAME = 'skuName'
+IMAGE_VERSION_NAME = 'versionName'
 NETWORK_NAME = 'virtualNetworkName'
 NETWORK_INTERFACE_NAME = 'networkInterfaceName'
 PUBLIC_IP_NAME = 'publicIpAddressName'
@@ -96,62 +107,20 @@ VOLUME_NAME = 'diskName'
 # Listing possible somewhat through:
 # azure.mgmt.devtestlabs.operations.GalleryImageOperations
 gallery_image_references = \
-    [GalleryImageReference(publisher='Canonical',
-                           offer='UbuntuServer',
-                           sku='16.04.0-LTS',
-                           version='latest'),
-     GalleryImageReference(publisher='Canonical',
-                           offer='UbuntuServer',
-                           sku='14.04.5-LTS',
-                           version='latest'),
-     GalleryImageReference(publisher='OpenLogic',
-                           offer='CentOS',
-                           sku='7.5',
-                           version='latest'),
-     GalleryImageReference(publisher='OpenLogic',
-                           offer='CentOS',
-                           sku='6.9',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftWindowsServer',
-                           offer='WindowsServer',
-                           sku='2016-Nano-Server',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftWindowsServer',
-                           offer='WindowsServer',
-                           sku='2016-Datacenter',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftWindowsDesktop',
-                           offer='Windows-10',
-                           sku='rs4-pron',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftVisualStudio',
-                           offer='Windows',
-                           sku='Windows-10-N-x64',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftVisualStudio',
-                           offer='VisualStudio',
-                           sku='VS-2017-Ent-WS2016',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftSQLServer',
-                           offer='SQL2017-WS2016',
-                           sku='Web',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftSQLServer',
-                           offer='SQL2017-WS2016',
-                           sku='Standard',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftSQLServer',
-                           offer='SQL2017-WS2016',
-                           sku='SQLDEV',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftSQLServer',
-                           offer='SQL2017-WS2016',
-                           sku='Express',
-                           version='latest'),
-     GalleryImageReference(publisher='MicrosoftSQLServer',
-                           offer='SQL2017-WS2016',
-                           sku='Enterprise',
-                           version='latest')]
+    ['Canonical:UbuntuServer:16.04.0-LTS:latest',
+     'Canonical:UbuntuServer:14.04.5-LTS:latest',
+     'OpenLogic:CentOS:7.5:latest',
+     'OpenLogic:CentOS:6.9:latest',
+     'MicrosoftWindowsServer:WindowsServer:2016-Nano-Server:latest',
+     'MicrosoftWindowsServer:WindowsServer:2016-Datacenter:latest',
+     'MicrosoftWindowsDesktop:Windows-10:rs4-pron:latest',
+     'MicrosoftVisualStudio:Windows:Windows-10-N-x64:latest',
+     'MicrosoftVisualStudio:VisualStudio:VS-2017-Ent-WS2016:latest',
+     'MicrosoftSQLServer:SQL2017-WS2016:Web:latest',
+     'MicrosoftSQLServer:SQL2017-WS2016:Standard:latest',
+     'MicrosoftSQLServer:SQL2017-WS2016:SQLDEV:latest',
+     'MicrosoftSQLServer:SQL2017-WS2016:Express:latest',
+     'MicrosoftSQLServer:SQL2017-WS2016:Enterprise:latest']
 
 
 class AzureClient(object):
@@ -555,7 +524,7 @@ class AzureClient(object):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
                                              image_id)
         # If it is a gallery image, it will always have an offer
-        return 'offer' in url_params
+        return 'offer' in url_params or 'offerName' in url_params
 
     def create_image(self, name, params):
         return self.compute_client.images. \
@@ -575,11 +544,8 @@ class AzureClient(object):
         return azure_images
 
     def get_marketplace_agreement(self, publisher_id, offer_id, plan_id):
-        try:
-            return self.marketplace_agreement_client.marketplace_agreements\
+        return self.marketplace_agreement_client.marketplace_agreements\
                 .get(publisher_id, offer_id, plan_id)
-        except Exception as e:
-            return None
 
     def accept_marketplace_agreement(self, publisher_id, offer_id,
                                      plan_id, agr):
@@ -588,19 +554,40 @@ class AzureClient(object):
                                         publisher_id, offer_id, plan_id, agr)
 
     def list_gallery_refs(self):
-        return gallery_image_references
+        return [self.get_image(urn) for urn in gallery_image_references]
 
     def get_image(self, image_id):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
                                              image_id)
+        # ID is a URN reference or a Marketplace image ID
         if self.is_gallery_image(image_id):
-            return GalleryImageReference(publisher=url_params['publisher'],
-                                         offer=url_params['offer'],
-                                         sku=url_params['sku'],
-                                         version=url_params['version'])
+            # Image ID is a URN reference. Get marketplace object for this
+            # image
+            if 'offer' in url_params:
+                publisher = url_params['publisher']
+                offer = url_params['offer']
+                sku = url_params['sku']
+                version = url_params['version']
+                # To avoid forcing users to keep track of versions, we fetch
+                # the latest version when a URN is specified with version
+                # 'latest'
+                if 'latest' in version.lower():
+                    imgs = self.compute_client.\
+                                virtual_machine_images.list(
+                                    self.region_name, publisher, offer, sku)
+                    versions = [ver_pckg.parse(v.name) for v in imgs]
+
+                    # This is necessary to make sure the form remains the same
+                    # because str(package.version.parse('1.01.2')) == 1.1.2
+                    max_ind = versions.index(max(versions))
+                    version = imgs[max_ind].name
+
+            return self.compute_client.virtual_machine_images.get(
+                self.region_name, publisher, offer, sku, version)
         else:
             name = url_params.get(IMAGE_NAME)
-            return self.compute_client.images.get(self.resource_group, name)
+            res_group = url_params.get(RES_GROUP_NAME, self.resource_group)
+            return self.compute_client.images.get(res_group, name)
 
     def update_image_tags(self, image_id, tags):
         url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,

cloudbridge/cloud/providers/azure/helpers.py

@@ -53,14 +53,21 @@ def parse_url(template_urls, original_url):
     return resource_param
 
 
-def generate_urn(gallery_image):
+MARKETPLACE_ID = '/Subscriptions/{subscriptionId}/Providers/'\
+                 'Microsoft.Compute/Locations/{locationName}'\
+                 '/Publishers/{publisherName}/ArtifactTypes/VMImage/'\
+                 'Offers/{offerName}/Skus/{skuName}/'\
+                 'Versions/{versionName}'
+
+def generate_urn(image_id):
     """
-    This function takes an azure gallery image and outputs a corresponding URN
+    This function takes an azure marketplace image ID and outputs a
+    corresponding URN
     :param gallery_image: a GalleryImageReference object
     :return: URN as string
     """
-    reference_dict = gallery_image.as_dict()
-    return ':'.join([reference_dict['publisher'],
-                     reference_dict['offer'],
-                     reference_dict['sku'],
-                     reference_dict['version']])
+    url_params = parse_url([MARKETPLACE_ID], image_id)
+    return ':'.join([url_params['publisherName'],
+                     url_params['offerName'],
+                     url_params['skuName'],
+                     url_params['versionName']])

cloudbridge/cloud/providers/azure/resources.py

@@ -6,8 +6,8 @@ import logging
 from uuid import uuid4
 
 from azure.common import AzureException
-from azure.mgmt.devtestlabs.models import GalleryImageReference
 from azure.mgmt.network.models import NetworkSecurityGroup
+from azure.mgmt.compute.models import VirtualMachineImage
 
 from msrestazure.azure_exceptions import CloudError
 
@@ -692,7 +692,7 @@ class AzureMachineImage(BaseMachineImage):
         # Image can be either a dict for public image reference
         # or the Azure iamge object
         self._image = image
-        if isinstance(self._image, GalleryImageReference):
+        if isinstance(self._image, VirtualMachineImage):
             self._state = 'Succeeded'
         else:
             self._state = self._image.provisioning_state
@@ -707,28 +707,22 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``str``
         :return: ID for this instance as returned by the cloud middleware.
         """
-        if self.is_gallery_image:
-            return azure_helpers.generate_urn(self._image)
-        else:
-            return self._image.id
+        return self._image.id
 
     @property
     def name(self):
-        if self.is_gallery_image:
-            return azure_helpers.generate_urn(self._image)
+        if self.is_marketplace_image:
+            return azure_helpers.generate_urn(self._image.id)
         else:
             return self._image.name
 
     @property
     def resource_id(self):
-        if self.is_gallery_image:
-            return azure_helpers.generate_urn(self._image)
-        else:
-            return self._image.id
+        return self._image.id
 
     @property
     def label(self):
-        if self.is_gallery_image:
+        if self.is_marketplace_image:
             return azure_helpers.generate_urn(self._image)
         else:
             return self._image.tags.get('Label', None)
@@ -738,7 +732,7 @@ class AzureMachineImage(BaseMachineImage):
         """
         Set the image label when it is a private image.
         """
-        if not self.is_gallery_image:
+        if not self.is_marketplace_image:
             self.assert_valid_resource_label(value)
             self._image.tags.update(Label=value or "")
             self._provider.azure_client. \
@@ -752,9 +746,9 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``str``
         :return: Description for this image as returned by the cloud middleware
         """
-        if self.is_gallery_image:
-            return 'Public gallery image from the Azure Marketplace: '\
-                    + self.name
+        if self.is_marketplace_image:
+            return \
+                'Public image from the Azure Marketplace: {}'.format(self.name)
         else:
             return self._image.tags.get('Description', None)
 
@@ -763,7 +757,7 @@ class AzureMachineImage(BaseMachineImage):
         """
         Set the image description.
         """
-        if not self.is_gallery_image:
+        if not self.is_marketplace_image:
             self._image.tags.update(Description=value or "")
             self._provider.azure_client. \
                 update_image_tags(self.id, self._image.tags)
@@ -779,7 +773,7 @@ class AzureMachineImage(BaseMachineImage):
         :rtype: ``int``
         :return: The minimum disk size needed by this image
         """
-        if self.is_gallery_image:
+        if self.is_marketplace_image:
             return 0
         else:
             return self._image.storage_profile.os_disk.disk_size_gb or 0
@@ -788,34 +782,39 @@ class AzureMachineImage(BaseMachineImage):
         """
         Delete this image
         """
-        if not self.is_gallery_image:
+        if not self.is_marketplace_image:
             self._provider.azure_client.delete_image(self.id)
 
     @property
     def state(self):
-        if self.is_gallery_image:
+        if self.is_marketplace_image:
             return MachineImageState.AVAILABLE
         else:
             return AzureMachineImage.IMAGE_STATE_MAP.get(
                 self._state, MachineImageState.UNKNOWN)
 
     @property
-    def is_gallery_image(self):
+    def is_marketplace_image(self):
         """
         Returns true if the image is a public reference and false if it
         is a private image in the resource group.
         """
-        return isinstance(self._image, GalleryImageReference)
+        return isinstance(self._image, VirtualMachineImage)
 
     def refresh(self):
         """
         Refreshes the state of this instance by re-querying the cloud provider
         for its latest state.
         """
-        if not self.is_gallery_image:
+        if not self.is_marketplace_image:
             try:
                 self._image = self._provider.azure_client.get_image(self.id)
-                self._state = self._image.provisioning_state
+                if isinstance(self._image, VirtualMachineImage):
+                    self._state = 'Succeeded'
+                else:
+                    self._state = self._image.provisioning_state
+                    if not self._image.tags:
+                        self._image.tags = {}
             except CloudError as cloud_error:
                 log.exception(cloud_error.message)
                 # image no longer exists