Merge pull request #105 from baizhang/gce

Add support for GCE resource get method to use either id (resource URL) or short name

cloudbridge/cloud/providers/gce/provider.py

@@ -8,8 +8,10 @@ import re
 import time
 from string import Template
 
+import cloudbridge as cb
 from cloudbridge.cloud.base import BaseCloudProvider
 
+import googleapiclient
 from googleapiclient import discovery
 
 from oauth2client.client import GoogleCredentials
@@ -55,8 +57,13 @@ class GCPResourceUrl(object):
 
 class GCPResources(object):
 
-    def __init__(self, connection):
+    def __init__(self, connection, project_name, region_name, default_zone):
         self._connection = connection
+        self._parameter_defaults = {
+            'project': project_name,
+            'region': region_name,
+            'zone': default_zone,
+        }
 
         # Resource descriptions are already pulled into the internal
         # _resourceDesc field of the connection.
@@ -157,6 +164,26 @@ class GCPResources(object):
                 out.parameters[parameter] = m.group(index + 1)
             return out
 
+    def get_resource_url_with_default(self, resource, url_or_name):
+        """
+        Build a GCPResourceUrl from a service's name and resource url or name.
+        If the url_or_name is a valid GCP resource URL, then we build the
+        GCPResourceUrl object by parsing this URL. If the url_or_name is its
+        short name, then we build the GCPResourceUrl object by constructing
+        the resource URL with default project, region, zone values.
+        """
+        # If url_or_name is a valid GCP resource URL, then parse it.
+        if url_or_name.startswith(self._root_url):
+            return self.parse_url(url_or_name)
+        # Otherwise, construct resource URL with default values.
+        if resource not in self._resources:
+            return None
+        parsed_url = GCPResourceUrl(resource, self._connection)
+        for key in self._resources[resource]['parameters']:
+            parsed_url.parameters[key] = self._parameter_defaults.get(
+                key, url_or_name)
+        return parsed_url
+
 
 class GCECloudProvider(BaseCloudProvider):
 
@@ -198,8 +225,12 @@ class GCECloudProvider(BaseCloudProvider):
         self._block_store = GCEBlockStoreService(self)
         self._object_store = GCSObjectStoreService(self)
 
-        self._compute_resources = GCPResources(self.gce_compute)
-        self._storage_resources = GCPResources(self.gcp_storage)
+        self._compute_resources = GCPResources(
+            self.gce_compute, self.project_name, self.region_name,
+            self.default_zone)
+        self._storage_resources = GCPResources(
+            self.gcp_storage, self.project_name, self.region_name,
+            self.default_zone)
 
     @property
     def compute(self):
@@ -270,3 +301,18 @@ class GCECloudProvider(BaseCloudProvider):
     def parse_url(self, url):
         out = self._compute_resources.parse_url(url)
         return out if out else self._storage_resources.parse_url(url)
+
+    def get_resource(self, resource, url_or_name):
+        resource_url = (
+            self._compute_resources.get_resource_url_with_default(
+                resource, url_or_name) or
+            self._storage_resources.get_resource_url_with_default(
+                resource, url_or_name))
+        if resource_url is None:
+            return None
+        try:
+            return resource_url.get_resource()
+        except googleapiclient.errors.HttpError as http_error:
+            cb.log.warning(
+                "googleapiclient.errors.HttpError: {0}".format(http_error))
+            return None

cloudbridge/cloud/providers/gce/resources.py

@@ -152,7 +152,7 @@ class GCEPlacementZone(BasePlacementZone):
         :rtype: ``str``
         :return: ID for this zone as returned by the cloud middleware.
         """
-        return self._gce_zone
+        return self._gce_zone.get('selfLink')
 
     @property
     def name(self):
@@ -161,7 +161,7 @@ class GCEPlacementZone(BasePlacementZone):
         :rtype: ``str``
         :return: Name for this zone as returned by the cloud middleware.
         """
-        return self._gce_zone
+        return self._gce_zone.get('name')
 
     @property
     def region_name(self):
@@ -181,16 +181,11 @@ class GCERegion(BaseRegion):
 
     @property
     def id(self):
-        # In GCE API, region has an 'id' property, whose values are '1220',
-        # '1100', '1000', '1230', etc. Here we use 'name' property (such
-        # as 'asia-east1', 'europe-west1', 'us-central1', 'us-east1') as
-        # 'id' to represent the region for the consistency with AWS
-        # implementation and ease of use.
-        return self._gce_region['name']
+        return self._gce_region.get('selfLink')
 
     @property
     def name(self):
-        return self._gce_region['name']
+        return self._gce_region.get('name')
 
     @property
     def zones(self):
@@ -204,7 +199,7 @@ class GCERegion(BaseRegion):
                               .execute())
         zones = [zone for zone in zones_response['items']
                  if zone['region'] == self._gce_region['selfLink']]
-        return [GCEPlacementZone(self._provider, zone['name'], self.name)
+        return [GCEPlacementZone(self._provider, zone, self.name)
                 for zone in zones]
 
 
@@ -825,11 +820,7 @@ class GCEInstance(BaseInstance):
         """
         Get the instance type name.
         """
-        machine_type_uri = self._gce_instance.get('machineType')
-        if machine_type_uri is None:
-            return None
-        parsed_uri = self._provider.parse_url(machine_type_uri)
-        return parsed_uri.parameters['machineType']
+        return self._gce_instance.get('machineType')
 
     @property
     def instance_type(self):
@@ -905,10 +896,7 @@ class GCEInstance(BaseInstance):
         """
         Get the placement zone id where this instance is running.
         """
-        zone_uri = self._gce_instance.get('zone')
-        if zone_uri is None:
-            return None
-        return self._provider.parse_url(zone_uri).parameters['zone']
+        return self._gce_instance.get('zone')
 
     @property
     def security_groups(self):
@@ -1203,7 +1191,7 @@ class GCENetwork(BaseNetwork):
 
     @property
     def id(self):
-        return self._network['id']
+        return self._network['selfLink']
 
     @property
     def name(self):
@@ -1293,7 +1281,7 @@ class GCEFloatingIP(BaseFloatingIP):
 
     @property
     def id(self):
-        return self._ip['id']
+        return self._ip['selfLink']
 
     @property
     def region(self):
@@ -1345,7 +1333,7 @@ class GCERouter(BaseRouter):
 
     @property
     def id(self):
-        return self._router['id']
+        return self._router['selfLink']
 
     @property
     def name(self):
@@ -1364,7 +1352,7 @@ class GCERouter(BaseRouter):
     def network_id(self):
         parsed_url = self._provider.parse_url(self._router['network'])
         network = parsed_url.get_resource()
-        return network['id']
+        return network['selfLink']
 
     def delete(self):
         response = (self._provider
@@ -1400,7 +1388,7 @@ class GCESubnet(BaseSubnet):
 
     @property
     def id(self):
-        return self._subnet['id']
+        return self._subnet['selfLink']
 
     @property
     def name(self):
@@ -1422,7 +1410,8 @@ class GCESubnet(BaseSubnet):
 
     @property
     def network_id(self):
-        return self._provider.parse_url(self.network_url).get_resource()['id']
+        return self._provider.parse_url(
+            self.network_url).get_resource()['selfLink']
 
     @property
     def region(self):
@@ -1743,7 +1732,7 @@ class GCSObject(BaseBucketObject):
 
     @property
     def id(self):
-        return self._obj['id']
+        return self._obj['selfLink']
 
     @property
     def name(self):
@@ -1807,7 +1796,7 @@ class GCSBucket(BaseBucket):
 
     @property
     def id(self):
-        return self._bucket['id']
+        return self._bucket['selfLink']
 
     @property
     def name(self):

cloudbridge/cloud/providers/gce/services.py

@@ -289,10 +289,8 @@ class GCEInstanceTypesService(BaseInstanceTypesService):
         return response['items']
 
     def get(self, instance_type_id):
-        for inst_type in self.instance_data:
-            if inst_type.get('id') == instance_type_id:
-                return GCEInstanceType(self.provider, inst_type)
-        return None
+        inst_type = self.provider.get_resource('machineTypes', instance_type_id)
+        return GCEInstanceType(self.provider, inst_type) if inst_type else None
 
     def find(self, **kwargs):
         matched_inst_types = []
@@ -322,20 +320,8 @@ class GCERegionService(BaseRegionService):
         super(GCERegionService, self).__init__(provider)
 
     def get(self, region_id):
-        try:
-            region = (self.provider
-                          .gce_compute
-                          .regions()
-                          .get(project=self.provider.project_name,
-                               region=region_id)
-                          .execute())
-        # Handle the case when region_id is not valid
-        except googleapiclient.errors.HttpError:
-            return None
-        if region:
-            return GCERegion(self.provider, region)
-        else:
-            return None
+        region = self.provider.get_resource('regions', region_id)
+        return GCERegion(self.provider, region) if region else None
 
     def list(self, limit=None, marker=None):
         max_result = limit if limit is not None and limit < 500 else 500
@@ -387,29 +373,8 @@ class GCEImageService(BaseImageService):
         """
         Returns an Image given its id
         """
-        try:
-            image = (self.provider
-                         .gce_compute
-                         .images()
-                         .get(project=self.provider.project_name,
-                              image=image_id)
-                         .execute())
-            if image:
-                return GCEMachineImage(self.provider, image)
-        except TypeError as type_error:
-            # The API will throw an TypeError, if parameter `image` does not
-            # match the pattern "[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?".
-            cb.log.warning("TypeError: {0}".format(type_error))
-        except googleapiclient.errors.HttpError as http_error:
-            # If the image is not found in project-specific private images,
-            # look for this image in public images.
-            self._retrieve_public_images()
-            for public_image in self._public_images:
-                if public_image.id == image_id:
-                    return public_image
-            cb.log.warning(
-                "googleapiclient.errors.HttpError: {0}".format(http_error))
-        return None
+        image = self.provider.get_resource('images', image_id)
+        return GCEMachineImage(self.provider, image) if image else None
 
     def find(self, name, limit=None, marker=None):
         """
@@ -490,13 +455,18 @@ class GCEInstanceService(BaseInstanceService):
                     config['tags']['items'] = sg_names
         else:
             config = launch_config
-        operation = (self.provider
-                         .gce_compute.instances()
-                         .insert(project=self.provider.project_name,
-                                 zone=self.provider.default_zone,
-                                 body=config)
-                         .execute())
-        if 'zone' not in operation:
+        try:
+            operation = (self.provider
+                             .gce_compute.instances()
+                             .insert(project=self.provider.project_name,
+                                     zone=zone,
+                                     body=config)
+                             .execute())
+        except googleapiclient.errors.HttpError as http_error:
+            # If the operation request fails, the API will raise
+            # googleapiclient.errors.HttpError.
+            cb.log.warning(
+                "googleapiclient.errors.HttpError: {0}".format(http_error))
             return None
         zone_url = self.provider.parse_url(operation['zone'])
         instance_id = operation.get('targetLink')
@@ -512,16 +482,8 @@ class GCEInstanceService(BaseInstanceService):
         A GCE instance is uniquely identified by its selfLink, which is used
         as its id.
         """
-        try:
-            return GCEInstance(
-                    self.provider,
-                    self.provider.parse_url(instance_id).get_resource())
-        except googleapiclient.errors.HttpError as http_error:
-            # If the instance is not found, the API will raise
-            # googleapiclient.errors.HttpError.
-            cb.log.warning(
-                "googleapiclient.errors.HttpError: {0}".format(http_error))
-        return None
+        instance = self.provider.get_resource('instances', instance_id)
+        return GCEInstance(self.provider, instance) if instance else None
 
     def find(self, name, limit=None, marker=None):
         """
@@ -594,16 +556,8 @@ class GCENetworkService(BaseNetworkService):
         self._subnet_svc = GCESubnetService(self.provider)
 
     def get(self, network_id):
-        if network_id is None:
-            return None
-        # Note: networks = self.list(filter='id eq %s' % network_id) does not
-        # work due to a GCE API bug that causes an error if the network_id has
-        # has more than 19 digits.
-        networks = self.list()
-        for network in networks:
-            if network.id == network_id:
-                return network
-        return None
+        network = self.provider.get_resource('networks', network_id)
+        return GCENetwork(self.provider, network) if network else None
 
     def get_by_name(self, network_name):
         if network_name is None:
@@ -752,10 +706,8 @@ class GCESubnetService(BaseSubnetService):
         super(GCESubnetService, self).__init__(provider)
 
     def get(self, subnet_id):
-        for subnet in self.list():
-            if subnet.id == subnet_id:
-                return subnet
-        return None
+        subnet = self.provider.get_resource('subnetworks', subnet_id)
+        return GCESubnet(self.provider, subnet) if subnet else None
 
     def list(self, network=None, zone=None, limit=None, marker=None):
         region = zone.region_name if zone else self.provider.region_name
@@ -870,15 +822,8 @@ class GCEVolumeService(BaseVolumeService):
         """
         Returns a volume given its id.
         """
-        try:
-            return GCEVolume(self.provider,
-                             self.provider.parse_url(volume_id).get_resource())
-        except googleapiclient.errors.HttpError as http_error:
-            # If the volume is not found, the API will raise
-            # googleapiclient.errors.HttpError.
-            cb.log.warning(
-                "googleapiclient.errors.HttpError: {0}".format(http_error))
-        return None
+        vol = self.provider.get_resource('disks', volume_id)
+        return GCEVolume(self.provider, vol) if vol else None
 
     def find(self, name, limit=None, marker=None):
         """
@@ -977,16 +922,8 @@ class GCESnapshotService(BaseSnapshotService):
         """
         Returns a snapshot given its id.
         """
-        try:
-            return GCESnapshot(
-                    self.provider,
-                    self.provider.parse_url(snapshot_id).get_resource())
-        except googleapiclient.errors.HttpError as http_error:
-            # If the volume is not found, the API will raise
-            # googleapiclient.errors.HttpError.
-            cb.log.warning(
-                "googleapiclient.errors.HttpError: {0}".format(http_error))
-        return None
+        snapshot = self.provider.get_resource('snapshots', snapshot_id)
+        return GCESnapshot(self.provider, snapshot) if snapshot else None
 
     def find(self, name, limit=None, marker=None):
         """
@@ -1076,23 +1013,8 @@ class GCSObjectStoreService(BaseObjectStoreService):
         does not exist or if the user does not have permission to access the
         bucket.
         """
-        try:
-            response = (self.provider
-                            .gcp_storage
-                            .buckets()
-                            .get(bucket=bucket_id)
-                            .execute())
-            if 'error' in response:
-                # response['error']['code'] is 404 if the bucket does not exist
-                # and 403 if the user does not have permission to access it.
-                if response['error']['code'] not in (403, 404):
-                    cb.log.warning('Unexpected error code (%d) when accessing '
-                                   'bucket %s', response['error']['code'],
-                                   bucket_id)
-                return None
-            return GCSBucket(self.provider, response)
-        except:
-            return None
+        bucket = self.provider.get_resource('buckets', bucket_id)
+        return GCSBucket(self.provider, bucket) if bucket else None
 
     def find(self, name, limit=None, marker=None):
         """