Reimplement resource URL parsing

Instead of heuristically parsing URLs, this time we use resource
descriptions retrieved from googleapis.com. Descriptions have
information like the URL patter of a resource, the parameters, and
the format of the parameters.

cloudbridge/cloud/providers/gce/provider.py

@@ -5,9 +5,10 @@ for GCE.
 
 
 from cloudbridge.cloud.base import BaseCloudProvider
-import cloudbridge as cb
 import json
 import os
+import re
+from string import Template
 import time
 
 from googleapiclient import discovery
@@ -19,6 +20,77 @@ from .services import GCENetworkService
 from .services import GCESecurityService
 
 
+class GCPResourceUrl(object):
+
+    def __init__(self, resource, connection):
+        self._resource = resource
+        self._connection = connection
+        self.parameters = {}
+
+    def get(self):
+        discovery_object = getattr(self._connection, self._resource)()
+        return discovery_object.get(**self.parameters).execute()
+
+
+class GCPResources(object):
+    
+    def __init__(self, connection):
+        self._connection = connection
+
+        # Sorry, but the most reliable source of resource descriptions is the
+        # internal _resourceDesc field of the connection.
+        #
+        # TODO: We could fetch compute resource descriptions from
+        # https://www.googleapis.com/discovery/v1/apis/compute/v1/rest and
+        # storage resource descriptions from
+        # https://www.googleapis.com/discovery/v1/apis/storage/v1/rest
+        # ourselves.
+        desc = connection._resourceDesc
+        self._root_url = desc['rootUrl']
+        self._service_path = desc['servicePath']
+        self._resources = {}
+
+        # We will not mutate self._desc; it's OK to use items() in Python 2.x.
+        for resource, resource_desc in desc['resources'].items():
+            methods = resource_desc['methods']
+            if 'get' not in methods:
+                continue
+            method = methods['get']
+            parameters = method['parameterOrder']
+
+            # We would like to change a path like
+            # {project}/regions/{region}/addresses/{address} to a pattern like
+            # (PROJECT REGEX)/regions/(REGION REGEX)/addresses/(ADDRESS REGEX).
+            template = Template('${'.join(method['path'].split('{')))
+            mapping = {}
+            for parameter in parameters:
+                parameter_desc = method['parameters'][parameter]
+                if 'pattern' in parameter_desc:
+                    mapping[parameter] = '(%s)' % parameter_desc['pattern']
+                else:
+                    mapping[parameter] = '([^/]+)'
+            pattern = template.substitute(**mapping)
+
+            # Store the parameters and the regex pattern of this resource.
+            self._resources[resource] = {'parameters': parameters,
+                                         'pattern': re.compile(pattern)}
+
+    def parse_url(self, url):
+        url = url.strip()
+        if url.startswith(self._root_url): url = url[len(self._root_url):]
+        if url.startswith(self._service_path):
+            url = url[len(self._service_path):]
+
+        for resource, desc in self._resources.items():
+            m = re.match(desc['pattern'], url)
+            if m is None or len(m.group(0)) < len(url):
+                continue
+            out = GCPResourceUrl(resource, self._connection)
+            for index, parameter in enumerate(desc['parameters']):
+                out.parameters[parameter] = m.group(index + 1)
+            return out
+
+
 class GCECloudProvider(BaseCloudProvider):
 
     PROVIDER_ID = 'gce'
@@ -48,12 +120,17 @@ class GCECloudProvider(BaseCloudProvider):
 
         # service connections, lazily initialized
         self._gce_compute = None
+        self._gcp_storage = None
 
         # Initialize provider services
         self._compute = GCEComputeService(self)
         self._security = GCESecurityService(self)
         self._network = GCENetworkService(self)
 
+        self._compute_resources = GCPResources(self.gce_compute)
+        # Enable when the storage 
+        # self._storage_resources = GCPResources(self.gcp_storage)
+
     @property
     def compute(self):
         return self._compute
@@ -84,15 +161,23 @@ class GCECloudProvider(BaseCloudProvider):
 
     @property
     def gcp_storage(self):
-        raise NotImplementedError("To be implemented")
+        if not self._gcp_storae:
+            self._gcp_storage = self._connect_gcp_storage()
+        return self._gcp_storage
 
-    def _connect_gce_compute(self):
+    @property
+    def _credentials(self):
         if self.credentials_dict:
-            credentials = ServiceAccountCredentials.from_json_keyfile_dict(
+            return ServiceAccountCredentials.from_json_keyfile_dict(
                 self.credentials_dict)
         else:
-            credentials = GoogleCredentials.get_application_default()
-        return discovery.build('compute', 'v1', credentials=credentials)
+            return GoogleCredentials.get_application_default()
+
+    def _connect_gcp_storage(self):
+        return discovery.build('storage', 'v1', credentials=self._credentials)
+
+    def _connect_gce_compute(self):
+        return discovery.build('compute', 'v1', credentials=self._credentials)
 
     def wait_for_operation(self, operation, region=None, zone=None):
         args = {'project': self.project_name, 'operation': operation['name']}
@@ -113,3 +198,7 @@ class GCECloudProvider(BaseCloudProvider):
                 return result
 
             time.sleep(0.5)
+
+    def parse_url(self, url):
+        out = self._compute_resources.parse_url(url)
+        return out if out else self._storage_resources.parse_url(url)

cloudbridge/cloud/providers/gce/resources.py

@@ -27,94 +27,6 @@ import re
 import uuid
 
 
-class GCPResourceType(object):
-
-    def __init__(self, platform, resource_type):
-        self.platform = platform
-        self.resource_type = resource_type
-
-    @property
-    def kind(self):
-        return '%s#%s' % (self.platform, self.resource_type)
-
-    @property
-    def url_field_name(self):
-        return '%ss' % self.resource_type
-
-
-# GCP resource types we are interested in
-FORWARDING_RULE_TYPE = GCPResourceType('compute', 'forwardingRule')
-INSTANCE_TYPE = GCPResourceType('compute', 'instance')
-NETWORK_TYPE = GCPResourceType('compute', 'network')
-REGION_TYPE = GCPResourceType('compute', 'region')
-TARGET_INSTANCE_TYPE = GCPResourceType('compute', 'targetInstance')
-
-ALL_RESOURCE_TYPES = [FORWARDING_RULE_TYPE,
-                      INSTANCE_TYPE,
-                      NETWORK_TYPE,
-                      REGION_TYPE,
-                      TARGET_INSTANCE_TYPE]
-
-
-class GCPResourceUrl(GCPResourceType):
-
-    def __init__(self, platform, resource_type, project=None, region=None,
-                 zone=None, name=None, connection=None):
-        super(GCPResourceUrl, self).__init__(platform, resource_type)
-        self.project = project
-        self.region = region
-        self.zone = zone
-        self.name = name
-        self.connection = connection
-
-    @staticmethod
-    def parse_url(url, provider=None):
-        parts = url.split('/')
-        out = None
-        for resource_type in ALL_RESOURCE_TYPES:
-            if parts[-2] == resource_type.url_field_name:
-                out = GCPResourceUrl(resource_type.platform,
-                                     resource_type.resource_type)
-        if not out:
-            return out
-        out.name = parts[-1]
-
-        # If a cloud provider is given, set the connection.
-        if provider:
-            if out.platform == 'compute':
-                out.connection = provider.gce_compute
-            elif out.platform == 'storage':
-                out.connection = provider.gcp_storage
-
-        # Set region, zone, and project fields.
-        i = 0
-        while i < len(parts) - 2:
-            if parts[i] == 'regions':
-                out.region = parts[i + 1]
-                i += 2
-            elif parts[i] == 'zones':
-                out.zone = parts[i + 1]
-                i += 2
-            elif parts[i] == 'projects':
-                out.project = parts[i + 1]
-                i += 2
-            else:
-                i += 1
-        return out
-
-    @property
-    def discovery_object(self):
-        if not self.connection:
-            raise Exception('Connection is not set, yet')
-        return getattr(self.connection, self.url_field_name)()
-
-    def get(self):
-        args = {'project': self.project, self.resource_type: self.name}
-        if self.region: args['region'] = self.region
-        if self.zone: args['zone'] = self.zone
-        return self.discovery_object.get(**args).execute()
-
-
 class GCEKeyPair(BaseKeyPair):
 
     def __init__(self, provider, kp_id, kp_name, kp_material=None):
@@ -289,15 +201,6 @@ class GCEFirewallsDelegate(object):
         md5.update("{0}-{1}".format(tag, network_name).encode('ascii'))
         return md5.hexdigest()
 
-    @staticmethod
-    def network_name(firewall):
-        """
-        Extract the network name of a firewall.
-        """
-        if 'network' not in firewall:
-            return GCEFirewallsDelegate.DEFAULT_NETWORK
-        return GCPResourceUrl.parse_url(firewall['network']).name
-
     @property
     def provider(self):
         return self._provider
@@ -309,11 +212,20 @@ class GCEFirewallsDelegate(object):
         """
         out = set()
         for firewall in self.iter_firewalls():
-            network_name = GCEFirewallsDelegate.network_name(firewall)
+            network_name = self.network_name(firewall)
             if network_name is not None:
                 out.add((firewall['targetTags'][0], network_name))
         return out
             
+    def network_name(self, firewall):
+        """
+        Extract the network name of a firewall.
+        """
+        if 'network' not in firewall:
+            return GCEFirewallsDelegate.DEFAULT_NETWORK
+        url = self._provider.parse_url(firewall['network'])
+        return url.parameters['network']
+
     def get_tag_network_from_id(self, tag_network_id):
         """
         Map an ID back to the (tag, network name) pair.
@@ -416,7 +328,7 @@ class GCEFirewallsDelegate(object):
             if ('ports' in firewall['allowed'][0] and
                 len(firewall['allowed'][0]['ports']) == 1):
                 info['port'] = firewall['allowed'][0]['ports'][0]
-            info['network_name'] = GCEFirewallsDelegate.network_name(firewall)
+            info['network_name'] = self.network_name(firewall)
             return info
         return info
 
@@ -448,7 +360,7 @@ class GCEFirewallsDelegate(object):
             if network_name is None:
                 yield firewall
                 continue
-            firewall_network_name = GCEFirewallsDelegate.network_name(firewall)
+            firewall_network_name = self.network_name(firewall)
             if firewall_network_name == network_name:
                 yield firewall
 
@@ -836,27 +748,29 @@ class GCEFloatingIP(BaseFloatingIP):
         # We use regional IPs to simulate floating IPs not global IPs because
         # global IPs can be forwarded only to load balancing resources, not to
         # a specific instance. Find out the region to which the IP belongs.
-        self._region = GCPResourceUrl.parse_url(self._ip['region']).name
+        url = provider.parse_url(self._ip['region'])
+        self._region = url.parameters['region']
 
         # Check if the address is used by a resource.
         self._rule = None
         self._target_instance = None
         if 'users' in floating_ip and len(floating_ip['users']) > 0:
             if len(floating_ip['users']) > 1:
-                cb.log.warning('IP is in user by more than one resource')
-            url = GCPResourceUrl.parse_url(floating_ip['users'][0], provider)
-            resource = url.get()
-            if resource['kind'] == FORWARDING_RULE_TYPE.kind:
+                cb.log.warning('Address "%s" in use by more than one resource',
+                               floating_ip['address'])
+            resource = provider.parse_url(floating_ip['users'][0]).get()
+            if resource['kind'] == 'compute#forwardingRule':
                 self._rule = resource
-                url = GCPResourceUrl.parse_url(resource['target'], provider)
-                target = url.get()
-                if target['kind'] == TARGET_INSTANCE_TYPE.kind:
-                    url = GCPResourceUrl.parse_url(target['instance'], provider)
+                target = provider.parse_url(resource['target']).get()
+                if target['kind'] == 'compute#targetInstance':
+                    url = provider.parse_url(target['instance'])
                     self._target_instance = url.get()
                 else:
-                    cb.log.warning('IP is forwarded to a %s' % target['kind'])
+                    cb.log.warning('Address "%s" is forwarded to a %s',
+                                   floating_ip['address'], target['kind'])
             else:
-                cb.log.warning('IP in use by a %s' % resource['kind'])
+                cb.log.warning('Address "%s" in use by a %s',
+                               floating_ip['address'], resource['kind'])
 
     @property
     def id(self):