Sync fork with upstream and merge the changes.

cloudbridge/cloud/providers/gce/provider.py

@@ -8,6 +8,8 @@ from cloudbridge.cloud.base import BaseCloudProvider
 import httplib2
 import json
 import os
+import re
+from string import Template
 import time
 
 from googleapiclient import discovery
@@ -20,6 +22,110 @@ from .services import GCENetworkService
 from .services import GCESecurityService
 
 
+class GCPResourceUrl(object):
+
+    def __init__(self, resource, connection):
+        self._resource = resource
+        self._connection = connection
+        self.parameters = {}
+
+    def get(self):
+        discovery_object = getattr(self._connection, self._resource)()
+        return discovery_object.get(**self.parameters).execute()
+
+
+class GCPResources(object):
+    
+    def __init__(self, connection):
+        self._connection = connection
+
+        # Resource descriptions are already pulled into the internal
+        # _resourceDesc field of the connection.
+        #
+        # TODO: We could fetch compute resource descriptions from
+        # https://www.googleapis.com/discovery/v1/apis/compute/v1/rest and
+        # storage resource descriptions from
+        # https://www.googleapis.com/discovery/v1/apis/storage/v1/rest
+        # ourselves.
+        #
+        # Resource descriptions are in JSON format which are then parsed into a
+        # Python dictionary. The main fields we are interested are:
+        #
+        # {
+        #   "rootUrl": "https://www.googleapis.com/",
+        #   "servicePath": COMPUTE OR STORAGE SERVICE PATH
+        #   "resources": {
+        #     RESOURCE_NAME: {
+        #       "methods": {
+        #         "get": {
+        #           "path": RESOURCE PATH PATTERN
+        #           "parameters": {
+        #             PARAMETER: {
+        #               "pattern": REGEXP FOR VALID VALUES
+        #               ...
+        #             },
+        #             ...
+        #           },
+        #           "parameterOrder": [LIST OF PARAMETERS]
+        #         },
+        #         ...
+        #       }
+        #     },
+        #     ...
+        #   }
+        #   ...
+        # }
+        desc = connection._resourceDesc
+        self._root_url = desc['rootUrl']
+        self._service_path = desc['servicePath']
+        self._resources = {}
+
+        # We will not mutate self._desc; it's OK to use items() in Python 2.x.
+        for resource, resource_desc in desc['resources'].items():
+            methods = resource_desc['methods']
+            if 'get' not in methods:
+                continue
+            method = methods['get']
+            parameters = method['parameterOrder']
+
+            # We would like to change a path like
+            # {project}/regions/{region}/addresses/{address} to a pattern like
+            # (PROJECT REGEX)/regions/(REGION REGEX)/addresses/(ADDRESS REGEX).
+            template = Template('${'.join(method['path'].split('{')))
+            mapping = {}
+            for parameter in parameters:
+                parameter_desc = method['parameters'][parameter]
+                if 'pattern' in parameter_desc:
+                    mapping[parameter] = '(%s)' % parameter_desc['pattern']
+                else:
+                    mapping[parameter] = '([^/]+)'
+            pattern = template.substitute(**mapping)
+
+            # Store the parameters and the regex pattern of this resource.
+            self._resources[resource] = {'parameters': parameters,
+                                         'pattern': re.compile(pattern)}
+
+    def parse_url(self, url):
+        """
+        Build a GCPResourceUrl from a resource's URL string. One can then call
+        the get() method on the returned object to fetch resource details from
+        GCP servers.
+        """
+        url = url.strip()
+        if url.startswith(self._root_url): url = url[len(self._root_url):]
+        if url.startswith(self._service_path):
+            url = url[len(self._service_path):]
+
+        for resource, desc in self._resources.items():
+            m = re.match(desc['pattern'], url)
+            if m is None or len(m.group(0)) < len(url):
+                continue
+            out = GCPResourceUrl(resource, self._connection)
+            for index, parameter in enumerate(desc['parameters']):
+                out.parameters[parameter] = m.group(index + 1)
+            return out
+
+
 class GCECloudProvider(BaseCloudProvider):
 
     PROVIDER_ID = 'gce'
@@ -47,16 +153,18 @@ class GCECloudProvider(BaseCloudProvider):
         self.region_name = self._get_config_value(
             'gce_region_name', 'us-central1')
 
-        # oauth2client.Credentials to be used for authentication
-        self._credentials = None
         # service connections, lazily initialized
         self._gce_compute = None
+        self._gcp_storage = None
 
         # Initialize provider services
         self._compute = GCEComputeService(self)
         self._security = GCESecurityService(self)
         self._network = GCENetworkService(self)
 
+        self._compute_resources = GCPResources(self.gce_compute)
+        self._storage_resources = GCPResources(self.gcp_storage)
+
     @property
     def compute(self):
         return self._compute
@@ -85,14 +193,19 @@ class GCECloudProvider(BaseCloudProvider):
             self._gce_compute = self._connect_gce_compute()
         return self._gce_compute
 
-    def _connect_gce_compute(self):
+    @property
+    def gcp_storage(self):
+        if not self._gcp_storage:
+            self._gcp_storage = self._connect_gcp_storage()
+        return self._gcp_storage
+
+    @property
+    def _credentials(self):
         if self.credentials_dict:
-            credentials = ServiceAccountCredentials.from_json_keyfile_dict(
+            return ServiceAccountCredentials.from_json_keyfile_dict(
                 self.credentials_dict)
         else:
-            credentials = GoogleCredentials.get_application_default()
-        self._credentials = credentials
-        return discovery.build('compute', 'v1', credentials=credentials)
+            return GoogleCredentials.get_application_default()
 
     def get_gce_resource_data(self, uri):
         """
@@ -118,9 +231,32 @@ class GCECloudProvider(BaseCloudProvider):
                 project=self.project_name,
                 operation=operation['name']).execute()
 
+    def _connect_gcp_storage(self):
+        return discovery.build('storage', 'v1', credentials=self._credentials)
+
+    def _connect_gce_compute(self):
+        return discovery.build('compute', 'v1', credentials=self._credentials)
+
+    def wait_for_operation(self, operation, region=None, zone=None):
+        args = {'project': self.project_name, 'operation': operation['name']}
+        if not region and not zone:
+            operations = self.gce_compute.globalOperations()
+        elif zone:
+            operations = self.gce_compute.zoneOperations()
+            args['zone'] = zone
+        else:
+            operations = self.gce_compute.regionOperations()
+            args['region'] = region
+
+        while True:
+            result = operations.get(**args).execute()
             if result['status'] == 'DONE':
                 if 'error' in result:
                     raise Exception(result['error'])
                 return result
 
             time.sleep(0.5)
+
+    def parse_url(self, url):
+        out = self._compute_resources.parse_url(url)
+        return out if out else self._storage_resources.parse_url(url)

cloudbridge/cloud/providers/gce/resources.py

@@ -1,6 +1,7 @@
 """
 DataTypes used by this provider
 """
+from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseInstance
 from cloudbridge.cloud.base.resources import BaseInstanceType
 from cloudbridge.cloud.base.resources import BaseKeyPair
@@ -13,6 +14,8 @@ from cloudbridge.cloud.base.resources import BaseSecurityGroupRule
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import MachineImageState
 
+import cloudbridge as cb
+
 # Older versions of Python do not have a built-in set data-structure.
 try:
     set
@@ -25,6 +28,7 @@ import json
 import re
 import uuid
 
+
 class GCEKeyPair(BaseKeyPair):
 
     def __init__(self, provider, kp_id, kp_name, kp_material=None):
@@ -203,20 +207,6 @@ class GCEFirewallsDelegate(object):
         md5.update("{0}-{1}".format(tag, network_name).encode('ascii'))
         return md5.hexdigest()
 
-    @staticmethod
-    def network_name(firewall):
-        """
-        Extract the network name of a firewall.
-        """
-        if 'network' not in firewall:
-            return GCEFirewallsDelegate.DEFAULT_NETWORK
-        match = re.search(
-                GCEFirewallsDelegate._NETWORK_URL_PREFIX + '([^/]*)$',
-                firewall['network'])
-        if match and len(match.groups()) == 1:
-            return match.group(1)
-        return None
-
     @property
     def provider(self):
         return self._provider
@@ -228,11 +218,20 @@ class GCEFirewallsDelegate(object):
         """
         out = set()
         for firewall in self.iter_firewalls():
-            network_name = GCEFirewallsDelegate.network_name(firewall)
+            network_name = self.network_name(firewall)
             if network_name is not None:
                 out.add((firewall['targetTags'][0], network_name))
         return out
             
+    def network_name(self, firewall):
+        """
+        Extract the network name of a firewall.
+        """
+        if 'network' not in firewall:
+            return GCEFirewallsDelegate.DEFAULT_NETWORK
+        url = self._provider.parse_url(firewall['network'])
+        return url.parameters['network']
+
     def get_tag_network_from_id(self, tag_network_id):
         """
         Map an ID back to the (tag, network name) pair.
@@ -286,7 +285,7 @@ class GCEFirewallsDelegate(object):
                                       .insert(project=project_name,
                                               body=firewall)
                                       .execute())
-            self._provider.wait_for_global_operation(response)
+            self._provider.wait_for_operation(response)
             # TODO: process the response and handle errors.
             return True
         except:
@@ -335,7 +334,7 @@ class GCEFirewallsDelegate(object):
             if ('ports' in firewall['allowed'][0] and
                 len(firewall['allowed'][0]['ports']) == 1):
                 info['port'] = firewall['allowed'][0]['ports'][0]
-            info['network_name'] = GCEFirewallsDelegate.network_name(firewall)
+            info['network_name'] = self.network_name(firewall)
             return info
         return info
 
@@ -367,7 +366,7 @@ class GCEFirewallsDelegate(object):
             if network_name is None:
                 yield firewall
                 continue
-            firewall_network_name = GCEFirewallsDelegate.network_name(firewall)
+            firewall_network_name = self.network_name(firewall)
             if firewall_network_name == network_name:
                 yield firewall
 
@@ -382,7 +381,7 @@ class GCEFirewallsDelegate(object):
                                       .delete(project=project_name,
                                               firewall=firewall['name'])
                                       .execute())
-            self._provider.wait_for_global_operation(response)
+            self._provider.wait_for_operation(response)
             # TODO: process the response and handle errors.
             return True
         except:
@@ -877,8 +876,8 @@ class GCEInstance(BaseInstance):
         """
         network_url = self._gce_instance.get('networkInterfaces')[0].get(
             'network')
-        network_name = GCEFirewallsDelegate.network_name(
-            {'network': network_url})
+        url = self._provider.parse_url(network_url)
+        network_name = url.parameters['network']
         if 'items' not in self._gce_instance['tags']:
             return []
         tags = self._gce_instance['tags']['items']
@@ -980,7 +979,7 @@ class GCENetwork(BaseNetwork):
                     .execute())
             if 'error' in response:
                 return False
-            self._provider.wait_for_global_operation(response)
+            self._provider.wait_for_operation(response)
             return True
         except:
             return False
@@ -993,3 +992,74 @@ class GCENetwork(BaseNetwork):
 
     def refresh(self):
         return self.state
+
+class GCEFloatingIP(BaseFloatingIP):
+
+    def __init__(self, provider, floating_ip):
+        super(GCEFloatingIP, self).__init__(provider)
+        self._ip = floating_ip
+
+        # We use regional IPs to simulate floating IPs not global IPs because
+        # global IPs can be forwarded only to load balancing resources, not to
+        # a specific instance. Find out the region to which the IP belongs.
+        url = provider.parse_url(self._ip['region'])
+        self._region = url.parameters['region']
+
+        # Check if the address is used by a resource.
+        self._rule = None
+        self._target_instance = None
+        if 'users' in floating_ip and len(floating_ip['users']) > 0:
+            if len(floating_ip['users']) > 1:
+                cb.log.warning('Address "%s" in use by more than one resource',
+                               floating_ip['address'])
+            resource = provider.parse_url(floating_ip['users'][0]).get()
+            if resource['kind'] == 'compute#forwardingRule':
+                self._rule = resource
+                target = provider.parse_url(resource['target']).get()
+                if target['kind'] == 'compute#targetInstance':
+                    url = provider.parse_url(target['instance'])
+                    self._target_instance = url.get()
+                else:
+                    cb.log.warning('Address "%s" is forwarded to a %s',
+                                   floating_ip['address'], target['kind'])
+            else:
+                cb.log.warning('Address "%s" in use by a %s',
+                               floating_ip['address'], resource['kind'])
+
+    @property
+    def id(self):
+        return self._ip['id']
+
+    @property
+    def public_ip(self):
+        return self._ip['address']
+
+    @property
+    def private_ip(self):
+        if not self._target_instance:
+            return None
+        return self._target_instance['networkInterfaces'][0]['networkIP']
+
+    def in_use(self):
+        return True if self._target_instance else False
+
+    def delete(self):
+       project_name = self._provider.project_name
+       # First, delete the forwarding rule, if there is any.
+       if self._rule:
+           response = (self._provider.gce_compute
+                                     .forwardingRules()
+                                     .delete(project=project_name,
+                                             region=self._region,
+                                             forwardingRule=self._rule['name'])
+                                     .execute())
+           self._provider.wait_for_operation(response, region=self._region)
+
+       # Release the address.
+       response = (self._provider.gce_compute
+                                 .addresses()
+                                 .delete(project=project_name,
+                                         region=self._region,
+                                         address=self._ip['name'])
+                                 .execute())
+       self._provider.wait_for_operation(response, region=self._region)

cloudbridge/cloud/providers/gce/services.py

@@ -20,7 +20,10 @@ import googleapiclient
 from retrying import retry
 import sys
 
+import uuid
+
 from .resources import GCEFirewallsDelegate
+from .resources import GCEFloatingIP
 from .resources import GCEInstance
 from .resources import GCEInstanceType
 from .resources import GCEKeyPair
@@ -30,6 +33,7 @@ from .resources import GCERegion
 from .resources import GCESecurityGroup
 from .resources import GCESecurityGroupRule
 
+
 class GCESecurityService(BaseSecurityService):
 
     def __init__(self, provider):
@@ -142,7 +146,7 @@ class GCEKeyPairService(BaseKeyPairService):
             # common_metadata will have the current fingerprint at this point
             operation = self.gce_projects.setCommonInstanceMetadata(
                 project=self.provider.project_name, body=metadata).execute()
-            self.provider.wait_for_global_operation(operation)
+            self.provider.wait_for_operation(operation)
 
         # Retry a few times if the fingerprints conflict
         retry_decorator = retry(stop_max_attempt_number=5)
@@ -602,7 +606,7 @@ class GCENetworkService(BaseNetworkService):
                                      .execute())
             if 'error' in response:
                 return None
-            self.provider.wait_for_global_operation(response)
+            self.provider.wait_for_operation(response)
             networks = self.list(filter='name eq %s' % name)
             return None if len(networks) == 0 else networks[0]
         except:
@@ -612,11 +616,45 @@ class GCENetworkService(BaseNetworkService):
     def subnets(self):
         raise NotImplementedError('To be implemented')
 
-    def floating_ips(self, network_id=None):
-        raise NotImplementedError('To be implemented')
+    def floating_ips(self, network_id=None, region=None):
+        if not region:
+            region = self.provider.region_name
+        try:
+            response = (self.provider.gce_compute
+                                     .addresses()
+                                     .list(project=self.provider.project_name,
+                                           region=region)
+                                     .execute())
+            ips = []
+            if 'items' in response:
+                for ip in response['items']:
+                    ips.append(GCEFloatingIP(self.provider, ip))
+            # TODO: if network_id is given, filter out IPs that are assigned to
+            # resources in a different network.
+            return ips
+        except:
+            return []
 
-    def create_floating_ip(self):
-        raise NotImplementedError('To be implemented')
+    def create_floating_ip(self, region=None):
+        if not region:
+            region = self.provider.region_name
+        ip_name = 'ip-{0}'.format(uuid.uuid4())
+        try:
+            response = (self.provider.gce_compute
+                                     .addresses()
+                                     .insert(project=self.provider.project_name,
+                                             region=region,
+                                             body={'name': ip_name})
+                                     .execute())
+            if 'error' in response:
+                return None
+            self.provider.wait_for_operation(response, region=region)
+            ips = self.floating_ips()
+            for ip in ips:
+                if ip.id == response["targetId"]:
+                    return ip
+        except:
+            return None
 
     def routers(self):
         raise NotImplementedError('To be implemented')