Added an implementation of the GCE keypairs service

cloudbridge/cloud/providers/gce/helpers.py

@@ -0,0 +1,9 @@
+import os
+from Crypto.PublicKey import RSA
+
+
+def generate_key_pair():
+    kp = RSA.generate(2048, os.urandom)
+    public_key = kp.publickey().exportKey("OpenSSH").split(" ")[1]
+    private_key = kp.exportKey("PEM")
+    return private_key, public_key

cloudbridge/cloud/providers/gce/provider.py

@@ -3,7 +3,16 @@ Provider implementation based on google-api-python-client library
 for GCE.
 """
 
+
 from cloudbridge.cloud.base import BaseCloudProvider
+import json
+import os
+import time
+
+from googleapiclient import discovery
+import httplib2
+from oauth2client.client import SignedJwtAssertionCredentials
+
 from .services import GCESecurityService
 
 
@@ -13,6 +22,23 @@ class GCECloudProvider(BaseCloudProvider):
 
     def __init__(self, config):
         super(GCECloudProvider, self).__init__(config)
+
+        # Initialize cloud connection fields
+        self.client_email = self._get_config_value(
+            'gce_client_email', os.environ.get('GCE_CLIENT_EMAIL'))
+        self.private_key = self._get_config_value(
+            'gce_private_key', os.environ.get('GCE_PRIVATE_KEY'))
+        self.project_name = self._get_config_value(
+            'gce_project_name', os.environ.get('GCE_PROJECT_NAME'))
+        self.credentials_file = self._get_config_value(
+            'gce_service_creds_file', os.environ.get('GCE_SERVICE_CREDS_FILE'))
+        self.default_zone = self._get_config_value(
+            'gce_default_zone', os.environ.get('GCE_DEFAULT_ZONE'))
+
+        # service connections, lazily initialized
+        self._gce_compute = None
+
+        # Initialize provider services
         self._security = GCESecurityService(self)
 
     @property
@@ -38,3 +64,37 @@ class GCECloudProvider(BaseCloudProvider):
     def object_store(self):
         raise NotImplementedError(
             "GCECloudProvider does not implement this service")
+
+    @property
+    def gce_compute(self):
+        if not self._gce_compute:
+            self._gce_compute = self._connect_gce_compute()
+        return self._gce_compute
+
+    def _connect_gce_compute(self):
+        if self.credentials_file:
+            with open(self.credentials_file) as f:
+                data = json.load(f)
+                credentials = SignedJwtAssertionCredentials(
+                    data['client_email'], data['private_key'],
+                    'https://www.googleapis.com/auth/compute')
+        else:
+            credentials = SignedJwtAssertionCredentials(
+                self.client_email, self.private_key,
+                'https://www.googleapis.com/auth/compute')
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+        return discovery.build('compute', 'v1', http=http)
+
+    def wait_for_global_operation(self, operation):
+        while True:
+            result = self.gce_compute.globalOperations().get(
+                project=self.project_name,
+                operation=operation['name']).execute()
+
+            if result['status'] == 'DONE':
+                if 'error' in result:
+                    raise Exception(result['error'])
+                return result
+
+            time.sleep(0.5)

cloudbridge/cloud/providers/gce/resources.py

@@ -0,0 +1,44 @@
+"""
+DataTypes used by this provider
+"""
+from cloudbridge.cloud.base.resources import BaseKeyPair
+
+
+class GCEKeyPair(BaseKeyPair):
+
+    def __init__(self, provider, kp_id, kp_name, kp_material=None):
+        super(GCEKeyPair, self).__init__(provider, None)
+        self._kp_id = kp_id
+        self._kp_name = kp_name
+        self._kp_material = kp_material
+
+    @property
+    def id(self):
+        return self._kp_id
+
+    @property
+    def name(self):
+        # use e-mail as keyname if possible, or ID if not
+        return self._kp_name or self.id
+
+    def delete(self):
+        svc = self._provider.security.key_pairs
+
+        def _delete_key(gce_kp_generator):
+            kp_list = []
+            for gce_kp in gce_kp_generator:
+                if svc.gce_kp_to_id(gce_kp) == self.id:
+                    continue
+                else:
+                    kp_list.append(gce_kp)
+            return kp_list
+
+        svc.gce_metadata_save_op(_delete_key)
+
+    @property
+    def material(self):
+        return self._kp_material
+
+    @material.setter
+    def material(self, value):
+        self._kp_material = value

cloudbridge/cloud/providers/gce/services.py

@@ -1,6 +1,15 @@
+from cloudbridge.cloud.base.resources import ClientPagedResultList
 from cloudbridge.cloud.base.services import BaseKeyPairService
 from cloudbridge.cloud.base.services import BaseSecurityGroupService
 from cloudbridge.cloud.base.services import BaseSecurityService
+from cloudbridge.cloud.providers.gce import helpers
+from collections import namedtuple
+import hashlib
+
+
+from retrying import retry
+
+from .resources import GCEKeyPair
 
 
 class GCESecurityService(BaseSecurityService):
@@ -10,7 +19,6 @@ class GCESecurityService(BaseSecurityService):
 
         # Initialize provider services
         self._key_pairs = GCEKeyPairService(provider)
-        self._security_groups = GCESecurityGroupService(provider)
 
     @property
     def key_pairs(self):
@@ -18,13 +26,159 @@ class GCESecurityService(BaseSecurityService):
 
     @property
     def security_groups(self):
-        return self._security_groups
+        raise NotImplementedError(
+            "GCECloudProvider does not implement this service")
 
 
 class GCEKeyPairService(BaseKeyPairService):
 
+    GCEKeyInfo = namedtuple('GCEKeyInfo', 'format public_key email')
+
     def __init__(self, provider):
         super(GCEKeyPairService, self).__init__(provider)
+        self._gce_projects = None
+
+    @property
+    def gce_projects(self):
+        if not self._gce_projects:
+            self._gce_projects = self.provider.gce_compute.projects()
+        return self._gce_projects
+
+    def get(self, key_pair_id):
+        """
+        Returns a KeyPair given its ID.
+        """
+        for kp in self.list():
+            if kp.id == key_pair_id:
+                return kp
+        else:
+            return None
+
+    def _iter_gce_key_pairs(self):
+        """
+        Iterates through the project's metadata, yielding a GCEKeyInfo object
+        for each entry in commonInstanceMetaData/items
+        """
+        metadata = self._get_common_metadata()
+        for kpinfo in self._iter_gce_ssh_keys(metadata):
+            yield kpinfo
+
+    def _get_common_metadata(self):
+        """
+        Get a project's commonInstanceMetadata entry
+        """
+        metadata = self.gce_projects.get(
+            project=self.provider.project_name).execute()
+        return metadata["commonInstanceMetadata"]
+
+    def _get_or_add_sshkey_entry(self, metadata):
+        """
+        Get the sshKeys entry from commonInstanceMetadata/items.
+        If an entry does not exist, adds a new empty entry
+        """
+        sshkey_entry = None
+        entries = [item for item in metadata["items"]
+                   if item["key"] == "sshKeys"]
+        if entries:
+            sshkey_entry = entries[0]
+        else:  # add a new entry
+            sshkey_entry = {"key": "sshKeys", "value": ""}
+            metadata["items"].append(sshkey_entry)
+        return sshkey_entry
+
+    def _iter_gce_ssh_keys(self, metadata):
+        """
+        Iterates through the ssh keys given a commonInstanceMetadata dict,
+        yielding a GCEKeyInfo object for each entry in
+        commonInstanceMetaData/items
+        """
+        sshkeys = self._get_or_add_sshkey_entry(metadata)["value"]
+        for key in sshkeys.split("\n"):
+            # elems should be "ssh-rsa <public_key> <email>"
+            elems = key.split(" ")
+            if elems and elems[0]:  # ignore blank lines
+                yield GCEKeyPairService.GCEKeyInfo(elems[0], elems[1],
+                                                   elems[2])
+
+    def gce_metadata_save_op(self, callback):
+        """
+        Carries out a metadata save operation. In GCE, a fingerprint based
+        locking mechanism is used to prevent lost updates. A new fingerprint
+        is returned each time metadata is retrieved. Therefore, this method
+        retrieves the metadata, invokes the provided callback with that
+        metadata, and saves the metadata using the original fingerprint
+        immediately afterwards, ensuring that update conflicts can be detected.
+        """
+        def _save_common_metadata():
+            metadata = self._get_common_metadata()
+            # add a new entry if one doesn'te xist
+            sshkey_entry = self._get_or_add_sshkey_entry(metadata)
+            gce_kp_list = callback(self._iter_gce_ssh_keys(metadata))
+
+            entry = ""
+            for gce_kp in gce_kp_list:
+                entry = entry + u"{0} {1} {2}\n".format(gce_kp.format,
+                                                        gce_kp.public_key,
+                                                        gce_kp.email)
+            sshkey_entry["value"] = entry.rstrip()
+            # common_metadata will have the current fingerprint at this point
+            operation = self.gce_projects.setCommonInstanceMetadata(
+                project=self.provider.project_name, body=metadata).execute()
+            self.provider.wait_for_global_operation(operation)
+
+        # Retry a few times if the fingerprints conflict
+        retry_decorator = retry(stop_max_attempt_number=5)
+        retry_decorator(_save_common_metadata)()
+
+    def gce_kp_to_id(self, gce_kp):
+        """
+        Accept a GCEKeyInfo object and return a unique
+        ID for it
+        """
+        md5 = hashlib.md5()
+        md5.update(gce_kp.public_key)
+        return md5.hexdigest()
+
+    def list(self, limit=None, marker=None):
+        key_pairs = []
+        for gce_kp in self._iter_gce_key_pairs():
+            kp_id = self.gce_kp_to_id(gce_kp)
+            kp_name = gce_kp.email
+            key_pairs.append(GCEKeyPair(self.provider, kp_id, kp_name))
+        return ClientPagedResultList(self.provider, key_pairs,
+                                     limit=limit, marker=marker)
+
+    def find(self, name, limit=None, marker=None):
+        """
+        Searches for a key pair by a given list of attributes.
+        """
+        found_kps = []
+        for kp in self.list():
+            if kp.name == name:
+                found_kps.append(kp)
+        return ClientPagedResultList(self.provider, found_kps,
+                                     limit=limit, marker=marker)
+
+    def create(self, name):
+        kp = self.find(name=name)
+        if kp:
+            return kp
+
+        private_key, public_key = helpers.generate_key_pair()
+        kp_info = GCEKeyPairService.GCEKeyInfo(name + u":ssh-rsa",
+                                               public_key, name)
+
+        def _add_kp(gce_kp_generator):
+            kp_list = []
+            # Add the new key pair
+            kp_list.append(kp_info)
+            for gce_kp in gce_kp_generator:
+                kp_list.append(gce_kp)
+            return kp_list
+
+        self.gce_metadata_save_op(_add_kp)
+        return GCEKeyPair(self.provider, self.gce_kp_to_id(kp_info), name,
+                          kp_material=private_key)
 
 
 class GCESecurityGroupService(BaseSecurityGroupService):

setup.py

@@ -21,7 +21,7 @@ openstack_reqs = ['python-novaclient==2.33.0',
                   'python-neutronclient==3.1.0',
                   'python-keystoneclient==2.0.0']
 aws_reqs = ['boto==2.38.0']
-gce_reqs = ['google-api-python-client==1.4.2']
+gce_reqs = ['google-api-python-client==1.4.2', "pycrypto"]
 full_reqs = base_reqs + aws_reqs + openstack_reqs + gce_reqs
 dev_reqs = (['httpretty==0.8.10', 'tox==2.1.1', 'moto==0.4.18',
              'sphinx==1.3.1'] + full_reqs)

tox.ini

@@ -8,7 +8,7 @@ envlist = py27, py35, pypy
 
 [testenv]
 commands = {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test
-passenv = AWS_ACCESS_KEY AWS_SECRET_KEY OS_AUTH_URL OS_PASSWORD OS_TENANT_NAME OS_USERNAME OS_REGION_NAME NOVA_SERVICE_NAME CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS CB_TEST_PROVIDER CB_USE_MOCK_PROVIDERS
+passenv = AWS_ACCESS_KEY AWS_SECRET_KEY GCE_CLIENT_EMAIL GCE_PRIVATE_KEY GCE_PROJECT_NAME GCE_DEFAULT_ZONE GCE_SERVICE_CREDS_FILE OS_AUTH_URL OS_PASSWORD OS_TENANT_NAME OS_USERNAME OS_REGION_NAME NOVA_SERVICE_NAME CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS CB_TEST_PROVIDER CB_USE_MOCK_PROVIDERS
 deps =
     -rrequirements.txt
     coverage