Merge branch 'gce' of https://github.com/CloudVE/cloudbridge into gce

cloudbridge/cloud/providers/gce/resources.py

@@ -469,6 +469,18 @@ class GCEVMFirewall(BaseVMFirewall):
         """
         return self._vm_firewall
 
+    @property
+    def label(self):
+        tag_name = "_".join(["firewall", self.name, "label"])
+        return helpers.get_metadata_item_value(self._provider, tag_name)
+        # TODO: Add removing metadata to delete function
+
+    @label.setter
+    def label(self, value):
+        self.assert_valid_resource_label(value)
+        tag_name = "_".join(["firewall", self.name, "label"])
+        helpers.modify_or_add_metadata_item(self._provider, tag_name, value)
+
     @property
     def description(self):
         """
@@ -797,10 +809,13 @@ class GCEMachineImage(BaseMachineImage):
         Refreshes the state of this instance by re-querying the cloud provider
         for its latest state.
         """
-        name = self.name
-        self._gce_image = self._provider.get_resource('images', self.id)
-        if not self._gce_image:
-            self._gce_image = {'name': name, 'status': 'UNKNOWN'}
+        image = self._provider.compute.images.get(self.id)
+        if image:
+            # pylint:disable=protected-access
+            self._gce_image = image._gce_image
+        else:
+            # image no longer exists
+            self._gce_image['status'] = MachineImageState.UNKNOWN
 
 
 class GCEInstance(BaseInstance):
@@ -1048,10 +1063,53 @@ class GCEInstance(BaseInstance):
         key in metadata.
         """
         try:
-            return next(iter(self._provider.security.key_pairs))
+            kp = next(iter(self._provider.security.key_pairs))
+            return kp.id if kp else None
         except StopIteration:
             return None
 
+    @key_pair_id.setter
+    # pylint:disable=arguments-differ
+    def key_pair_id(self, value):
+        self.assert_valid_resource_label(value)
+        key_pair = None
+        if not isinstance(value, GCEKeyPair):
+            key_pair = self._provider.security.key_pairs.get(value)
+        if key_pair:
+            key_pair_name = key_pair.name
+        kp = None
+        for kpi in self._provider.security.key_pairs._iter_gce_key_pairs(
+                self._provider):
+            if kpi.email == key_pair_name:
+                kp = kpi
+                break
+        if kp:
+            kp_items = [{
+                "key": "ssh-keys",
+                # FIXME: ssh username & key format are fixed here while they
+                # should correspond to the operating system, or be customizable
+                "value": "ubuntu:ssh-rsa {0} {1}".format(kp.public_key,
+                                                         kp.email)
+            }]
+            config = {
+                "items": kp_items,
+                "fingerprint": self._gce_instance['metadata']['fingerprint']
+            }
+            try:
+                (self._provider
+                    .gce_compute
+                    .instances()
+                    .setMetadata(project=self._provider.project_name,
+                                 zone=self._provider.default_zone,
+                                 instance=self.name,
+                                 body=config)
+                    .execute())
+            except Exception as e:
+                cb.log.warning('Exception while setting instance key pair: %s',
+                               e)
+                raise e
+            self.refresh()
+
     @property
     def inet_gateway(self):
         if self._inet_gateway:
@@ -1293,10 +1351,13 @@ class GCEInstance(BaseInstance):
         Refreshes the state of this instance by re-querying the cloud provider
         for its latest state.
         """
-        name = self.name
-        self._gce_instance = self._provider.get_resource('instances', self.id)
-        if not self._gce_instance:
-            self._gce_instance = {'name': name, 'status': 'UNKNOWN'}
+        inst = self._provider.compute.instances.get(self.id)
+        if inst:
+            # pylint:disable=protected-access
+            self._gce_instance = inst._gce_instance
+        else:
+            # instance no longer exists
+            self._gce_instance['status'] = InstanceState.UNKNOWN
 
     def add_vm_firewall(self, sg):
         tag = sg.name if isinstance(sg, GCEVMFirewall) else sg
@@ -1354,6 +1415,7 @@ class GCENetwork(BaseNetwork):
 
     @label.setter
     def label(self, value):
+        self.assert_valid_resource_label(value)
         tag_name = "_".join(["network", self.name, "label"])
         helpers.modify_or_add_metadata_item(self._provider, tag_name, value)
 
@@ -1440,9 +1502,13 @@ class GCENetwork(BaseNetwork):
             label, self, cidr_block, zone)
 
     def refresh(self):
-        self._network = self._provider.get_resource('networks', self.id)
-        if not self._network:
-            self._network = {'status': 'UNKNOWN'}
+        net = self._provider.networking.networks.get(self.id)
+        if net:
+            # pylint:disable=protected-access
+            self._network = net._network
+        else:
+            # network no longer exists
+            self._network['status'] = NetworkState.UNKNOWN
 
     @property
     def gateways(self):
@@ -1456,7 +1522,8 @@ class GCEFloatingIPContainer(BaseFloatingIPContainer):
 
     def get(self, floating_ip_id):
         fip = self._provider.get_resource('addresses', floating_ip_id)
-        return GCEFloatingIP(self._provider, fip) if fip else None
+        return (GCEFloatingIP(self._provider, self.gateway, fip)
+                if fip else None)
 
     def list(self, limit=None, marker=None):
         max_result = limit if limit is not None and limit < 500 else 500
@@ -1469,7 +1536,7 @@ class GCEFloatingIPContainer(BaseFloatingIPContainer):
                                   maxResults=max_result,
                                   pageToken=marker)
                             .execute())
-            ips = [GCEFloatingIP(self._provider, ip)
+            ips = [GCEFloatingIP(self._provider, self.gateway, ip)
                    for ip in response.get('items', [])]
             if len(ips) > max_result:
                 cb.log.warning('Expected at most %d results; got %d',
@@ -1504,8 +1571,9 @@ class GCEFloatingIPContainer(BaseFloatingIPContainer):
 class GCEFloatingIP(BaseFloatingIP):
     _DEAD_INSTANCE = 'dead instance'
 
-    def __init__(self, provider, floating_ip):
+    def __init__(self, provider, gateway, floating_ip):
         super(GCEFloatingIP, self).__init__(provider)
+        self._gateway = gateway
         self._ip = floating_ip
         self._process_ip_users()
 
@@ -1561,11 +1629,10 @@ class GCEFloatingIP(BaseFloatingIP):
         self._provider.wait_for_operation(response, region=self.region_name)
 
     def refresh(self):
-        self._ip = self._provider.get_resource('addresses', self.id)
-        if not self._ip:
-            self._ip = {'status': 'UNKNOWN'}
-        else:
-            self._process_ip_users()
+        fip = self.gateway.floating_ips.get(self.id)
+        # pylint:disable=protected-access
+        self._ip = fip._ip
+        self._process_ip_users()
 
     def _process_ip_users(self):
         self._rule = None
@@ -1642,9 +1709,13 @@ class GCERouter(BaseRouter):
         return parsed_url.parameters['region']
 
     def refresh(self):
-        self._router = self._provider.get_resource('routers', self.id)
-        if not self._router:
-            self._router = {'status': 'UNKNOWN'}
+        router = self._provider.networking.routers.get(self.id)
+        if router:
+            # pylint:disable=protected-access
+            self._router = router._router
+        else:
+            # router no longer exists
+            self._router['status'] = RouterState.UNKNOWN
 
     @property
     def state(self):
@@ -1774,30 +1845,14 @@ class GCESubnet(BaseSubnet):
 
     @property
     def label(self):
-        return self._subnet.get('description')
+        tag_name = "_".join(["subnet", self.name, "label"])
+        return helpers.get_metadata_item_value(self._provider, tag_name)
 
     @label.setter
     def label(self, value):
         self.assert_valid_resource_label(value)
-        request_body = {
-            'description': value.replace(' ', '_').lower(),
-            'fingerprint': self._subnet.get('fingerprint')
-        }
-        try:
-            (self._provider
-                 .gce_compute
-                 .subnetworks()
-                 .patch(project=self._provider.project_name,
-                        region=self.region_name,
-                        subnetwork=self.name,
-                        body=request_body)
-                 .execute())
-        except Exception as e:
-            cb.log.warning('Exception while setting subnet label: %s. '
-                           'Check for invalid characters in label. '
-                           'Should conform to RFC1035.', e)
-            raise e
-        self.refresh()
+        tag_name = "_".join(["subnet", self.name, "label"])
+        helpers.modify_or_add_metadata_item(self._provider, tag_name, value)
 
     @property
     def cidr_block(self):
@@ -1834,9 +1889,13 @@ class GCESubnet(BaseSubnet):
         return SubnetState.AVAILABLE
 
     def refresh(self):
-        self._subnet = self._provider.get_resource('subnetworks', self.id)
-        if not self._subnet:
-            self._subnet = {'status': 'UNKNOWN'}
+        subnet = self._provider.networking.subnets.get(self.id)
+        if subnet:
+            # pylint:disable=protected-access
+            self._subnet = subnet._subnet
+        else:
+            # subnet no longer exists
+            self._subnet['status'] = SubnetState.UNKNOWN
 
 
 class GCEVolume(BaseVolume):
@@ -2048,9 +2107,13 @@ class GCEVolume(BaseVolume):
         Refreshes the state of this volume by re-querying the cloud provider
         for its latest state.
         """
-        self._volume = self._provider.get_resource('disks', self.id)
-        if not self._volume:
-            self._volume = {'status': 'UNKNOWN'}
+        vol = self._provider.storage.volumes.get(self.id)
+        if vol:
+            # pylint:disable=protected-access
+            self._volume = vol._volume
+        else:
+            # volume no longer exists
+            self._volume['status'] = VolumeState.UNKNOWN
 
 
 class GCESnapshot(BaseSnapshot):
@@ -2153,9 +2216,13 @@ class GCESnapshot(BaseSnapshot):
         Refreshes the state of this snapshot by re-querying the cloud provider
         for its latest state.
         """
-        self._snapshot = self._provider.get_resource('snapshots', self.id)
-        if not self._snapshot:
-            self._snapshot = {'status': 'UNKNOWN'}
+        snap = self._provider.storage.snapshots.get(self.id)
+        if snap:
+            # pylint:disable=protected-access
+            self._snapshot = snap._snapshot
+        else:
+            # snapshot no longer exists
+            self._snapshot['status'] = SnapshotState.UNKNOWN
 
     def delete(self):
         """
@@ -2284,6 +2351,7 @@ class GCSObject(BaseBucketObject):
                                               url_encoded_signature))
 
     def refresh(self):
+        # pylint:disable=protected-access
         self._obj = self.bucket.objects.get(self.id)._obj
 
 

cloudbridge/cloud/providers/gce/services.py

@@ -252,6 +252,7 @@ class GCEVMFirewallService(BaseVMFirewallService):
         fw.rules.create_with_priority(
                 direction=TrafficDirection.OUTBOUND, protocol='tcp',
                 priority=65534, cidr='0.0.0.0/0')
+        fw.label = label
         return fw
 
     def find(self, name, limit=None, marker=None):
@@ -532,8 +533,7 @@ class GCEInstanceService(BaseInstanceService):
             'name': GCEInstance._generate_name_from_label(label, 'cb-inst'),
             'machineType': vm_type.resource_url,
             'disks': disks,
-            'networkInterfaces': [network_interface],
-            'labels': {'cblabel': label.replace(' ', '_').lower()}
+            'networkInterfaces': [network_interface]
         }
 
         if vm_firewalls and isinstance(vm_firewalls, list):
@@ -560,7 +560,11 @@ class GCEInstanceService(BaseInstanceService):
             return None
         instance_id = operation.get('targetLink')
         self.provider.wait_for_operation(operation, zone=zone_name)
-        return self.get(instance_id)
+        cb_inst = self.get(instance_id)
+        cb_inst.label = label
+        if key_pair:
+            cb_inst.key_pair_id = key_pair
+        return cb_inst
 
     def get(self, instance_id):
         """
@@ -714,6 +718,7 @@ class GCENetworkService(BaseNetworkService):
         False: For creating a custom mode VPC network. Subnetworks should be
                created manually.
         """
+        GCENetwork.assert_valid_resource_label(label)
         if create_subnetworks is not None and cidr_block is not None:
             cb.log.warning('cidr_block is ignored in non-legacy networks. '
                            'Auto mode networks use the default CIDR of '
@@ -737,13 +742,7 @@ class GCENetworkService(BaseNetworkService):
             if 'error' in response:
                 return None
             self.provider.wait_for_operation(response)
-            gce_net = (self.provider
-                       .gce_compute
-                       .networks()
-                       .get(project=self.provider.project_name,
-                            network=name)
-                       .execute())
-            return gce_net
+            return self.get(name)
         except googleapiclient.errors.HttpError as http_error:
             cb.log.warning('googleapiclient.errors.HttpError: %s', http_error)
             return None
@@ -753,9 +752,7 @@ class GCENetworkService(BaseNetworkService):
         Creates an auto mode VPC network with default subnets. It is possible
         to add additional subnets later.
         """
-        GCENetwork.assert_valid_resource_label(label)
-        gce_net = self._create(label, cidr_block, False)
-        cb_net = GCENetwork(self.provider, gce_net)
+        cb_net = self._create(label, cidr_block, False)
         cb_net.label = label
         return cb_net
 
@@ -800,14 +797,11 @@ class GCERouterService(BaseRouterService):
     def get(self, router_id):
         return self._get_in_region(router_id)
 
-    def find(self, name, limit=None, marker=None):
-        routers = []
-        for region in self.provider.compute.regions.list():
-            router = self._get_in_region(name, region.name)
-            if router:
-                routers.append(router)
-        return ClientPagedResultList(self.provider, routers, limit=limit,
-                                     marker=marker)
+    def find(self, **kwargs):
+        obj_list = self
+        filters = ['name', 'label']
+        matches = cb_helpers.generic_find(filters, kwargs, obj_list)
+        return ClientPagedResultList(self._provider, list(matches))
 
     def list(self, limit=None, marker=None):
         region = self.provider.region_name
@@ -947,8 +941,7 @@ class GCESubnetService(BaseSubnetService):
         body = {'ipCidrRange': cidr_block,
                 'name': name,
                 'network': network.resource_url,
-                'region': region_name,
-                'labels': {'cblabel': label.replace(' ', '_').lower()}
+                'region': region_name
                 }
         try:
             response = (self.provider
@@ -963,7 +956,9 @@ class GCESubnetService(BaseSubnetService):
                                response['error'])
                 return None
             self.provider.wait_for_operation(response, region=region_name)
-            return self.get(name)
+            cb_subnet = self.get(name)
+            cb_subnet.label = label
+            return cb_subnet
         except googleapiclient.errors.HttpError as http_error:
             cb.log.warning('googleapiclient.errors.HttpError: %s', http_error)
             return None
@@ -1126,7 +1121,6 @@ class GCEVolumeService(BaseVolumeService):
             'type': 'zones/{0}/diskTypes/{1}'.format(zone_name, 'pd-standard'),
             'sourceSnapshot': snapshot_id,
             'description': description,
-            'labels': {'cblabel': label.replace(' ', '_').lower()}
         }
         operation = (self.provider
                          .gce_compute
@@ -1136,7 +1130,9 @@ class GCEVolumeService(BaseVolumeService):
                              zone=zone_name,
                              body=disk_body)
                          .execute())
-        return self.get(operation.get('targetLink'))
+        cb_vol = self.get(operation.get('targetLink'))
+        cb_vol.label = label
+        return cb_vol
 
 
 class GCESnapshotService(BaseSnapshotService):
@@ -1204,8 +1200,7 @@ class GCESnapshotService(BaseSnapshotService):
         volume_name = volume.name if isinstance(volume, GCEVolume) else volume
         snapshot_body = {
             "name": name,
-            "description": description,
-            "labels": {"cblabel": label}
+            "description": description
         }
         operation = (self.provider
                          .gce_compute
@@ -1219,11 +1214,9 @@ class GCESnapshotService(BaseSnapshotService):
             return None
         self.provider.wait_for_operation(operation,
                                          zone=self.provider.default_zone)
-        snapshots = self.provider.storage.snapshots.find(label=label)
-        if snapshots:
-            return snapshots[0]
-        else:
-            return None
+        cb_snap = self.get(name)
+        cb_snap.label = label
+        return cb_snap
 
 
 class GCSBucketService(BaseBucketService):

setup.py

@@ -41,7 +41,7 @@ REQS_AZURE = ['msrest>=0.5.4,<0.6',
               'azure-storage-blob==1.3.1',
               'azure-cosmosdb-table==1.0.4',
               'pysftp==0.2.9']
-REQS_GCP = ['google-api-python-client', 'oauth2client', 'retrying']
+REQS_GCP = ['google-api-python-client', 'oauth2client']
 REQS_OPENSTACK = [
     'openstacksdk>=0.12.0,<=0.17',
     'python-novaclient>=7.0.0,<=11.0',