Merge branch 'master' into middleware

README.rst

@@ -51,7 +51,7 @@ Build Status Tests
 +--------------------------+--------------+--------------+
 | **Azure**                | |azure-py27| | |azure-py36| |
 +--------------------------+--------------+--------------+
-| **GCE (alpha)**          | |gce-py27|   | |gce-py36|   |
+| **GCE**                  | |gce-py27|   | |gce-py36|   |
 +--------------------------+--------------+--------------+
 
 Installation
@@ -81,7 +81,7 @@ exploring the API:
   print(provider.security.key_pairs.list())
 
 The exact same command (as well as any other CloudBridge method) will run with
-any of the supported providers: ``ProviderList.[AWS | AZURE | OPENSTACK]``!
+any of the supported providers: ``ProviderList.[AWS | AZURE | GCE | OPENSTACK]``!
 
 
 Citation

cloudbridge/cloud/providers/gce/helpers.py

@@ -143,3 +143,39 @@ def remove_metadata_item(provider, key):
 
     gce_metadata_save_op(provider, _remove_metadata_by_key)
     return True
+
+
+def __if_label_fingerprint_differs(e):
+    # return True if the CloudError exception is due to subnet being in use
+    if isinstance(e, HttpError):
+        expected_message = 'Labels fingerprint either invalid or ' \
+                           'resource labels have changed'
+        # str wrapper required for Python 2.7
+        if expected_message in str(e.content):
+            return True
+    return False
+
+
+@tenacity.retry(stop=tenacity.stop_after_attempt(10),
+                retry=tenacity.retry_if_exception(
+                    __if_label_fingerprint_differs),
+                wait=tenacity.wait_exponential(max=10),
+                reraise=True)
+def change_label(resource, key, value, res_att, request):
+    resource.assert_valid_resource_label(value)
+    labels = getattr(resource, res_att).get("labels", {})
+    print(labels)
+    labels[key] = value
+    request_body = {
+        "labels": labels,
+        "labelFingerprint":
+            getattr(resource, res_att).get('labelFingerprint'),
+    }
+    try:
+        request.body = str(request_body)
+        request.body_size = len(str(request_body))
+        response = request.execute()
+        resource._provider.wait_for_operation(response,
+                                              zone=resource.zone_name)
+    finally:
+        resource.refresh()

cloudbridge/cloud/providers/gce/resources.py

@@ -473,7 +473,6 @@ class GCEVMFirewall(BaseVMFirewall):
     def label(self):
         tag_name = "_".join(["firewall", self.name, "label"])
         return helpers.get_metadata_item_value(self._provider, tag_name)
-        # TODO: Add removing metadata to delete function
 
     @label.setter
     def label(self, value):
@@ -764,30 +763,14 @@ class GCEMachineImage(BaseMachineImage):
     @label.setter
     # pylint:disable=arguments-differ
     def label(self, value):
-        self.assert_valid_resource_label(value)
-        # Refresh to update fingerprint and current labels
-        self.refresh()
-        labels = self._gce_image.get('labels', {})
-        labels['cblabel'] = value
-        request_body = {
-            'labels': labels,
-            'labelFingerprint': self._gce_image.get('labelFingerprint'),
-        }
-        try:
-            response = (self._provider
-                        .gce_compute
-                        .images()
-                        .setLabels(project=self._provider.project_name,
-                                   resource=self.name,
-                                   body=request_body)
-                        .execute())
-            self._provider.wait_for_operation(response)
-        except Exception as e:
-            cb.log.warning('Exception while setting image label: %s. '
-                           'Check for invalid characters in label. '
-                           'Should conform to RFC1035.', e)
-            raise e
-        self.refresh()
+        req = (self._provider
+                   .gce_compute
+                   .images()
+                   .setLabels(project=self._provider.project_name,
+                              resource=self.name,
+                              body={}))
+
+        helpers.change_label(self, 'cblabel', value, '_gce_image', req)
 
     @property
     def description(self):
@@ -887,31 +870,15 @@ class GCEInstance(BaseInstance):
     @label.setter
     # pylint:disable=arguments-differ
     def label(self, value):
-        self.assert_valid_resource_label(value)
-        # Refresh to update fingerprint and current labels
-        self.refresh()
-        labels = self._gce_instance.get('labels', {})
-        labels['cblabel'] = value
-        request_body = {
-            'labels': labels,
-            'labelFingerprint': self._gce_instance.get('labelFingerprint'),
-        }
-        try:
-            response = (self._provider
-                        .gce_compute
-                        .instances()
-                        .setLabels(project=self._provider.project_name,
-                                   zone=self.zone_name,
-                                   instance=self.name,
-                                   body=request_body)
-                        .execute())
-            self._provider.wait_for_operation(response, zone=self.zone_name)
-        except Exception as e:
-            cb.log.warning('Exception while setting instance label: %s. '
-                           'Check for invalid characters in label. '
-                           'Should conform to RFC1035.', e)
-            raise e
-        self.refresh()
+        req = (self._provider
+                   .gce_compute
+                   .instances()
+                   .setLabels(project=self._provider.project_name,
+                              zone=self.zone_name,
+                              instance=self.name,
+                              body={}))
+
+        helpers.change_label(self, 'cblabel', value, '_gce_instance', req)
 
     @property
     def public_ips(self):
@@ -1092,42 +1059,6 @@ class GCEInstance(BaseInstance):
                     return item.get("value").split(" ")[-1]
         return None
 
-    @key_pair_id.setter
-    # pylint:disable=arguments-differ
-    def key_pair_id(self, value):
-        key_pair = value
-        if not isinstance(value, GCEKeyPair):
-            key_pair = self._provider.security.key_pairs.get(value)
-        if key_pair:
-            kp = key_pair._key_pair
-            kp_items = [{
-                "key": "ssh-keys",
-                # Format is not removed from public key portion
-                "value": "{}:{} {}".format(self._provider.vm_default_user_name,
-                                           kp.public_key,
-                                           kp.name)
-            }]
-            config = {
-                "items": kp_items,
-                "fingerprint": self._gce_instance['metadata']['fingerprint']
-            }
-            try:
-                response = (self._provider
-                            .gce_compute
-                            .instances()
-                            .setMetadata(project=self._provider.project_name,
-                                         zone=self.zone_name,
-                                         instance=self.name,
-                                         body=config)
-                            .execute())
-                self._provider.wait_for_operation(response,
-                                                  zone=self.zone_name)
-            except Exception as e:
-                cb.log.warning('Exception while setting instance key pair: %s',
-                               e)
-                raise e
-            self.refresh()
-
     @property
     def inet_gateway(self):
         if self._inet_gateway:
@@ -1878,31 +1809,15 @@ class GCEVolume(BaseVolume):
 
     @label.setter
     def label(self, value):
-        self.assert_valid_resource_label(value)
-        # Refresh to update fingerprint and current labels
-        self.refresh()
-        labels = self._volume.get('labels', {})
-        labels['cblabel'] = value
-        request_body = {
-            'labels': labels,
-            'labelFingerprint': self._volume.get('labelFingerprint'),
-        }
-        try:
-            response = (self._provider
-                        .gce_compute
-                        .disks()
-                        .setLabels(project=self._provider.project_name,
-                                   zone=self.zone_name,
-                                   resource=self.name,
-                                   body=request_body)
-                        .execute())
-            self._provider.wait_for_operation(response, zone=self.zone_name)
-        except Exception as e:
-            cb.log.warning('Exception while setting volume name: %s. '
-                           'Check for invalid characters in name. '
-                           'Should conform to RFC1035.', e)
-            raise e
-        self.refresh()
+        req = (self._provider
+                   .gce_compute
+                   .disks()
+                   .setLabels(project=self._provider.project_name,
+                              zone=self.zone_name,
+                              resource=self.name,
+                              body={}))
+
+        helpers.change_label(self, 'cblabel', value, '_volume', req)
 
     @property
     def description(self):
@@ -1913,31 +1828,15 @@ class GCEVolume(BaseVolume):
 
     @description.setter
     def description(self, value):
-        # Refresh to update fingerprint and current labels
-        self.refresh()
-        labels = self._volume.get('labels', {})
-        labels['description'] = value.replace(' ', '_').lower()
-        request_body = {
-            'labels': labels,
-            'labelFingerprint': self._volume.get('labelFingerprint'),
-        }
-        try:
-            response = (self._provider
-                        .gce_compute
-                        .disks()
-                        .setLabels(project=self._provider.project_name,
-                                   zone=self.zone_name,
-                                   resource=self.name,
-                                   body=request_body)
-                        .execute())
-            self._provider.wait_for_operation(response,
-                                              zone=self.zone_name)
-        except Exception as e:
-            cb.log.warning('Exception while setting volume description: %s. '
-                           'Check for invalid characters in description. '
-                           'Should confirm to RFC1035.', e)
-            raise e
-        self.refresh()
+        req = (self._provider
+               .gce_compute
+               .disks()
+               .setLabels(project=self._provider.project_name,
+                          zone=self.zone_name,
+                          resource=self.name,
+                          body={}))
+
+        helpers.change_label(self, 'description', value, '_volume', req)
 
     @property
     def size(self):
@@ -2108,30 +2007,14 @@ class GCESnapshot(BaseSnapshot):
     @label.setter
     # pylint:disable=arguments-differ
     def label(self, value):
-        self.assert_valid_resource_label(value)
-        # Refresh to update fingerprint and current labels
-        self.refresh()
-        labels = self._snapshot.get('labels', {})
-        labels['cblabel'] = value
-        request_body = {
-            'labels': labels,
-            'labelFingerprint': self._snapshot.get('labelFingerprint'),
-        }
-        try:
-            response = (self._provider
-                        .gce_compute
-                        .snapshots()
-                        .setLabels(project=self._provider.project_name,
-                                   resource=self.name,
-                                   body=request_body)
-                        .execute())
-            self._provider.wait_for_operation(response)
-        except Exception as e:
-            cb.log.warning('Exception while setting snapshot label: %s. '
-                           'Check for invalid characters in label. '
-                           'Should conform to RFC1035.', e)
-            raise e
-        self.refresh()
+        req = (self._provider
+                   .gce_compute
+                   .snapshots()
+                   .setLabels(project=self._provider.project_name,
+                              resource=self.name,
+                              body={}))
+
+        helpers.change_label(self, 'cblabel', value, '_snapshot', req)
 
     @property
     def description(self):
@@ -2142,29 +2025,14 @@ class GCESnapshot(BaseSnapshot):
 
     @description.setter
     def description(self, value):
-        # Refresh to update fingerprint and current labels
-        self.refresh()
-        labels = self._snapshot.get('labels', {})
-        labels['description'] = value.replace(' ', '_').lower()
-        request_body = {
-            'labels': labels,
-            'labelFingerprint': self._snapshot.get('labelFingerprint'),
-        }
-        try:
-            response = (self._provider
-                        .gce_compute
-                        .snapshots()
-                        .setLabels(project=self._provider.project_name,
-                                   resource=self.name,
-                                   body=request_body)
-                        .execute())
-            self._provider.wait_for_operation(response)
-        except Exception as e:
-            cb.log.warning('Exception while setting volume description: %s. '
-                           'Check for invalid characters in description. '
-                           'Should confirm to RFC1035.', e)
-            raise e
-        self.refresh()
+        req = (self._provider
+               .gce_compute
+               .snapshots()
+               .setLabels(project=self._provider.project_name,
+                          resource=self.name,
+                          body={}))
+
+        helpers.change_label(self, 'description', value, '_snapshot', req)
 
     @property
     def size(self):

cloudbridge/cloud/providers/gce/services.py

@@ -1,3 +1,4 @@
+import ipaddress
 import json
 import logging
 import time
@@ -446,6 +447,32 @@ class GCEInstanceService(BaseInstanceService):
                 config['tags'] = {}
                 config['tags']['items'] = vm_firewall_names
 
+        if user_data:
+            entry = {'key': 'user-data', 'value': user_data}
+            config['metadata'] = {'items': [entry]}
+
+        if key_pair:
+            if not isinstance(key_pair, GCEKeyPair):
+                key_pair = self._provider.security.key_pairs.get(key_pair)
+            if key_pair:
+                kp = key_pair._key_pair
+                kp_entry = {
+                    "key": "ssh-keys",
+                    # Format is not removed from public key portion
+                    "value": "{}:{} {}".format(
+                        self.provider.vm_default_user_name,
+                        kp.public_key,
+                        kp.name)
+                    }
+                meta = config.get('metadata', {})
+                if meta:
+                    items = meta.get('items', [])
+                    items.append(kp_entry)
+                else:
+                    config['metadata'] = {'items': [kp_entry]}
+
+        config['labels'] = {'cblabel': label}
+
         operation = (self.provider
                          .gce_compute.instances()
                          .insert(project=self.provider.project_name,
@@ -455,9 +482,6 @@ class GCEInstanceService(BaseInstanceService):
         instance_id = operation.get('targetLink')
         self.provider.wait_for_operation(operation, zone=zone_name)
         cb_inst = self.get(instance_id)
-        cb_inst.label = label
-        if key_pair:
-            cb_inst.key_pair_id = key_pair
         return cb_inst
 
     def get(self, instance_id):
@@ -749,7 +773,7 @@ class GCESubnetService(BaseSubnetService):
             filter = 'network eq %s' % network.resource_url
         region_names = []
         if zone:
-            region_names.append(self._zone_to_region_name(zone))
+            region_names.append(self._zone_to_region(zone))
         else:
             for r in self.provider.compute.regions:
                 region_names.append(r.name)
@@ -779,7 +803,7 @@ class GCESubnetService(BaseSubnetService):
         """
         GCESubnet.assert_valid_resource_label(label)
         name = GCESubnet._generate_name_from_label(label, 'cbsubnet')
-        region_name = self._zone_to_region_name(zone)
+        region_name = self._zone_to_region(zone)
 #         for subnet in self.iter(network=network):
 #            if BaseNetwork.cidr_blocks_overlap(subnet.cidr_block, cidr_block):
 #                 if subnet.region_name != region_name:
@@ -811,20 +835,42 @@ class GCESubnetService(BaseSubnetService):
 
     def get_or_create_default(self, zone):
         """
-        Every GCP project comes with a default auto mode VPC network. An auto
-        mode VPC network has exactly one subnetwork per region. This method
-        returns the subnetwork of the default network that spans the given
-        zone.
+        Return an existing or create a new subnet for the supplied zone.
+
+        In GCP, subnets are a regional resource so a single subnet can services
+        an entire region. The supplied zone parameter is used to derive the
+        parent region under which the default subnet then exists.
         """
-        sn = self.find(label=GCESubnet.CB_DEFAULT_SUBNET_LABEL)
-        if sn:
-            return sn[0]
-        # No default subnet look for default network, then create subnet
+        # In case the supplied zone param is `None`, resort to the default one
+        region = self._zone_to_region(zone or self.provider.default_zone,
+                                      return_name_only=False)
+        # Check if a default subnet already exists for the given region/zone
+        for sn in self.find(label=GCESubnet.CB_DEFAULT_SUBNET_LABEL):
+            if sn.region == region.id:
+                return sn
+        # No default subnet in the supplied zone. Look for a default network,
+        # then create a subnet whose address space does not overlap with any
+        # other existing subnets. If there are existing subnets, this process
+        # largely assumes the subnet address spaces are contiguous when it
+        # does the calculations (e.g., 10.0.0.0/24, 10.0.1.0/24).
+        cidr_block = GCESubnet.CB_DEFAULT_SUBNET_IPV4RANGE
         net = self.provider.networking.networks.get_or_create_default()
+        if net.subnets:
+            max_sn = net.subnets[0]
+            # Find the maximum address subnet address space within the network
+            for esn in net.subnets:
+                if (ipaddress.ip_network(esn.cidr_block) >
+                        ipaddress.ip_network(max_sn.cidr_block)):
+                    max_sn = esn
+            max_sn_ipa = ipaddress.ip_network(max_sn.cidr_block)
+            # Find the next available subnet after the max one, based on the
+            # max subnet size
+            next_sn_address = (
+                next(max_sn_ipa.hosts()) + max_sn_ipa.num_addresses - 1)
+            cidr_block = "{}/{}".format(next_sn_address, max_sn_ipa.prefixlen)
         sn = self.provider.networking.subnets.create(
                 label=GCESubnet.CB_DEFAULT_SUBNET_LABEL,
-                cidr_block=GCESubnet.CB_DEFAULT_SUBNET_IPV4RANGE,
-                network=net, zone=zone)
+                cidr_block=cidr_block, network=net, zone=zone)
         router = self.provider.networking.routers.get_or_create_default(net)
         router.attach_subnet(sn)
         gateway = net.gateways.get_or_create_inet_gateway()
@@ -846,14 +892,24 @@ class GCESubnetService(BaseSubnetService):
             log.warning('No label was found associated with this subnet '
                         '"{}" when deleted.'.format(subnet.name))
 
-    def _zone_to_region_name(self, zone):
+    def _zone_to_region(self, zone, return_name_only=True):
+        """
+        Given a GCE zone, return parent region.
+
+        Supplied `zone` param can be a `str` or `GCEPlacementZone`.
+
+        If ``return_name_only`` is set, return the region name as a string;
+        otherwise, return a GCERegion object.
+        """
+        region_name = self.provider.region_name
         if zone:
-            if not isinstance(zone, GCEPlacementZone):
-                zone = GCEPlacementZone(
-                    self.provider,
-                    self.provider.get_resource('zones', zone))
-            return zone.region_name
-        return self.provider.region_name
+            if isinstance(zone, GCEPlacementZone):
+                region_name = zone.region_name
+            else:
+                region_name = zone[:-2]
+        if return_name_only:
+            return region_name
+        return self.provider.compute.regions.get(region_name)
 
 
 class GCPStorageService(BaseStorageService):
@@ -973,6 +1029,7 @@ class GCEVolumeService(BaseVolumeService):
             'type': 'zones/{0}/diskTypes/{1}'.format(zone_name, 'pd-standard'),
             'sourceSnapshot': snapshot_id,
             'description': description,
+            'labels': {'cblabel': label}
         }
         operation = (self.provider
                          .gce_compute
@@ -983,7 +1040,6 @@ class GCEVolumeService(BaseVolumeService):
                              body=disk_body)
                          .execute())
         cb_vol = self.get(operation.get('targetLink'))
-        cb_vol.label = label
         return cb_vol
 
 
@@ -1052,7 +1108,8 @@ class GCESnapshotService(BaseSnapshotService):
         volume_name = volume.name if isinstance(volume, GCEVolume) else volume
         snapshot_body = {
             "name": name,
-            "description": description
+            "description": description,
+            "labels": {'cblabel': label}
         }
         operation = (self.provider
                          .gce_compute
@@ -1067,7 +1124,6 @@ class GCESnapshotService(BaseSnapshotService):
         self.provider.wait_for_operation(operation,
                                          zone=self.provider.default_zone)
         cb_snap = self.get(name)
-        cb_snap.label = label
         return cb_snap