Correctly parse launch config of a new instance

Changes:
- Launcing an instance with a launch config does not work. It is fixed.
- If a subnet is given, launch the instance in the given subnet.
- provider.get_resource does not work for project, region, and zone, which is
  fixed now. Also, provided a way to override default values.

Test stats after this CL
========================
an 58 tests in 1599.534s

FAILED (SKIP=1, errors=24, failures=6)

cloudbridge/cloud/providers/gce/provider.py

@@ -2,6 +2,7 @@
 Provider implementation based on google-api-python-client library
 for GCE.
 """
+import copy
 import json
 import logging
 import os
@@ -167,7 +168,8 @@ class GCPResources(object):
                 out.parameters[parameter] = m.group(index + 1)
             return out
 
-    def get_resource_url_with_default(self, resource, url_or_name):
+    def get_resource_url_with_default(self, resource, url_or_name,
+                                      project=None, region=None, zone=None):
         """
         Build a GCPResourceUrl from a service's name and resource url or name.
         If the url_or_name is a valid GCP resource URL, then we build the
@@ -181,9 +183,15 @@ class GCPResources(object):
         # Otherwise, construct resource URL with default values.
         if resource not in self._resources:
             return None
+
+        parameter_defaults = copy.copy(self._parameter_defaults)
+        if region:
+            parameter_defaults['region'] = region
+        if zone:
+            parameter_defaults['zone'] = zone
         parsed_url = GCPResourceUrl(resource, self._connection)
         for key in self._resources[resource]['parameters']:
-            parsed_url.parameters[key] = self._parameter_defaults.get(
+            parsed_url.parameters[key] = parameter_defaults.get(
                 key, url_or_name)
         return parsed_url
 
@@ -308,12 +316,13 @@ class GCECloudProvider(BaseCloudProvider):
         out = self._compute_resources.parse_url(url)
         return out if out else self._storage_resources.parse_url(url)
 
-    def get_resource(self, resource, url_or_name):
+    def get_resource(self, resource, url_or_name, project=None, region=None,
+                     zone=None):
         resource_url = (
             self._compute_resources.get_resource_url_with_default(
-                resource, url_or_name) or
+                resource, url_or_name, project, region, zone) or
             self._storage_resources.get_resource_url_with_default(
-                resource, url_or_name))
+                resource, url_or_name, project, region, zone))
         if resource_url is None:
             return None
         try:

cloudbridge/cloud/providers/gce/services.py

@@ -329,7 +329,8 @@ class GCERegionService(BaseRegionService):
         super(GCERegionService, self).__init__(provider)
 
     def get(self, region_id):
-        region = self.provider.get_resource('regions', region_id)
+        region = self.provider.get_resource('regions', region_id,
+                                            region=region_id)
         return GCERegion(self.provider, region) if region else None
 
     def list(self, limit=None, marker=None):
@@ -436,47 +437,99 @@ class GCEInstanceService(BaseInstanceService):
         GCEInstance.assert_valid_resource_name(name)
         if not zone:
             zone = self.provider.default_zone
-        if not isinstance(image, GCEMachineImage):
-            image = self.provider.compute.images.get(image)
         if not isinstance(vm_type, GCEVMType):
             vm_type = self.provider.compute.vm_types.get(vm_type)
-        if not launch_config:
-            if subnet:
-                network = self.provider.networking.networks.get(
-                        subnet.network_id)
-                network_url = (network.resource_url
-                               if isinstance(network, GCENetwork) else network)
-            else:
-                network_url = 'global/networks/default'
-            config = {
-                'name': name,
-                'machineType': vm_type.resource_url,
-                'disks': [{'boot': True,
-                           'autoDelete': True,
-                           'initializeParams': {
-                               'sourceImage': image.resource_url,
-                           }
-                           }],
-                'networkInterfaces': [
-                    {
-                        # TODO: Should replace network below with subnetwork
-                        'network': network_url,
-                        'accessConfigs': [{'type': 'ONE_TO_ONE_NAT',
-                                           'name': 'External NAT'}]
-                    }],
-            }
-            if vm_firewalls and isinstance(vm_firewalls, list):
-                vm_firewall_names = []
-                if isinstance(vm_firewalls[0], VMFirewall):
-                    vm_firewall_names = [f.name for f in vm_firewalls]
-                elif isinstance(vm_firewalls[0], str):
-                    vm_firewall_names = vm_firewalls
-                if len(vm_firewall_names) > 0:
-                    config['tags'] = {}
-                    config['tags']['items'] = vm_firewall_names
+
+        network_interface = {'accessConfigs': [{'type': 'ONE_TO_ONE_NAT',
+                                                'name': 'External NAT'}]}
+        if subnet:
+            network_interface['subnetwork'] = subnet.id
         else:
-            config = launch_config
+            network_interface['network'] = 'global/networks/default'
+
+        num_roots = 0
+        disks = []
+        boot_disk = None
+        if isinstance(launch_config, GCELaunchConfig):
+            for disk in launch_config.block_devices:
+                if not disk.source:
+                    volume_name = 'disk-{0}'.format(uuid.uuid4())
+                    volume_size = disk.size if disk.size else 1
+                    volume = self.provider.storage.volumes.create(
+                        volume_name, volume_size, zone)
+                    volume.wait_till_ready()
+                    source_field = 'source'
+                    source_value = volume.id
+                elif isinstance(disk.source, GCEMachineImage):
+                    source_field = 'initializeParams'
+                    # Explicitly set diskName; otherwise, instance name will be
+                    # used by default which may collide with existing disks.
+                    source_value = {
+                        'sourceImage': disk.source.id,
+                        'diskName': 'image-disk-{0}'.format(uuid.uuid4())}
+                elif isinstance(disk.source, GCEVolume):
+                    source_field = 'source'
+                    source_value = disk.source.id
+                elif isinstance(disk.source, GCESnapshot):
+                    volume = disk.source.create_volume(zone, size=disk.size)
+                    volume.wait_till_ready()
+                    source_field = 'source'
+                    source_value = volume.id
+                else:
+                    cb.log.warning('Unknown disk source')
+                    continue
+                autoDelete = True
+                if disk.delete_on_terminate is not None:
+                    autoDelete = disk.delete_on_terminate
+                num_roots += 1 if disk.is_root else 0
+                if disk.is_root and not boot_disk:
+                    boot_disk = {'boot': True,
+                                 'autoDelete': autoDelete,
+                                 source_field: source_value}
+                else:
+                    disks.append({'boot': False,
+                                  'autoDelete': autoDelete,
+                                  source_field: source_value})
+
+        if num_roots > 1:
+            cb.log.warning('The launch config contains %d boot disks. Will '
+                           'use the first one', num_roots)
+        if image:
+            if boot_disk:
+                cb.log.warning('A boot image is given while the launch config '
+                               'contains a boot disk, too. The launch config '
+                               'will be used')
+            else:
+                if not isinstance(image, GCEMachineImage):
+                    image = self.provider.compute.images.get(image)
+                boot_disk = {'boot': True,
+                             'autoDelete': True,
+                             'initializeParams': {'sourceImage': image.id}}
+
+        if not boot_disk:
+            cb.log.warning('No boot disk is given')
+            return None
+        # The boot disk must be the first disk attached to the instance.
+        disks.insert(0, boot_disk)
+
+        config = {
+            'name': name,
+            'machineType': vm_type.resource_url,
+            'disks': disks,
+            'networkInterfaces': [network_interface]
+        }
+
+        if vm_firewalls and isinstance(vm_firewalls, list):
+            vm_firewall_names = []
+            if isinstance(vm_firewalls[0], VMFirewall):
+                vm_firewall_names = [f.name for f in vm_firewalls]
+            elif isinstance(vm_firewalls[0], str):
+                vm_firewall_names = vm_firewalls
+            if len(vm_firewall_names) > 0:
+                config['tags'] = {}
+                config['tags']['items'] = vm_firewall_names
         try:
+            cb.log.warning('config: %s', config)
             operation = (self.provider
                              .gce_compute.instances()
                              .insert(project=self.provider.project_name,

test/test_compute_service.py

@@ -272,14 +272,18 @@ class CloudComputeServiceTestCase(ProviderTestBase):
                     "vm_type")
                 vm_type = self.provider.compute.vm_types.find(
                     name=vm_type_name)[0]
-                for _ in range(vm_type.num_ephemeral_disks):
+                # Some providers, e.g. GCP, has a limit on total number of
+                # attached disks; it does not matter how many of them are
+                # ephemeral or persistent. So, wee keep in mind that we have
+                # attached 4 disks already, and add ephemeral disks accordingly
+                # to not exceed the limit.
+                for _ in range(vm_type.num_ephemeral_disks - 4):
                     lc.add_ephemeral_device()
 
                 net, subnet = helpers.create_test_network(self.provider, name)
 
                 with helpers.cleanup_action(lambda:
                                             helpers.delete_test_network(net)):
-
                     inst = helpers.create_test_instance(
                         self.provider,
                         name,