Merge pull request #26 from chiniforooshan/network_part2

Merge master into gce and resolve network id issues

README.rst

@@ -55,6 +55,15 @@ exploring the API:
   print(provider.security.key_pairs.list())
 
 
+Citation
+~~~~~~~~
+
+N. Goonasekera, A. Lonie, J. Taylor, and E. Afgan,
+"CloudBridge: a Simple Cross-Cloud Python Library,"
+presented at the Proceedings of the XSEDE16 Conference on Diversity, Big Data, and Science at Scale, Miami, USA, 2016.
+DOI: http://dx.doi.org/10.1145/2949550.2949648
+
+
 Documentation
 ~~~~~~~~~~~~~
 Documentation can be found at https://cloudbridge.readthedocs.org.

cloudbridge/cloud/base/provider.py

@@ -1,15 +1,25 @@
-"""
-Base implementation of a provider interface
-"""
+"""Base implementation of a provider interface."""
 import os
+try:
+    from configparser import SafeConfigParser
+except ImportError:  # Python 2
+    from ConfigParser import SafeConfigParser
+from os.path import expanduser
 
 from cloudbridge.cloud.interfaces import CloudProvider
 from cloudbridge.cloud.interfaces.resources import Configuration
+from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 
 DEFAULT_RESULT_LIMIT = 50
 DEFAULT_WAIT_TIMEOUT = 600
 DEFAULT_WAIT_INTERVAL = 5
 
+# By default, use two locations for CloudBridge configuration
+CloudBridgeConfigPath = '/etc/cloudbridge.ini'
+CloudBridgeConfigLocations = [CloudBridgeConfigPath]
+UserConfigPath = os.path.join(expanduser('~'), '.cloudbridge')
+CloudBridgeConfigLocations.append(UserConfigPath)
+
 
 class BaseConfiguration(Configuration):
 
@@ -61,6 +71,8 @@ class BaseCloudProvider(CloudProvider):
 
     def __init__(self, config):
         self._config = BaseConfiguration(config)
+        self._config_parser = SafeConfigParser()
+        self._config_parser.read(CloudBridgeConfigLocations)
 
     @property
     def config(self):
@@ -70,6 +82,19 @@ class BaseCloudProvider(CloudProvider):
     def name(self):
         return str(self.__class__.__name__)
 
+    def authenticate(self):
+        """
+        A basic implementation which simply runs a low impact command to
+        check whether cloud credentials work. Providers should override with
+        more efficient implementations.
+        """
+        try:
+            self.security.key_pairs.list()
+            return True
+        except Exception as e:
+            raise ProviderConnectionException(
+                "Authentication with cloud provider failed: %s" % (e,))
+
     def has_service(self, service_type):
         """
         Checks whether this provider supports a given service.
@@ -100,9 +125,11 @@ class BaseCloudProvider(CloudProvider):
 
         :return: a configuration value for the supplied ``key``
         """
-        if isinstance(self.config, dict):
+        if isinstance(self.config, dict) and self.config.get(key):
             return self.config.get(key, default_value)
-        else:
-            return getattr(self.config, key) if hasattr(
-                self.config, key) and getattr(self.config, key) else \
-                default_value
+        elif hasattr(self.config, key) and getattr(self.config, key):
+            return getattr(self.config, key)
+        elif (self._config_parser.has_option(self.PROVIDER_ID, key) and
+              self._config_parser.get(self.PROVIDER_ID, key)):
+            return self._config_parser.get(self.PROVIDER_ID, key)
+        return default_value

cloudbridge/cloud/base/resources.py

@@ -1,7 +1,17 @@
 """
 Base implementation for data objects exposed through a provider or service
 """
-from cloudbridge.cloud.interfaces.resources \
+import inspect
+import itertools
+import json
+import logging
+import os
+import shutil
+import time
+
+import six
+
+from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
 from cloudbridge.cloud.interfaces.resources import AttachmentInfo
 from cloudbridge.cloud.interfaces.resources import Bucket
@@ -30,15 +40,7 @@ from cloudbridge.cloud.interfaces.resources import Subnet
 from cloudbridge.cloud.interfaces.resources import FloatingIP
 from cloudbridge.cloud.interfaces.resources import Volume
 from cloudbridge.cloud.interfaces.resources import VolumeState
-from cloudbridge.cloud.interfaces.resources import WaitStateException
-import inspect
-import itertools
-import json
-import logging
-import shutil
-import time
-
-import six
+from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 
 
 log = logging.getLogger(__name__)
@@ -268,7 +270,6 @@ class BaseLaunchConfig(LaunchConfig):
     def __init__(self, provider):
         self.provider = provider
         self.block_devices = []
-        self.network_interfaces = []
 
     class BlockDeviceMapping(object):
         """
@@ -324,9 +325,6 @@ class BaseLaunchConfig(LaunchConfig):
             is_volume=True, source=source, is_root=is_root, size=size,
             delete_on_terminate=delete_on_terminate)
 
-    def add_network_interface(self, net_id):
-        self.network_interfaces.append(net_id)
-
 
 class BaseMachineImage(
         BaseCloudResource, BaseObjectLifeCycleMixin, MachineImage):
@@ -519,8 +517,8 @@ class BaseSecurityGroup(SecurityGroup, BaseCloudResource):
         return self._security_group.delete()
 
     def __repr__(self):
-        return "<CB-{0}: {1}>".format(self.__class__.__name__,
-                                      self.id)
+        return "<CB-{0}: {1} ({2})>".format(self.__class__.__name__,
+                                            self.id, self.name)
 
 
 class BaseSecurityGroupRule(SecurityGroupRule, BaseCloudResource):
@@ -643,6 +641,9 @@ class BaseBucket(BasePageableObjectMixin, Bucket, BaseCloudResource):
 
 class BaseNetwork(BaseCloudResource, Network, BaseObjectLifeCycleMixin):
 
+    CB_DEFAULT_NETWORK_NAME = os.environ.get('CB_DEFAULT_NETWORK_NAME',
+                                             'CloudBridgeNet')
+
     def __init__(self, provider):
         super(BaseNetwork, self).__init__(provider)
 

cloudbridge/cloud/base/services.py

@@ -146,6 +146,8 @@ class BaseNetworkService(
         super(BaseNetworkService, self).__init__(provider)
 
     def delete(self, network_id):
+        if network_id is None:
+            return True
         network = self.get(network_id)
         if network:
             network.delete()

cloudbridge/cloud/interfaces/__init__.py

@@ -5,10 +5,10 @@ from .provider import CloudProvider  # noqa
 from .provider import TestMockHelperMixin  # noqa
 from .resources import CloudServiceType  # noqa
 from .resources import InstanceState  # noqa
-from .resources import InvalidConfigurationException  # noqa
 from .resources import LaunchConfig  # noqa
 from .resources import MachineImageState  # noqa
 from .resources import NetworkState  # noqa
 from .resources import Region  # noqa
 from .resources import SnapshotState  # noqa
 from .resources import VolumeState  # noqa
+from .exceptions import InvalidConfigurationException  # noqa

cloudbridge/cloud/interfaces/exceptions.py

@@ -0,0 +1,38 @@
+"""
+Specification for exceptions raised by a provider
+"""
+
+
+class CloudBridgeBaseException(Exception):
+    """
+    Base class for all CloudBridge exceptions
+    """
+    pass
+
+
+class WaitStateException(CloudBridgeBaseException):
+    """
+    Marker interface for object wait exceptions.
+    Thrown when a timeout or errors occurs waiting for an object does not reach
+    the expected state within a specified time limit.
+    """
+    pass
+
+
+class InvalidConfigurationException(CloudBridgeBaseException):
+    """
+    Marker interface for invalid launch configurations.
+    Thrown when a combination of parameters in a LaunchConfig
+    object results in an illegal state.
+    """
+    pass
+
+
+class ProviderConnectionException(CloudBridgeBaseException):
+    """
+    Marker interface for connection errors to a cloud provider.
+    Thrown when cloudbridge is unable to connect with a provider,
+    for example, when credentials are incorrect, or connection
+    settings are invalid.
+    """
+    pass

cloudbridge/cloud/interfaces/provider.py

@@ -54,6 +54,32 @@ class CloudProvider(object):
                   configuration properties.
         """
 
+    @abstractmethod
+    def authenticate(self):
+        """
+        Checks whether a provider can be successfully authenticated with the
+        configured settings. Clients are *not* required to call this method
+        prior to accessing provider services, as most cloud connections are
+        initialized lazily. The authenticate() method will return True if
+        cloudbridge can establish a successful connection to the provider.
+        It will raise an exception with the appropriate error details
+        otherwise.
+
+        Example:
+
+        .. code-block:: python
+
+            try:
+                if provider.authenticate():
+                   print("Provider connection successful")
+            except ProviderConnectionException as e:
+                print("Could not authenticate with provider: %s" % (e, ))
+
+        :rtype: :class:`bool`
+        :return: ``True`` if authentication is successful.
+        """
+        pass
+
     @abstractmethod
     def has_service(self, service_type):
         """

cloudbridge/cloud/interfaces/resources.py

@@ -52,34 +52,6 @@ class CloudResource(object):
         pass
 
 
-class CloudBridgeBaseException(Exception):
-
-    """
-    Base class for all CloudBridge exceptions
-    """
-    pass
-
-
-class WaitStateException(CloudBridgeBaseException):
-
-    """
-    Marker interface for object wait exceptions.
-    Thrown when a timeout or errors occurs waiting for an object does not reach
-    the expected state within a specified time limit.
-    """
-    pass
-
-
-class InvalidConfigurationException(CloudBridgeBaseException):
-
-    """
-    Marker interface for invalid launch configurations.
-    Thrown when a combination of parameters in a LaunchConfig
-    object results in an illegal state.
-    """
-    pass
-
-
 class Configuration(dict):
     """
     Represents a cloudbridge configuration object
@@ -632,9 +604,11 @@ class MachineImageState(object):
 
 class LaunchConfig(object):
     """
-    Represents an advanced launch configuration object, containing
-    information such as BlockDeviceMappings, NetworkInterface configurations,
-    and other advanced options which may be useful when launching an instance.
+    Represents an advanced launch configuration object.
+
+    Theis object can contain information such as BlockDeviceMappings
+    configurations, and other advanced options which may be useful when
+    launching an instance.
 
     Example:
 
@@ -642,10 +616,9 @@ class LaunchConfig(object):
 
         lc = provider.compute.instances.create_launch_config()
         lc.add_block_device(...)
-        lc.add_network_interface(...)
 
         inst = provider.compute.instances.create(name, image, instance_type,
-                                               launch_config=lc)
+                                                 network, launch_config=lc)
     """
 
     @abstractmethod
@@ -737,34 +710,6 @@ class LaunchConfig(object):
         """
         pass
 
-    @abstractmethod
-    def add_network_interface(self, net_id):
-        """
-        Add a private network info to the launch configuration.
-
-        Example:
-
-        .. code-block:: python
-
-            lc = provider.compute.instances.create_launch_config()
-
-            # 1. Add a VPC subnet for use with AWS
-            lc.add_network_interface('subnet-c24aeaff')
-
-            # 2. Add a network ID for use with OpenStack
-            lc.add_network_interface('5820c766-75fe-4fc6-96ef-798f67623238')
-
-        :type net_id: ``str``
-        :param net_id: Network ID to launch an instance into. This is a
-                       preliminary implementation (pending full private cloud
-                       support within CloudBridge) so native network IDs need
-                       to be supplied. For OpenStack, this is the Neutron
-                       network ID. For AWS, this is a VPC subnet ID. For the
-                       time being, only a single network interface can be
-                       supplied.
-        """
-        pass
-
 
 class MachineImage(ObjectLifeCycleMixin, CloudResource):
 
@@ -1844,6 +1789,16 @@ class SecurityGroup(CloudResource):
         """
         pass
 
+    @abstractproperty
+    def network_id(self):
+        """
+        Network ID with which this security group is associated.
+
+        :rtype: ``str``
+        :return: Provider-supplied network ID or ``None`` is not available.
+        """
+        pass
+
     @abstractproperty
     def rules(self):
         """

cloudbridge/cloud/interfaces/services.py

@@ -200,7 +200,7 @@ class InstanceService(PageableObjectMixin, CloudService):
         pass
 
     @abstractmethod
-    def create(self, name, image, instance_type, zone=None,
+    def create(self, name, image, instance_type, network=None, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None,
                **kwargs):
@@ -218,6 +218,20 @@ class InstanceService(PageableObjectMixin, CloudService):
         :param instance_type: The InstanceType or name, specifying the size of
                               the instance to boot into
 
+        :type  network:  ``Network`` or ``str``
+        :param network:  The Network or an ID with which the instance should
+                         be associated. If no network was specified, this
+                         method will attempt to find a 'default' one and launch
+                         the instance using that network. A 'default' network
+                         is one tagged as such by the native API. If such tag
+                         or functionality does not exist, an attempt to create
+                         a new network (by default called 'CloudBridgeNet')
+                         will be made. If that falls through, an attempt will
+                         be made to launch the instance without specifying the
+                         network parameter (this is under the assumption the
+                         private networking functionality is not available on
+                         the provider).
+
         :type  zone: ``Zone`` or ``str``
         :param zone: The Zone or its name, where the instance should be placed.
 
@@ -238,7 +252,7 @@ class InstanceService(PageableObjectMixin, CloudService):
         :type  launch_config: ``LaunchConfig`` object
         :param launch_config: A ``LaunchConfig`` object which
                describes advanced launch configuration options for an instance.
-               This includes block_device_mappings and network_interfaces. To
+               Currently, this includes only block_device_mappings. To
                construct a launch configuration object, call
                provider.compute.instances.create_launch_config()
 
@@ -935,7 +949,7 @@ class SecurityGroupService(PageableObjectMixin, CloudService):
         pass
 
     @abstractmethod
-    def create(self, name, description, network_id=None):
+    def create(self, name, description, network_id):
         """
         Create a new SecurityGroup.
 
@@ -946,9 +960,7 @@ class SecurityGroupService(PageableObjectMixin, CloudService):
         :param description: The description of the new security group.
 
         :type  network_id: ``str``
-        :param network_id: An optional network ID under which to create the
-                           security group that may be supported by some
-                           providers.
+        :param network_id: Network ID under which to create the security group.
 
         :rtype: ``object`` of :class:`.SecurityGroup`
         :return:  A SecurityGroup instance or ``None`` if one was not created.

cloudbridge/cloud/providers/aws/provider.py

@@ -178,29 +178,27 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
     "family": "General Purpose",
     "enhanced_networking": false,
     "vCPU": 1,
-    "generation": "previous",
+    "generation": "current",
     "ebs_iops": 0,
     "network_performance": "Low",
     "ebs_throughput": 0,
     "vpc": {
-      "ips_per_eni": 4,
+      "ips_per_eni": 2,
       "max_enis": 2
     },
     "arch": [
-      "i386",
       "x86_64"
     ],
-    "linux_virtualization_types": [],
+    "linux_virtualization_types": [
+        "HVM"
+    ],
     "ebs_optimized": false,
-    "storage": {
-      "ssd": false,
-      "devices": 1,
-      "size": 160
-    },
+    "storage": null,
     "max_bandwidth": 0,
-    "instance_type": "t1.micro",
-    "ECU": 1.0,
-    "memory": 1.7
+    "instance_type": "t2.nano",
+    "ECU": "variable,
+    "memory": 0.5,
+    "ebs_max_bandwidth": 0
   }
 ]
 """

cloudbridge/cloud/providers/aws/resources.py

@@ -608,6 +608,10 @@ class AWSSecurityGroup(BaseSecurityGroup):
     def __init__(self, provider, security_group):
         super(AWSSecurityGroup, self).__init__(provider, security_group)
 
+    @property
+    def network_id(self):
+        return self._security_group.vpc_id
+
     @property
     def rules(self):
         return [AWSSecurityGroupRule(self._provider, r, self)
@@ -671,8 +675,8 @@ class AWSSecurityGroup(BaseSecurityGroup):
                rule.from_port == from_port and
                rule.to_port == to_port and
                rule.grants[0].cidr_ip == cidr_ip) or \
-               (rule.grants[0].name == src_group.name if src_group and
-               hasattr(rule.grants[0], 'name') else False):
+               (rule.grants[0].group_id == src_group.id if src_group and
+               hasattr(rule.grants[0], 'group_id') else False):
                 return AWSSecurityGroupRule(self._provider, rule, self)
         return None
 
@@ -681,6 +685,8 @@ class AWSSecurityGroup(BaseSecurityGroup):
         js = {k: v for(k, v) in attr if not k.startswith('_')}
         json_rules = [r.to_json() for r in self.rules]
         js['rules'] = [json.loads(r) for r in json_rules]
+        if js.get('network_id'):
+            js.pop('network_id')  # Omit for consistency across cloud providers
         return json.dumps(js, sort_keys=True)
 
 
@@ -725,9 +731,9 @@ class AWSSecurityGroupRule(BaseSecurityGroupRule):
     @property
     def group(self):
         if len(self._rule.grants) > 0:
-            if self._rule.grants[0].name:
+            if self._rule.grants[0].group_id:
                 cg = self._provider.ec2_conn.get_all_security_groups(
-                    groupnames=[self._rule.grants[0].name])[0]
+                    group_ids=[self._rule.grants[0].group_id])[0]
                 return AWSSecurityGroup(self._provider, cg)
         return None
 
@@ -742,6 +748,9 @@ class AWSSecurityGroupRule(BaseSecurityGroupRule):
         if self.group:
             # pylint:disable=protected-access
             self.parent._security_group.revoke(
+                ip_protocol=self.ip_protocol,
+                from_port=self.from_port,
+                to_port=self.to_port,
                 src_group=self.group._security_group)
         else:
             # pylint:disable=protected-access
@@ -1134,17 +1143,3 @@ class AWSLaunchConfig(BaseLaunchConfig):
 
     def __init__(self, provider):
         super(AWSLaunchConfig, self).__init__(provider)
-
-    def add_network_interface(self, net_id):
-        """
-        Extract a subnet within the network identified by ``net_id``.
-
-        AWS requires a subnet ID to be supplied vs. a network (i.e., VPC) ID
-        so just pull out one subnet within the network (currently, the first
-        one).
-        """
-        net = self.provider.network.get(net_id)
-        sns = net.subnets()
-        if sns:
-            sn = sns[0]
-        self.network_interfaces.append(sn.id)

cloudbridge/cloud/providers/aws/services.py

@@ -23,11 +23,11 @@ from cloudbridge.cloud.base.services import BaseSnapshotService
 from cloudbridge.cloud.base.services import BaseSubnetService
 from cloudbridge.cloud.base.services import BaseVolumeService
 from cloudbridge.cloud.interfaces.resources import InstanceType
-from cloudbridge.cloud.interfaces.resources \
+from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
-# from cloudbridge.cloud.interfaces.resources import Network
+from cloudbridge.cloud.interfaces.resources import Network
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
@@ -54,6 +54,7 @@ import cloudbridge as cb
 # Uncomment to enable logging by default for this module
 # cb.set_stream_logger(__name__)
 
+
 class AWSSecurityService(BaseSecurityService):
 
     def __init__(self, provider):
@@ -107,7 +108,6 @@ class AWSKeyPairService(BaseKeyPairService):
         :rtype: ``list`` of :class:`.KeyPair`
         :return:  list of KeyPair objects
         """
-        cb.log.trace("Listing AWS key pairs.")
         key_pairs = [AWSKeyPair(self.provider, kp)
                      for kp in self.provider.ec2_conn.get_all_key_pairs()]
         return ClientPagedResultList(self.provider, key_pairs,
@@ -171,7 +171,7 @@ class AWSSecurityGroupService(BaseSecurityGroupService):
         return ClientPagedResultList(self.provider, sgs,
                                      limit=limit, marker=marker)
 
-    def create(self, name, description, network_id=None):
+    def create(self, name, description, network_id):
         """
         Create a new SecurityGroup.
 
@@ -182,8 +182,8 @@ class AWSSecurityGroupService(BaseSecurityGroupService):
         :param description: The description of the new security group.
 
         :type  network_id: ``str``
-        :param network_id: The ID of the VPC to create the security group in,
-                           if any.
+        :param network_id: The ID of the VPC under which to create the security
+                           group.
 
         :rtype: ``object`` of :class:`.SecurityGroup`
         :return:  A SecurityGroup instance or ``None`` if one was not created.
@@ -467,31 +467,24 @@ class AWSInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(AWSInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, zone=None,
+    def create(self, name, image, instance_type, network=None, zone=None,
                key_pair=None, security_groups=None, user_data=None,
-               launch_config=None,
-               **kwargs):
-        """
-        Creates a new virtual machine instance.
-
-        If no VPC/subnet was specified (via ``launch_config`` parameter), this
-        method will search for a default VPC and attempt to launch an instance
-        into that VPC.
-        """
+               launch_config=None, **kwargs):
         image_id = image.id if isinstance(image, MachineImage) else image
         instance_size = instance_type.id if \
             isinstance(instance_type, InstanceType) else instance_type
+        network_id = network.id if isinstance(network, Network) else network
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         key_pair_name = key_pair.name if isinstance(
             key_pair,
             KeyPair) else key_pair
         if launch_config:
             bdm = self._process_block_device_mappings(launch_config, zone_id)
-            subnet_id = self._get_net_id(launch_config)
         else:
-            bdm = subnet_id = None
+            bdm = None
+
         subnet_id, zone_id, security_group_ids = \
-            self._resolve_launch_options(subnet_id, zone_id, security_groups)
+            self._resolve_launch_options(zone_id, network_id, security_groups)
 
         reservation = self.provider.ec2_conn.run_instances(
             image_id=image_id, instance_type=instance_size,
@@ -505,113 +498,202 @@ class AWSInstanceService(BaseInstanceService):
             instance.name = name
         return instance
 
-    def _resolve_launch_options(self, subnet_id, zone_id, security_groups):
+    def _resolve_launch_options(self, zone_id=None, vpc_id=None,
+                                security_groups=None):
         """
-        Resolve inter-dependent launch options.
+        Work out interdependent launch options.
+
+        Some launch options are required and interdependent so work through
+        those constraints. There are 8 subsets of options so the logic works
+        through each of those combinations to figure out the proper launch
+        options.
+
+        :type zone_id: ``str``
+        :param zone_id: ID of the zone where the launch should happen.
+
+        :type vpc_id: ``str``
+        :param vpc_id: ID of the network within which to launch.
 
-        With launching into VPC only, try to figure out a default
-        VPC to launch into, making placement decisions along the way that
-        are implied from the zone a subnet exists in.
+        :type security_groups: ``list`` of ``str``
+        :param zone_id: List of security group names.
+
+        :rtype: triplet of ``str``
+        :return: Subnet ID, zone ID and security group IDs for launch.
+
+        :raise ValueError: In case a conflicting combination is found or the
+                           method cannot infer the defaults, raise.
         """
-        def _deduce_subnet_and_zone(vpc, zone_id=None):
+        def _get_default_vpc(vpcs, exc="No default network found."):
             """
-            Figure out subnet ID from a VPC (and zone_id, if not supplied).
+            Inspect supplied VPCs to figure out a default one or create one.
+
+            Default VPC either has ``is_default`` property set or matches the
+            default network name used by this library. If a default network
+            is not found, an attempt to create one is made.
+
+            :type vpcs: ``list``
+            :param vpcs: A list of boto VPC objects.
+
+            :type exc: ``str``
+            :type exc: A string value to use if/when raising ValueError.
+
+            :rtype: ``str``
+            :return: Default VPC ID.
             """
-            if zone_id:
-                # A placement zone was specified. Choose the default
-                # subnet in that zone.
-                for sn in vpc.subnets():
-                    if sn._subnet.availability_zone == zone_id:
-                        subnet_id = sn.id
-            else:
-                # No zone was requested, so just pick one subnet
-                sn = vpc.subnets()[0]
-                subnet_id = sn.id
-                zone_id = sn._subnet.availability_zone
-            return subnet_id, zone_id
-
-        vpc_id = None
-        sg_ids = []
-        if subnet_id:
-            # Subnet was supplied - get the VPC so named SGs can be resolved
-            subnet = self.provider.vpc_conn.get_all_subnets(subnet_id)[0]
-            vpc_id = subnet.vpc_id
-            # zone_id must match zone where the requested subnet lives
-            if zone_id and subnet.availability_zone != zone_id:
-                raise ValueError("Requested placement zone ({0}) must match "
-                                 "specified subnet's availability zone ({1})."
-                                 .format(zone_id, subnet.availability_zone))
-        if security_groups:
-            # Try to get a subnet via specified SGs. This will work only if
-            # the specified SGs are within a VPC (which is a prerequisite to
-            # launch into VPC anyhow).
-            _sg_ids = self._process_security_groups(security_groups, vpc_id)
-            # Must iterate through all the SGs here because a SG name may
-            # exist in a VPC or EC2-Classic so opt for the VPC SG. This
-            # applies in the case no subnet was specified.
-            if not subnet_id:
-                for sg_id in _sg_ids:
-                    sg = self.provider.security.security_groups.get(sg_id)
-                    if sg._security_group.vpc_id:
-                        if sg_ids and sg_id not in sg_ids:
-                            raise ValueError("Multiple matches for VPC "
-                                             "security group(s) {0}."
-                                             .format(security_groups))
-                        else:
-                            sg_ids.append(sg_id)
-                        vpc = self.provider.network.get(
-                            sg._security_group.vpc_id)
-                        subnet_id, zone_id = _deduce_subnet_and_zone(
-                            vpc, zone_id)
+            default_vpc = None
+            for vpc in vpcs:
+                if vpc.is_default:
+                    default_vpc = vpc.id
+                    break
+            if not default_vpc:
+                for vpc in vpcs:
+                    if vpc.tags.get('Name', '') == \
+                       AWSNetwork.CB_DEFAULT_NETWORK_NAME:
+                        default_vpc = vpc.id
+                        break
+            if not default_vpc:
+                net = self.provider.network.create(
+                    name=AWSNetwork.CB_DEFAULT_NETWORK_NAME)
+                default_vpc = net.id
+            return default_vpc
+
+        def _get_potential_subnets(filters, exc):
+            """
+            Query existing subnets through supplied filters.
+
+            :type filters: ``dict``
+            :param filters: A dictionary supplying desired filters.
+
+            :type exc: ``str``
+            :type exc: A string value to use if/when raising ValueError.
+
+            :rtype: tuple of ``str``
+            :return: A tuple with a random subnet that matches supplied
+                     filters and an availability zone where the given subnet
+                     is defined.
+
+            :raise ValueError: If no subnet can be found, raise a ValueError.
+            """
+            potential_subnets = self.provider.vpc_conn.get_all_subnets(
+                filters=filters)
+            if potential_subnets and len(potential_subnets) > 0:
+                sn_id = potential_subnets[0].id
+                zone_id = potential_subnets[0].availability_zone
             else:
-                sg_ids = _sg_ids
-            if not subnet_id:
-                raise AttributeError("Supplied security group(s) ({0}) must "
-                                     "be associated with a VPC."
-                                     .format(security_groups))
-        if not subnet_id and not security_groups:
-            # No VPC/subnet was supplied, search for the default VPC.
-            for vpc in self.provider.network.list():
-                if vpc._vpc.is_default:
-                    vpc_id = vpc.id
-                    subnet_id, zone_id = _deduce_subnet_and_zone(vpc, zone_id)
-            if not vpc_id:
-                raise AttributeError("No default VPC exists. Supply a "
-                                     "subnet to launch into (via "
-                                     "launch_config param).")
-        return subnet_id, zone_id, sg_ids
-
-    def _process_security_groups(self, security_groups, vpc_id=None):
-        """
-        Process security groups to create a list of SG ID's for launching.
-
-        :type security_groups: A ``list`` of ``SecurityGroup`` objects or a
-                               list of ``str`` names
-        :param security_groups: A list of ``SecurityGroup`` objects or a list
-                                of ``SecurityGroup`` names, which should be
-                                assigned to this instance.
+                raise ValueError(exc)
+            return sn_id, zone_id
 
-        :type vpc_id: ``str``
-        :param vpc_id: A VPC ID within which the supplied security groups exist
+        def _get_security_groups(security_groups, vpc_id=None, obj=False):
+            """
+            Resolve exact security groups to use.
 
-        :rtype: ``list``
-        :return: A list of security group IDs.
-        """
-        if isinstance(security_groups, list) and \
-                isinstance(security_groups[0], SecurityGroup):
-            sg_ids = [sg.id for sg in security_groups]
+            :type security_groups: A ``list`` of ``SecurityGroup`` objects or
+                                   a list of ``str`` names.
+            :param security_groups: A list of ``SecurityGroup`` objects or a
+                                    list of ``SecurityGroup`` names, which
+                                    should be resolved.
+
+            :type vpc_id: ``str``
+            :param vpc_id: ID of the network within which to launch.
+
+            :type obj: ``bool``
+            :param obj: If True, return provider-native security group objects.
+                        Otherwise, return the IDs.
+
+            :rtype: list
+            :return: provider-native security group objects or the IDs (see
+                    ``obj`` param).
+            """
+            if isinstance(security_groups, list) and \
+               isinstance(security_groups[0], SecurityGroup):
+                return [sg._security_group if obj else sg.id
+                        for sg in security_groups]
+            else:
+                flters = {'group_name': security_groups}
+                if vpc_id:
+                    flters['vpc_id'] = vpc_id
+                sgs = self.provider.ec2_conn.get_all_security_groups(
+                    filters=flters)
+                return list(set([sg if obj else sg.id for sg in sgs]))
+
+        if zone_id and vpc_id and security_groups:
+            exc = "No subnets found in zone {0} for network {1}.".format(
+                zone_id, vpc_id)
+            flters = {'availabilityZone': zone_id, 'state': 'available',
+                      'vpcId': vpc_id}
+            sn_id, _ = _get_potential_subnets(flters, exc)
+            sg_ids = _get_security_groups(security_groups, vpc_id)
+        elif vpc_id and security_groups:
+            sg_ids = _get_security_groups(security_groups, vpc_id)
+            exc = "No subnets found in network {0}.".format(vpc_id)
+            flters = {'state': 'available', 'vpcId': vpc_id}
+            sn_id, zone_id = _get_potential_subnets(flters, exc)
+        elif vpc_id and zone_id:
+            flters = {'availabilityZone': zone_id, 'state': 'available',
+                      'vpcId': vpc_id}
+            exc = "No subnets found in zone {0} for network {1}.".format(
+                zone_id, vpc_id)
+            sn_id, _ = _get_potential_subnets(flters, exc)
+            sg_ids = None
+        elif zone_id and security_groups:
+            sgs = _get_security_groups(security_groups, obj=True)
+            # Get VPCs the supplied SGs belong to
+            vpc_ids = list(set([sg.vpc_id for sg in sgs if sg.vpc_id]))
+            vpcs = []
+            if vpc_ids:
+                vpcs = self.provider.vpc_conn.get_all_vpcs(vpc_ids=vpc_ids)
+            exc = ("No default network found for zone {0} and security groups "
+                   "{1}".format(zone_id, security_groups))
+            default_vpc = _get_default_vpc(vpcs, exc)
+            # Filter only the SGs within the default VPC
+            sg_ids = _get_security_groups(security_groups, default_vpc)
+            flters = {'availabilityZone': zone_id, 'state': 'available',
+                      'vpc_id': default_vpc}
+            exc = "No subnets found in zone {0} for default network {1}." \
+                .format(zone_id, default_vpc)
+            sn_id, _ = _get_potential_subnets(flters, exc)
+        elif vpc_id:
+            flters = {'state': 'available', 'vpcId': vpc_id}
+            exc = "No subnets found for network {0}.".format(vpc_id)
+            sn_id, zone_id = _get_potential_subnets(flters, exc)
+            sg_ids = None
+        elif zone_id:
+            vpcs = self.provider.vpc_conn.get_all_vpcs()
+            exc = "No default network exists for security zone {0}.".format(
+                zone_id)
+            default_vpc = _get_default_vpc(vpcs, exc)
+            flters = {'availabilityZone': zone_id, 'state': 'available',
+                      'vpcId': default_vpc}
+            exc = "No subnets found in zone {0} for default network {1}." \
+                .format(zone_id, default_vpc)
+            sn_id, _ = _get_potential_subnets(flters, exc)
+            sg_ids = None
+        elif security_groups:
+            sgs = _get_security_groups(security_groups, obj=True)
+            # Get VPCs the supplied SGs belong to
+            vpc_ids = list(set([sg.vpc_id for sg in sgs if sg.vpc_id]))
+            vpcs = []
+            if vpc_ids:
+                vpcs = self.provider.vpc_conn.get_all_vpcs(vpc_ids=vpc_ids)
+            exc = "No default network exists for security groups {0}.".format(
+                security_groups)
+            default_vpc = _get_default_vpc(vpcs, exc)
+            # Filter only the SGs within the default VPC
+            sg_ids = _get_security_groups(security_groups, default_vpc)
+            flters = {'state': 'available', 'vpcId': default_vpc}
+            exc = "No subnets found in network {0}.".format(default_vpc)
+            sn_id, zone_id = _get_potential_subnets(flters, exc)
         else:
-            # SG names were supplied, need to map them to SG IDs.
-            sg_ids = []
-            # If a VPC was specified, need to map to the SGs in the VPC.
-            flters = None
-            if vpc_id:
-                flters = {'vpc_id': vpc_id}
-            sgs = self.provider.ec2_conn.get_all_security_groups(
-                filters=flters)
-            sg_ids = [sg.id for sg in sgs if sg.name in security_groups]
+            # Nothing was defined, use all defaults
+            vpcs = self.provider.vpc_conn.get_all_vpcs()
+            default_vpc = _get_default_vpc(vpcs)
+            flters = {'state': 'available', 'vpcId': default_vpc}
+            exc = "No subnets found for default network {1}.".format(
+                default_vpc)
+            sn_id, zone_id = _get_potential_subnets(flters, exc)
+            sg_ids = None
 
-        return sg_ids
+        return sn_id, zone_id, sg_ids
 
     def _process_block_device_mappings(self, launch_config, zone=None):
         """
@@ -664,11 +746,6 @@ class AWSInstanceService(BaseInstanceService):
 
         return bdm
 
-    def _get_net_id(self, launch_config):
-        return (launch_config.network_interfaces[0]
-                if len(launch_config.network_interfaces) > 0
-                else None)
-
     def create_launch_config(self):
         return AWSLaunchConfig(self.provider)
 
@@ -729,6 +806,15 @@ class AWSInstanceTypesService(BaseInstanceTypesService):
     @property
     def instance_data(self):
         """
+        Fetch info about the available instances.
+
+        To update this information, update the file pointed to by the
+        ``AWS_INSTANCE_DATA_DEFAULT_URL`` above. The content for this file
+        should be obtained from this repo
+        https://github.com/powdahound/ec2instances.info (in particular, this
+        file: https://raw.githubusercontent.com/powdahound/ec2instances.info/
+        master/www/instances.json).
+
         TODO: Needs a caching function with timeout
         """
         r = requests.get(self.provider.config.get(
@@ -785,8 +871,8 @@ class AWSNetworkService(BaseNetworkService):
                                      limit=limit, marker=marker)
 
     def create(self, name=None):
-        # AWS requried CIDR block to be specified when creating a network
-        # so set a default one and use the largest possible netmask.
+        # AWS requires CIDR block to be specified when creating a network
+        # so set a default one and use the largest allowed netmask.
         default_cidr = '10.0.0.0/16'
         network = self.provider.vpc_conn.create_vpc(cidr_block=default_cidr)
         cb_network = AWSNetwork(self.provider, network)

cloudbridge/cloud/providers/gce/resources.py

@@ -21,6 +21,7 @@ import hashlib
 import inspect
 import json
 import re
+import uuid
 
 class GCEKeyPair(BaseKeyPair):
 
@@ -188,16 +189,16 @@ class GCEFirewallsDelegate(object):
         self._list_response = None
 
     @staticmethod
-    def tag_network_id(tag, network):
+    def tag_network_id(tag, network_name):
         """
-        Generate an ID for a (tag, network) pair.
+        Generate an ID for a (tag, network name) pair.
         """
         md5 = hashlib.md5()
-        md5.update("{0}-{1}".format(tag, network).encode('ascii'))
+        md5.update("{0}-{1}".format(tag, network_name).encode('ascii'))
         return md5.hexdigest()
 
     @staticmethod
-    def network(firewall):
+    def network_name(firewall):
         """
         Extract the network name of a firewall.
         """
@@ -217,60 +218,51 @@ class GCEFirewallsDelegate(object):
     @property
     def tag_networks(self):
         """
-        List all (tag, network) pairs that are used in at least one firewall.
+        List all (tag, network name) pairs that are in at least one firewall.
         """
         out = set()
         for firewall in self.iter_firewalls():
-            network = GCEFirewallsDelegate.network(firewall)
-            if network is not None:
-                out.add((firewall['targetTags'][0], network))
+            network_name = GCEFirewallsDelegate.network_name(firewall)
+            if network_name is not None:
+                out.add((firewall['targetTags'][0], network_name))
         return out
             
     def get_tag_network_from_id(self, tag_network_id):
         """
-        Map an ID back to the (tag, network) pair.
+        Map an ID back to the (tag, network name) pair.
         """
-        for tag, network in self.tag_networks:
-            current_id = GCEFirewallsDelegate.tag_network_id(tag, network)
+        for tag, network_name in self.tag_networks:
+            current_id = GCEFirewallsDelegate.tag_network_id(tag, network_name)
             if current_id == tag_network_id:
-                return (tag, network)
+                return (tag, network_name)
         return (None, None)
 
     def delete_tag_network_with_id(self, tag_network_id):
         """
         Delete all firewalls in a given network with a specific target tag.
         """
-        tag, network = self.get_tag_network_from_id(tag_network_id)
+        tag, network_name = self.get_tag_network_from_id(tag_network_id)
         if tag is None:
             return
-        for firewall in self.iter_firewalls(tag, network):
+        for firewall in self.iter_firewalls(tag, network_name):
             self._delete_firewall(firewall)
         self._update_list_response()
 
     def add_firewall(self, tag, ip_protocol, port, source_range, source_tag,
-                     description, network):
+                     description, network_name):
         """
         Create a new firewall.
         """
         if self.find_firewall(tag, ip_protocol, port, source_range,
-                              source_tag, network) is not None:
+                              source_tag, network_name) is not None:
             return True
         # Do not let the user accidentally open traffic from the world by not
         # explicitly specifying the source.
         if source_tag is None and source_range is None:
             return False
-        firewall_number = 1
-        suffixes = []
-        for firewall in self.iter_firewalls(tag, network):
-            suffix = firewall['name'].split('-')[-1]
-            if suffix.isdigit():
-                suffixes.append(int(suffix))
-        for suffix in sorted(suffixes):
-            if firewall_number == suffix:
-                firewall_number += 1
         firewall = {
-            'name': '%s-%s-rule-%d' % (network, tag, firewall_number),
-            'network': GCEFirewallsDelegate._NETWORK_URL_PREFIX + network,
+            'name': 'firewall-{0}'.format(uuid.uuid4()),
+            'network': GCEFirewallsDelegate._NETWORK_URL_PREFIX + network_name,
             'allowed': [{'IPProtocol': str(ip_protocol)}],
             'targetTags': [tag]}
         if description is not None:
@@ -297,13 +289,13 @@ class GCEFirewallsDelegate(object):
             self._update_list_response()
 
     def find_firewall(self, tag, ip_protocol, port, source_range, source_tag,
-                      network):
+                      network_name):
         """
         Find a firewall with give parameters.
         """
         if source_range is None and source_tag is None:
             source_range = '0.0.0.0/0'
-        for firewall in self.iter_firewalls(tag, network):
+        for firewall in self.iter_firewalls(tag, network_name):
             if firewall['allowed'][0]['IPProtocol'] != ip_protocol:
                 continue
             if not self._check_list_in_dict(firewall['allowed'][0], 'ports',
@@ -337,7 +329,7 @@ class GCEFirewallsDelegate(object):
             if ('ports' in firewall['allowed'][0] and
                 len(firewall['allowed'][0]['ports']) == 1):
                 info['port'] = firewall['allowed'][0]['ports'][0]
-            info['network'] = GCEFirewallsDelegate.network(firewall)
+            info['network_name'] = GCEFirewallsDelegate.network_name(firewall)
             return info
         return info
 
@@ -350,7 +342,7 @@ class GCEFirewallsDelegate(object):
                 self._delete_firewall(firewall)
         self._update_list_response()
 
-    def iter_firewalls(self, tag=None, network=None):
+    def iter_firewalls(self, tag=None, network_name=None):
         """
         Iterate through all firewalls. Can optionally iterate through firewalls
         with a given tag and/or in a network.
@@ -366,11 +358,11 @@ class GCEFirewallsDelegate(object):
                 continue
             if tag is not None and firewall['targetTags'][0] != tag:
                 continue
-            if network is None:
+            if network_name is None:
                 yield firewall
                 continue
-            firewall_network = GCEFirewallsDelegate.network(firewall)
-            if firewall_network == network:
+            firewall_network_name = GCEFirewallsDelegate.network_name(firewall)
+            if firewall_network_name == network_name:
                 yield firewall
 
     def _delete_firewall(self, firewall):
@@ -415,15 +407,15 @@ class GCEFirewallsDelegate(object):
 
 class GCESecurityGroup(BaseSecurityGroup):
 
-    def __init__(self, delegate, tag,
-                 network=GCEFirewallsDelegate.DEFAULT_NETWORK,
-                 description=None):
+    def __init__(self, delegate, tag, network=None, description=None):
         super(GCESecurityGroup, self).__init__(delegate.provider, tag)
         self._description = description
         self._delegate = delegate
-        self._network = network
-        if self._network is None:
-            self._network = GCEFirewallsDelegate.DEFAULT_NETWORK 
+        if network is None:
+            self._network = delegate.provider.network.get_by_name(
+                    GCEFirewallsDelegate.DEFAULT_NETWORK)
+        else:
+            self._network = network
 
     @property
     def id(self):
@@ -432,7 +424,7 @@ class GCESecurityGroup(BaseSecurityGroup):
         network and the target tag corresponding to this security group.
         """
         return GCEFirewallsDelegate.tag_network_id(self._security_group,
-                                                   self._network)
+                                                   self._network.name)
 
     @property
     def name(self):
@@ -454,16 +446,20 @@ class GCESecurityGroup(BaseSecurityGroup):
         if self._description is not None:
             return self._description
         for firewall in self._delegate.iter_firewalls(self._security_group,
-                                                      self._network):
+                                                      self._network.name):
             if 'description' in firewall:
                 return firewall['description']
         return None
 
+    @property
+    def network_id(self):
+        return self._network.id
+
     @property
     def rules(self):
         out = []
         for firewall in self._delegate.iter_firewalls(self._security_group,
-                                                      self._network):
+                                                      self._network.name):
             out.append(GCESecurityGroupRule(self._delegate, firewall['id']))
         return out
 
@@ -482,7 +478,7 @@ class GCESecurityGroup(BaseSecurityGroup):
         src_tag = src_group.name if src_group is not None else None
         self._delegate.add_firewall(self._security_group, ip_protocol, port,
                                     cidr_ip, src_tag, self.description,
-                                    self._network)
+                                    self._network.name)
         return self.get_rule(ip_protocol, from_port, to_port, cidr_ip,
                              src_group)
 
@@ -492,7 +488,7 @@ class GCESecurityGroup(BaseSecurityGroup):
         src_tag = src_group.name if src_group is not None else None
         firewall_id = self._delegate.find_firewall(
                 self._security_group, ip_protocol, port, cidr_ip, src_tag,
-                self._network)
+                self._network.name)
         if firewall_id is None:
             return None
         return GCESecurityGroupRule(self._delegate, firewall_id)
@@ -522,10 +518,14 @@ class GCESecurityGroupRule(BaseSecurityGroupRule):
         Return the security group to which this rule belongs.
         """
         info = self._delegate.get_firewall_info(self._rule)
-        if info is None or 'target_tag' not in info or info['network'] is None:
+        if info is None:
             return None
-        return GCESecurityGroup(self._delegate, info['target_tag'],
-                                info['network'])
+        if 'target_tag' not in info or info['network_name'] is None:
+            return None
+        network = delegate.network.get_by_name(info['network_name'])
+        if network is None:
+            return None
+        return GCESecurityGroup(self._delegate, info['target_tag'], network)
 
     @property
     def id(self):
@@ -584,10 +584,15 @@ class GCESecurityGroupRule(BaseSecurityGroupRule):
         Return the security group from which this rule allows traffic.
         """
         info = self._delegate.get_firewall_info(self._rule)
-        if info is None or 'source_tag' not in info or info['network'] is None:
+        if info is None:
+            return None
+        if 'source_tag' not in info or info['network_name'] is None:
+            return None
+        network = self._delegate.provider.network.get_by_name(
+                info['network_name'])
+        if network is None:
             return None
-        return GCESecurityGroup(self._delegate, info['source_tag'],
-                                info['network'])
+        return GCESecurityGroup(self._delegate, info['source_tag'], network)
 
     def to_json(self):
         attr = inspect.getmembers(self, lambda a: not(inspect.isroutine(a)))

cloudbridge/cloud/providers/gce/services.py

@@ -202,19 +202,24 @@ class GCESecurityGroupService(BaseSecurityGroupService):
         self._delegate = GCEFirewallsDelegate(provider)
 
     def get(self, group_id):
-        tag, network = self._delegate.get_tag_network_from_id(group_id)
+        tag, network_name = self._delegate.get_tag_network_from_id(group_id)
         if tag is None:
             return None
+        network = self.provider.network.get_by_name(network_name)
         return GCESecurityGroup(self._delegate, tag, network)
 
     def list(self, limit=None, marker=None):
-        security_groups = [GCESecurityGroup(self._delegate, x, y)
-                           for x, y in self._delegate.tag_networks]
+        security_groups = []
+        for tag, network_name in self._delegate.tag_networks:
+            network = self.provider.network.get_by_name(network_name)
+            security_group = GCESecurityGroup(self._delegate, tag, network)
+            security_groups.append(security_group)
         return ClientPagedResultList(self.provider, security_groups,
                                      limit=limit, marker=marker)
 
     def create(self, name, description, network_id=None):
-        return GCESecurityGroup(self._delegate, name, network_id, description)
+        network = self.provider.network.get(network_id)
+        return GCESecurityGroup(self._delegate, name, network, description)
 
     def find(self, name, limit=None, marker=None):
         """
@@ -223,8 +228,9 @@ class GCESecurityGroupService(BaseSecurityGroupService):
         is returned.
         """
         out = []
-        for tag, network in self._delegate.tag_networks:
+        for tag, network_name in self._delegate.tag_networks:
             if tag == name:
+                network = self.provider.network.get_by_name(network_name)
                 out.append(GCESecurityGroup(self._delegate, name, network))
         return out
 
@@ -427,7 +433,22 @@ class GCENetworkService(BaseNetworkService):
         super(GCENetworkService, self).__init__(provider)
 
     def get(self, network_id):
-        networks = self.list(filter='id eq %s' % network_id)
+        if network_id is None:
+            return None
+        # networks = self.list(filter='id eq %s' % network_id) would be better.
+        # But, there is a GCE API bug that causes an error if the network_id
+        # has more than 19 digits. So, we list all networks and filter
+        # ourselves.
+        networks = self.list()
+        for network in networks:
+            if network.id == network_id:
+                return network
+        return None
+
+    def get_by_name(self, network_name):
+        if network_name is None:
+            return None
+        networks = self.list(filter='name eq %s' % network_name)
         return None if len(networks) == 0 else networks[0]
 
     def list(self, limit=None, marker=None, filter=None):
@@ -447,6 +468,10 @@ class GCENetworkService(BaseNetworkService):
 
     def create(self, name):
         try:
+            networks = self.list(filter='name eq %s' % name)
+            if len(networks) > 0:
+                return networks[0]
+
             response = (self.provider.gce_compute
                                      .networks()
                                      .insert(project=self.provider.project_name,
@@ -457,7 +482,7 @@ class GCENetworkService(BaseNetworkService):
             self.provider.wait_for_global_operation(response)
             networks = self.list(filter='name eq %s' % name)
             return None if len(networks) == 0 else networks[0]
-        except Exception as e:
+        except:
             return None
 
     @property

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,13 +1,10 @@
-"""
-Provider implementation based on OpenStack Python clients for OpenStack
-compatible clouds.
-"""
+"""Provider implementation based on OpenStack Python clients for OpenStack."""
 
 import os
 
 from cinderclient import client as cinder_client
+from keystoneauth1 import session
 from keystoneclient import client as keystone_client
-from keystoneclient import session
 from neutronclient.v2_0 import client as neutron_client
 from novaclient import client as nova_client
 from novaclient import shell as nova_shell
@@ -23,6 +20,7 @@ from .services import OpenStackSecurityService
 
 
 class OpenStackCloudProvider(BaseCloudProvider):
+    """OpenStack provider implementation."""
 
     PROVIDER_ID = 'openstack'
 
@@ -35,14 +33,13 @@ class OpenStackCloudProvider(BaseCloudProvider):
             'os_username', os.environ.get('OS_USERNAME', None))
         self.password = self._get_config_value(
             'os_password', os.environ.get('OS_PASSWORD', None))
-        self.tenant_name = self._get_config_value(
-            'os_tenant_name', os.environ.get('OS_TENANT_NAME', None))
+        self.project_name = self._get_config_value(
+            'os_project_name', os.environ.get('OS_PROJECT_NAME', None) or
+            os.environ.get('OS_TENANT_NAME', None))
         self.auth_url = self._get_config_value(
             'os_auth_url', os.environ.get('OS_AUTH_URL', None))
         self.region_name = self._get_config_value(
             'os_region_name', os.environ.get('OS_REGION_NAME', None))
-        self.project_name = self._get_config_value(
-            'os_project_name', os.environ.get('OS_PROJECT_NAME', None))
         self.project_domain_name = self._get_config_value(
             'os_project_domain_name',
             os.environ.get('OS_PROJECT_DOMAIN_NAME', None))
@@ -51,23 +48,6 @@ class OpenStackCloudProvider(BaseCloudProvider):
         self.identity_api_version = self._get_config_value(
             'os_identity_api_version',
             os.environ.get('OS_IDENTITY_API_VERSION', None))
-        # Allow individual service connections to have their own values but
-        # default to a the ones defined above.
-        self.swift_username = self._get_config_value(
-            'os_swift_username',
-            os.environ.get('OS_SWIFT_USERNAME', self.username))
-        self.swift_password = self._get_config_value(
-            'os_swift_password',
-            os.environ.get('OS_SWIFT_PASSWORD', self.password))
-        self.swift_tenant_name = self._get_config_value(
-            'os_swift_tenant_name',
-            os.environ.get('OS_SWIFT_TENANT_NAME', self.tenant_name))
-        self.swift_auth_url = self._get_config_value(
-            'os_swift_auth_url',
-            os.environ.get('OS_SWIFT_AUTH_URL', self.auth_url))
-        self.swift_region_name = self._get_config_value(
-            'os_swift_region_name',
-            os.environ.get('OS_SWIFT_REGION_NAME', self.region_name))
 
         # Service connections, lazily initialized
         self._nova = None
@@ -77,6 +57,9 @@ class OpenStackCloudProvider(BaseCloudProvider):
         self._swift = None
         self._neutron = None
 
+        # Additional cached variables
+        self._cached_keystone_session = None
+
         # Initialize provider services
         self._compute = OpenStackComputeService(self)
         self._network = OpenStackNetworkService(self)
@@ -114,27 +97,28 @@ class OpenStackCloudProvider(BaseCloudProvider):
         """
         Connect to Keystone and return a session object.
 
-        :rtype: :class:`keystoneclient.session.Session`
+        :rtype: :class:`keystoneauth1.session.Session`
         :return: A Keystone session object.
         """
-        def connect_v2():
-            from keystoneclient.auth.identity import Password as Password_v2
-            auth = Password_v2(self.auth_url, username=self.username,
-                               password=self.password,
-                               tenant_name=self.tenant_name)
-            return session.Session(auth=auth)
+        if self._cached_keystone_session:
+            return self._cached_keystone_session
 
-        def connect_v3():
-            from keystoneclient.auth.identity.v3 import Password as Password_v3
+        if self._keystone_version == 3:
+            from keystoneauth1.identity.v3 import Password as Password_v3
             auth = Password_v3(auth_url=self.auth_url,
                                username=self.username,
                                password=self.password,
                                user_domain_name=self.user_domain_name,
                                project_domain_name=self.project_domain_name,
                                project_name=self.project_name)
-            return session.Session(auth=auth)
-
-        return connect_v3() if self._keystone_version == 3 else connect_v2()
+            self._cached_keystone_session = session.Session(auth=auth)
+        else:
+            from keystoneauth1.identity.v2 import Password as Password_v2
+            auth = Password_v2(self.auth_url, username=self.username,
+                               password=self.password,
+                               tenant_name=self.project_name)
+            self._cached_keystone_session = session.Session(auth=auth)
+        return self._cached_keystone_session
 
 #     @property
 #     def glance(self):
@@ -184,29 +168,9 @@ class OpenStackCloudProvider(BaseCloudProvider):
         return self._connect_nova_region(self.region_name)
 
     def _connect_nova_region(self, region_name):
-        """
-        Get an OpenStack Nova (compute) client object for the given cloud.
-        """
-        def connect_pwd():
-            """
-            Connect using username and password parameters.
-            """
-            nova = nova_client.Client(
-                api_version, username=self.username, api_key=self.password,
-                project_id=self.tenant_name, auth_url=self.auth_url,
-                region_name=region_name, service_name=service_name,
-                http_log_debug=True if self.config.debug_mode else False)
-            nova.authenticate()
-            return nova
-
-        def connect_sess():
-            """
-            Connect using a Keystone session object.
-            """
-            return nova_client.Client(
-                api_version, session=self._keystone_session,
-                service_name=service_name,
-                http_log_debug=True if self.config.debug_mode else False)
+        """Get an OpenStack Nova (compute) client object."""
+        # Force reauthentication with Keystone
+        self._cached_keystone_session = None
 
         api_version = self._get_config_value(
             'os_compute_api_version',
@@ -218,13 +182,20 @@ class OpenStackCloudProvider(BaseCloudProvider):
         if self.config.debug_mode:
             nova_shell.OpenStackComputeShell().setup_debugging(True)
 
-        return connect_sess() if self._keystone_version == 3 else connect_pwd()
+        nova = nova_client.Client(
+                api_version, session=self._keystone_session,
+                auth_url=self.auth_url,
+                region_name=region_name,
+                service_name=service_name,
+                http_log_debug=True if self.config.debug_mode else False)
+        return nova
 
     def _connect_keystone(self):
-        """
-        Get an OpenStack Keystone (identity) client object for the given cloud.
-        """
-        def connect_v2():
+        """Get an OpenStack Keystone (identity) client object."""
+        if self._keystone_version == 3:
+            return keystone_client.Client(session=self._keystone_session,
+                                          auth_url=self.auth_url)
+        else:
             # Wow, the internal keystoneV2 implementation is terribly buggy. It
             # needs both a separate Session object and the username, password
             # again for things to work correctly. Plus, a manual call to
@@ -235,42 +206,21 @@ class OpenStackCloudProvider(BaseCloudProvider):
                 auth_url=self.auth_url,
                 username=self.username,
                 password=self.password,
-                tenant_name=self.tenant_name,
+                project_name=self.project_name,
                 region_name=self.region_name)
             keystone.authenticate()
             return keystone
 
-        def connect_v3():
-            return keystone_client.Client(session=self._keystone_session,
-                                          auth_url=self.auth_url)
-
-        return connect_v3() if self._keystone_version == 3 else connect_v2()
-
     def _connect_cinder(self):
-        """
-        Get an OpenStack Cinder (block storage) client object for the given
-        cloud.
-        """
-        def connect_pwd():
-            """
-            Connect using username and password parameters.
-            """
-            return cinder_client.Client(
-                api_version, username=self.username, api_key=self.password,
-                project_id=self.tenant_name, auth_url=self.auth_url)
-
-        def connect_sess():
-            """
-            Connect using a Keystone session object.
-            """
-            return cinder_client.Client(
-                api_version, session=self._keystone_session)
-
+        """Get an OpenStack Cinder (block storage) client object."""
         api_version = self._get_config_value(
             'os_volume_api_version',
             os.environ.get('OS_VOLUME_API_VERSION', 2))
 
-        return connect_sess() if self._keystone_version == 3 else connect_pwd()
+        return cinder_client.Client(api_version,
+                                    auth_url=self.auth_url,
+                                    session=self._keystone_session,
+                                    region_name=self.region_name)
 
 #     def _connect_glance(self):
 #         """
@@ -285,47 +235,12 @@ class OpenStackCloudProvider(BaseCloudProvider):
 #                                     session=self.keystone.session)
 
     def _connect_swift(self):
-        """
-        Get an OpenStack Swift (object store) client object for the given
-        cloud.
-        """
-        def connect_v2():
-            os_options = {'region_name': self.swift_region_name}
-            return swift_client.Connection(
-                authurl=self.swift_auth_url, auth_version='2',
-                user=self.swift_username, key=self.swift_password,
-                tenant_name=self.swift_tenant_name,
-                os_options=os_options)
-
-        def connect_v3():
-            os_options = {'region_name': self.swift_region_name,
-                          'user_domain_name': self.user_domain_name,
-                          'project_domain_name': self.project_domain_name,
-                          'project_name': self.project_name}
-            return swift_client.Connection(
-                authurl=self.swift_auth_url, auth_version='3',
-                user=self.swift_username, key=self.swift_password,
-                os_options=os_options)
-
-        return connect_v3() if self._keystone_version == 3 else connect_v2()
+        """Get an OpenStack Swift (object store) client object cloud."""
+        return swift_client.Connection(authurl=self.auth_url,
+                                       session=self._keystone_session)
 
     def _connect_neutron(self):
-        """
-        Get an OpenStack Neutron (networking) client object for the given
-        cloud.
-        """
-        def connect_pwd():
-            """
-            Connect using username and password parameters.
-            """
-            return neutron_client.Client(
-                username=self.username, password=self.password,
-                tenant_name=self.tenant_name, auth_url=self.auth_url)
-
-        def connect_sess():
-            """
-            Connect using a Keystone session object.
-            """
-            return neutron_client.Client(session=self._keystone_session)
-
-        return connect_sess() if self._keystone_version == 3 else connect_pwd()
+        """Get an OpenStack Neutron (networking) client object cloud."""
+        return neutron_client.Client(auth_url=self.auth_url,
+                                     session=self._keystone_session,
+                                     region_name=self.region_name)

cloudbridge/cloud/providers/openstack/resources.py

@@ -853,6 +853,15 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
     def __init__(self, provider, security_group):
         super(OpenStackSecurityGroup, self).__init__(provider, security_group)
 
+    @property
+    def network_id(self):
+        """
+        OpenStack does not associate a SG with a network so default to None.
+
+        :return: Always return ``None``.
+        """
+        return None
+
     @property
     def rules(self):
         # Update SG object; otherwise, recently added rules do now show

cloudbridge/cloud/providers/openstack/services.py

@@ -2,6 +2,7 @@
 Services implemented by the OpenStack provider.
 """
 import fnmatch
+import logging
 import re
 
 from cinderclient.exceptions import NotFound as CinderNotFound
@@ -25,6 +26,7 @@ from cloudbridge.cloud.base.services import BaseVolumeService
 from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
+from cloudbridge.cloud.interfaces.resources import Network
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
@@ -47,6 +49,8 @@ from .resources import OpenStackSnapshot
 from .resources import OpenStackSubnet
 from .resources import OpenStackVolume
 
+log = logging.getLogger(__name__)
+
 
 class OpenStackSecurityService(BaseSecurityService):
 
@@ -196,7 +200,7 @@ class OpenStackSecurityGroupService(BaseSecurityGroupService):
         return ClientPagedResultList(self.provider, sgs,
                                      limit=limit, marker=marker)
 
-    def create(self, name, description, network_id=None):
+    def create(self, name, description, network_id):
         """
         Create a new security group under the current account.
 
@@ -496,7 +500,13 @@ class OpenStackRegionService(BaseRegionService):
         return next(region, None)
 
     def list(self, limit=None, marker=None):
-        def keystone_v2():
+        # pylint:disable=protected-access
+        if self.provider._keystone_version == 3:
+            os_regions = [OpenStackRegion(self.provider, region)
+                          for region in self.provider.keystone.regions.list()]
+            return ClientPagedResultList(self.provider, os_regions,
+                                         limit=limit, marker=marker)
+        else:
             # Keystone v3 onwards supports directly listing regions
             # but for v2, this convoluted method is necessary.
             regions = (
@@ -511,26 +521,10 @@ class OpenStackRegionService(BaseRegionService):
             return ClientPagedResultList(self.provider, os_regions,
                                          limit=limit, marker=marker)
 
-        def keystone_v3():
-            os_regions = [OpenStackRegion(self.provider, region)
-                          for region in self.provider.keystone.regions.list()]
-            return ClientPagedResultList(self.provider, os_regions,
-                                         limit=limit, marker=marker)
-
-        return keystone_v3() if self.provider._keystone_version == 3 else \
-            keystone_v2()  # pylint:disable=protected-access
-
     @property
     def current(self):
-        if self.provider.keystone.has_service_catalog():
-            nova_region = [
-                endpoint.get('region') or endpoint.get('region_id')
-                for svc in self.provider.keystone.service_catalog.get_data()
-                for endpoint in svc.get('endpoints', [])
-                if endpoint.get('publicURL', None) ==
-                self.provider.nova.client.management_url]
-            return self.get(nova_region[0])
-        return None
+        nova_region = self.provider.nova.client.region_name
+        return self.get(nova_region) if nova_region else None
 
 
 class OpenStackComputeService(BaseComputeService):
@@ -564,18 +558,17 @@ class OpenStackInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(OpenStackInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, zone=None,
+    def create(self, name, image, instance_type, network=None, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None,
                **kwargs):
-        """
-        Creates a new virtual machine instance.
-        """
+        """Create a new virtual machine instance."""
         image_id = image.id if isinstance(image, MachineImage) else image
         instance_size = instance_type.id if \
             isinstance(instance_type, InstanceType) else \
             self.provider.compute.instance_types.find(
                 name=instance_type)[0].id
+        network_id = network.id if isinstance(network, Network) else network
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         key_pair_name = key_pair.name if \
             isinstance(key_pair, KeyPair) else key_pair
@@ -587,11 +580,10 @@ class OpenStackInstanceService(BaseInstanceService):
                 security_groups_list = security_groups
         else:
             security_groups_list = None
+        bdm = None
         if launch_config:
             bdm = self._to_block_device_mapping(launch_config)
-            nics = self._format_nics(launch_config)
-        else:
-            bdm = nics = None
+        net = self._get_network(network_id)
 
         os_instance = self.provider.nova.servers.create(
             name,
@@ -604,7 +596,7 @@ class OpenStackInstanceService(BaseInstanceService):
             security_groups=security_groups_list,
             userdata=user_data,
             block_device_mapping_v2=bdm,
-            nics=nics)
+            nics=net)
         return OpenStackInstance(self.provider, os_instance)
 
     def _to_block_device_mapping(self, launch_config):
@@ -657,14 +649,42 @@ class OpenStackInstanceService(BaseInstanceService):
                 return True
         return False
 
-    def _format_nics(self, launch_config):
+    def _get_network(self, network_id=None):
         """
-        Format network IDs for the API call.
+        Format the network ID for the API call, figuring out a default network.
+
+        If a network_id is not supplied, figure out which is the default
+        network and use it. A default network is either marked as such by the
+        provider or matches the default network name defined within this
+        library (by default CloudBridgeNet). If a default network cannot be
+        found, attempt to create a new one.
         """
-        nics = []
-        for net_id in launch_config.network_interfaces:
-            nics.append({'net-id': net_id})
-        return nics
+        if network_id:
+            return [{'net-id': network_id}]
+        for net in self.provider.network.list():
+            if net.name == OpenStackNetwork.CB_DEFAULT_NETWORK_NAME:
+                return [{'net-id': net.id}]
+        try:
+            # Try to create a complete, Internet-connected new network
+            net = self.provider.network.create(
+                name=OpenStackNetwork.CB_DEFAULT_NETWORK_NAME)
+            sn = net.create_subnet('10.0.0.0/16', '{0}Subnet'.format(
+                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME))
+            router = self.provider.network.create_router('{0}Router'.format(
+                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME))
+            for n in self.provider.network.list():
+                if n.external:
+                    external_net = n
+                    break
+            router.attach_network(external_net.id)
+            router.add_route(sn.id)
+            return [{'net-id': net.id}]
+        except Exception as exc:
+            # At this point we assume the provider does support user-defined
+            # networks so return None
+            log.warn("Exception occurred trying to create a default "
+                     "CloudBridge network: {0}".format(exc))
+            return None
 
     def create_launch_config(self):
         return BaseLaunchConfig(self.provider)

docs/getting_started.rst

@@ -40,9 +40,9 @@ OpenStack (with Keystone authentication v2):
 
     config = {'os_username': 'username',
               'os_password': 'password',
-              'os_tenant_name': 'tenant name',
               'os_auth_url': 'authentication URL',
-              'os_region_name': 'region name'}
+              'os_region_name': 'region name',
+              'os_project_name': 'project name'}
     provider = CloudProviderFactory().create_provider(ProviderList.OPENSTACK,
                                                       config)
     image_id = 'c1f4b7bc-a563-4feb-b439-a2e071d861aa'  # Ubuntu 14.04 @ NeCTAR
@@ -56,9 +56,9 @@ OpenStack (with Keystone authentication v3):
     config = {'os_username': 'username',
               'os_password': 'password',
               'os_auth_url': 'authentication URL',
-              'os_user_domain_name': 'domain name',
+              'os_project_name': 'project name',
               'os_project_domain_name': 'project domain name',
-              'os_project_name': 'project name'}
+              'os_user_domain_name': 'domain name'}
     provider = CloudProviderFactory().create_provider(ProviderList.OPENSTACK,
                                                       config)
     image_id = '97755049-ee4f-4515-b92f-ca00991ee99a'  # Ubuntu 14.04 @ Jetstream
@@ -91,23 +91,6 @@ on disk as a read-only file.
     import os
     os.chmod('cloudbridge_intro.pem', 0400)
 
-Configure a private network
----------------------------
-We want to provision our instance into a private network to give us flexibility
-in the future. Also, providers these days are increasingly requiring use of
-private networks. Setting up a private network requires several steps:
-(1) create a network; (2) create a subnet within the network; (3) create a
-router; (4) attach the router to an external network; and (5) add a route to
-the router that links with with a subnet.
-
-.. code-block:: python
-
-    net = provider.network.create('cloudbridge_intro')
-    sn = net.create_subnet('10.0.0.1/28', 'cloudbridge-intro')
-    router = provider.network.create_router('cloudbridge-intro')
-    router.attach_network(net.id)
-    router.add_route(sn.id)
-
 Create a security group
 -----------------------
 Next, we need to create a security group and add a rule to allow ssh access.
@@ -130,11 +113,9 @@ also add the network interface as a launch argument.
     inst_type = sorted([t for t in provider.compute.instance_types.list()
                         if t.vcpus >= 2 and t.ram >= 4],
                        key=lambda x: x.vcpus*x.ram)[0]
-    lc = provider.compute.instances.create_launch_config()
-    lc.add_network_interface(net.id)
     inst = provider.compute.instances.create(
         name='CloudBridge-intro', image=img, instance_type=inst_type,
-        key_pair=kp, security_groups=[sg], launch_config=lc)
+        key_pair=kp, security_groups=[sg])
     # Wait until ready
     inst.wait_till_ready()  # This is a blocking call
     # Show instance state
@@ -147,6 +128,8 @@ To access the instance, let's assign a public IP address to the instance. For
 this step, we'll first need to allocate a floating IP address for our account
 and then associate it with the instance.
 
+.. code-block:: python
+
     fip = provider.network.create_floating_ip()
     inst.add_floating_ip(fip.public_ip)
     inst.refresh()

docs/topics/launch.rst

@@ -1,110 +1,63 @@
 Launching instances
 ===================
-Depending on the cloud provider, instances can be launched using
-software-managed networking (e.g., VPC on AWS, Neutron on OpenStack) or the
-classic networking approach. Before being able to run below command, you will
-need a ``provider`` object (see `this page <setup.html>`_).
+Before being able to run below commands, you will need a ``provider`` object
+(see `this page <setup.html>`_).
 
 Common launch data
 ------------------
-Before launching an instance, you need to decide on what image to launch
-as well as what type of instance. We will create those objects here are use
-them in both options below. The specified image ID is a base Ubuntu image on
-AWS so feel free to change it as desired.
+Before launching an instance, you need to decide what image to launch
+as well as what type of instance. We will create those objects here. The
+specified image ID is a base Ubuntu image on AWS so feel free to change it as
+desired. For instance type, we're going to let CloudBridge figure out what's
+the appropriate name on a given provider for an instance with at least 2 CPUs
+and 4 GB RAM.
 
 .. code-block:: python
 
-    img = provider.compute.images.get('ami-d85e75b0')  # Ubuntu 14.04 on AWS
-    inst_type = provider.compute.instance_types.find(name='m1.small')[0]
+    img = provider.compute.images.get('ami-5ac2cd4d')  # Ubuntu 14.04 on AWS
+    inst_type = sorted([t for t in provider.compute.instance_types.list()
+                        if t.vcpus >= 2 and t.ram >= 4],
+                       key=lambda x: x.vcpus*x.ram)[0]
 
 When launching an instance, you can also specify several optional arguments
 such as the security group, a key pair, or instance user data. To allow you to
 connect to the launched instances, we will also supply those parameters (note
 that we're making an assumption here these resources exist; if you don't have
-those resources, take a look at the `Getting Started <../getting_started.html>`_
-guide).
+those resources under your account, take a look at the
+`Getting Started <../getting_started.html>`_ guide).
 
 .. code-block:: python
 
     kp = provider.security.key_pairs.find(name='cloudbridge_intro')[0]
     sg = provider.security.security_groups.list()[0]
 
-Private networking setup
-------------------------
-Private networking gives you control over the networking setup for your
-instance(s) and is considered the preferred method for launching instances.
-
-Create a new private network
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-To start, we will create a private network and a corresponding subnet into
-which an instance will be launched. When creating the subnet, we need to
-set the address pool. For OpenStack, any address pool is acceptable while for
-the AWS cloud, the subnet address pool needs to belong to the private network
-address space; we can obtain the private network address space via
-network object's ``cidr_block`` field (e.g., ``10.0.0.0/16``). Let's crate a
-subnet starting from the beginning of the block and allow up to 32 IP addresses
-into the subnet (``/27``):
-
-.. code-block:: python
-
-    net = provider.network.create(name="CloudBridge-net")
-    net.cidr_block  # '10.0.0.0/16'
-    sn = net.create_subnet('10.0.0.1/27', "CloudBridge-subnet")
-
-Note that it may be necessary to also create a route for this new network. If
-that's the case, take a look at the
-`Getting Started <../getting_started.html>`_ document for an example.
-
-Retrieve an existing private network
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-If you already have existing networks, we can simply reuse an existing one:
-
-.. code-block:: python
-
-    provider.network.list()  # Find a desired network ID
-    net = provider.network.get('desired network ID')
-    sn = net.subnets()[0]  # Get a handle on a desired subnet
-
 Launch an instance
 ------------------
-Once we have a handle on a private network, we'll define a launch configuration
-object to aggregate all the launch configuration options. The launch config
-can contain other launch options, such as the block storage mappings (see
-below). Finally, we can launch the instance:
+Once we have all the desired pieces, we'll use them to launch an instance:
 
 .. code-block:: python
 
-    lc = provider.compute.instances.create_launch_config()
-    lc.add_network_interface(net.id)
     inst = provider.compute.instances.create(
-        name='CloudBridge-VPC', image=img,  instance_type=inst_type,
-        launch_config=lc, key_pair=kp, security_groups=[sg])
-
-.. warning::
-
-    CloudBridge version 0.1.0 does not uniformly deal with network abstractions
-    for AWS and OpenStack providers. AWS takes a subnet ID for it's launch
-    config while OpenStack takes a network ID. As a result, the user needs to
-    make this distinction in their code and supply the correct value. For
-    example, for AWS, above code needs to look like the following:
-    ``lc.add_network_interface(sn.id)``. This has been corrected in newer code.
-
-Launch with default networking
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Launching an instance with the default networking model is straightforward,
-only needing to specify the basic parameters. This will only work for the case
-a default network exists for your account, which is provider-dependent and may
-not necessarily exist.
+        name='CloudBridge-VPC', image=img, instance_type=inst_type,
+        key_pair=kp, security_groups=[sg])
 
-For the case of AWS, an instance will be launched into the VPC where the
-specified security group belongs to. If no security group is specified, the
-instance will get launched into the *default* VPC, assuming such VPC exists.
+Private networking
+~~~~~~~~~~~~~~~~~~
+Private networking gives you control over the networking setup for your
+instance(s) and is considered the preferred method for launching instances. To
+launch an instance with an explicit private network, just supply it as an
+additional argument to the ``create`` method:
 
 .. code-block:: python
 
+    provider.network.list()  # Find a desired network ID
+    net = provider.network.get('desired network ID')
     inst = provider.compute.instances.create(
-        name='CloudBridge-basic', image=img, instance_type=inst_type,
-        key_pair=kp, security_groups=[sg])
+        name='CloudBridge-VPC', image=img, instance_type=inst_type,
+        network=net, key_pair=kp, security_groups=[sg])
+
+For more information on how to create and setup a private network, take a look
+at `Networking <./networking.html>`_.
 
 Block device mapping
 ~~~~~~~~~~~~~~~~~~~~

docs/topics/networking.rst

@@ -0,0 +1,50 @@
+Private networking
+==================
+Private networking gives you control over the networking setup for your
+instance(s) and is considered the preferred method for launching instances.
+Also, providers these days are increasingly requiring use of private networks.
+
+If you do not explicitly specify a private network to use when launching an
+instance, CloudBridge will attempt to use a default one. A 'default' network is
+one tagged as such by the native API. If such tag or functionality does not
+exist, CloudBridge will look for one with a predefined name (by default, called
+'CloudBridgeNet', which can be overridden with environment variable
+``CB_DEFAULT_NETWORK_NAME``).
+
+Create a new private network
+----------------------------
+Creating a private network is a simple, one-line command but appropriately
+connecting it so it has Internet access is a multi-step process:
+(1) create a network; (2) create a subnet within the network; (3) create a
+router; (4) attach the router to an external network; and (5) add a route to
+the router that links with with a subnet. For some providers, any network can
+be external (ie, connected to the Internet) while for others it's a specific,
+pre-defined one that exists in the an account by default. In order to properly
+connect the router, we need to ensure we're using an external network.
+
+When creating the subnet, we need to set an address pool. We can obtain the
+private network address space via network object's ``cidr_block`` field (e.g.,
+``10.0.0.0/16``). Below, we'll create a subnet starting from the beginning of
+the block and allow up to 16 IP addresses into the subnet (``/28``).
+
+.. code-block:: python
+
+    net = provider.network.create('cloudbridge_intro')
+    sn = net.create_subnet('10.0.0.0/28', 'cloudbridge-intro')
+    router = provider.network.create_router('cloudbridge-intro')
+    if not net.external:
+        for n in self.provider.network.list():
+            if n.external:
+                external_net = n
+                break
+    router.attach_network(external_net.id)
+    router.add_route(sn.id)
+
+Retrieve an existing private network
+------------------------------------
+If you already have existing networks, we can query for those:
+
+.. code-block:: python
+
+    provider.network.list()  # Find a desired network ID
+    net = provider.network.get('desired network ID')

docs/topics/overview.rst

@@ -8,6 +8,7 @@ Introductions to all the key parts of CloudBridge you'll need to know:
     How to install CloudBridge <install.rst>
     Connection and authentication setup <setup.rst>
     Launching instances <launch.rst>
+    Networking <networking.rst>
     Object states and lifecycles <object_lifecycles.rst>
     Paging and iteration <paging_and_iteration.rst>
     Using block storage <block_storage.rst>

docs/topics/setup.rst

@@ -28,29 +28,10 @@ Mandatory variables  Optional Variables
 OS_AUTH_URL			 NOVA_SERVICE_NAME
 OS_USERNAME			 OS_COMPUTE_API_VERSION
 OS_PASSWORD			 OS_VOLUME_API_VERSION
-OS_TENANT_NAME
+OS_PROJECT_NAME
 OS_REGION_NAME
 ===================  ==================
 
-If you'd like, you can specify service-specific variables for OpenStack.
-This can be used to create a multi-cloud provider object for example where
-the compute service is using one cloud while the object store service uses
-another. This can be useful is a given cloud does not supply all the desired
-services. If these variables are not supplied, the default ones from above
-are used across all OpenStack services.
-
-=================================== ==============
-Optional service-specific variables Example values
------------------------------------ --------------
-Swift service
-==================================================
-OS_SWIFT_AUTH_URL                   https://keystone.rc.nectar.org.au:5000/v2.0/
-OS_SWIFT_USERNAME                   your.name@example.com
-OS_SWIFT_PASSWORD                   GcsGgcbsdilcbUIYGcsdc
-OS_SWIFT_REGION_NAME                RegionOne
-OS_SWIFT_TENANT_NAME                GalaxyProject
-=================================== ==============
-
 
 Once the environment variables are set, you can create a connection as follows:
 

setup.py

@@ -14,12 +14,12 @@ with open(os.path.join('cloudbridge', '__init__.py')) as f:
             break
 
 base_reqs = ['bunch>=1.0.1', 'six>=1.10.0', 'retrying>=1.3.3']
-openstack_reqs = ['python-novaclient>=2.33.0',
-                  'python-glanceclient',
-                  'python-cinderclient>=1.4.0',
-                  'python-swiftclient>=2.6.0',
-                  'python-neutronclient>=3.1.0',
-                  'python-keystoneclient>=2.0.0']
+openstack_reqs = ['python-novaclient>=7.0.0',
+                  'python-glanceclient>=2.5.0',
+                  'python-cinderclient>=1.9.0',
+                  'python-swiftclient>=3.2.0',
+                  'python-neutronclient>=6.0.0',
+                  'python-keystoneclient>=3.8.0']
 aws_reqs = ['boto>=2.38.0']
 gce_reqs = ['google-api-python-client>=1.4.2', "pycrypto"]
 full_reqs = base_reqs + aws_reqs + openstack_reqs + gce_reqs

test/helpers.py

@@ -44,14 +44,16 @@ def cleanup_action(cleanup_func):
         except Exception as e:
             print("Error during exception cleanup: {0}".format(e))
         reraise(ex_class, ex_val, ex_traceback)
-    cleanup_func()
+    try:
+        cleanup_func()
+    except Exception as e:
+        print("Error during cleanup: {0}".format(e))
 
 
 TEST_DATA_CONFIG = {
     "AWSCloudProvider": {
-        "image": os.environ.get('CB_IMAGE_AWS', 'ami-d85e75b0'),
-        "instance_type": os.environ.get('CB_INSTANCE_TYPE_AWS',
-                                        't1.micro'),
+        "image": os.environ.get('CB_IMAGE_AWS', 'ami-5ac2cd4d'),
+        "instance_type": os.environ.get('CB_INSTANCE_TYPE_AWS', 't2.nano'),
         "placement": os.environ.get('CB_PLACEMENT_AWS', 'us-east-1a'),
     },
     "OpenStackCloudProvider": {
@@ -91,18 +93,19 @@ def delete_test_network(network):
     """
     Delete the supplied network, first deleting any contained subnets.
     """
-    for sn in network.subnets():
-        sn.delete()
-    network.delete()
+    with cleanup_action(lambda: network.delete()):
+        for sn in network.subnets():
+            sn.delete()
 
 
 def create_test_instance(
-        provider, instance_name, zone=None, launch_config=None,
+        provider, instance_name, network, zone=None, launch_config=None,
         key_pair=None, security_groups=None):
     return provider.compute.instances.create(
         instance_name,
         get_provider_test_data(provider, 'image'),
         get_provider_test_data(provider, 'instance_type'),
+        network=network,
         zone=zone,
         key_pair=key_pair,
         security_groups=security_groups,
@@ -112,12 +115,10 @@ def create_test_instance(
 def get_test_instance(provider, name, key_pair=None, security_groups=None,
                       network=None):
     launch_config = None
-    if network:
-        launch_config = provider.compute.instances.create_launch_config()
-        launch_config.add_network_interface(network.id)
     instance = create_test_instance(
         provider,
         name,
+        network=network,
         key_pair=key_pair,
         security_groups=security_groups,
         launch_config=launch_config)
@@ -127,17 +128,13 @@ def get_test_instance(provider, name, key_pair=None, security_groups=None,
 
 def cleanup_test_resources(instance=None, network=None, security_group=None,
                            key_pair=None):
-    if instance:
-        instance.terminate()
-        instance.wait_for(
-            [InstanceState.TERMINATED, InstanceState.UNKNOWN],
-            terminal_states=[InstanceState.ERROR])
-    if security_group:
-        security_group.delete()
-    if key_pair:
-        key_pair.delete()
-    if network:
-        delete_test_network(network)
+    with cleanup_action(lambda: delete_test_network(network)):
+        with cleanup_action(lambda: key_pair.delete()):
+            with cleanup_action(lambda: security_group.delete()):
+                instance.terminate()
+                instance.wait_for(
+                    [InstanceState.TERMINATED, InstanceState.UNKNOWN],
+                    terminal_states=[InstanceState.ERROR])
 
 
 class ProviderTestBase(object):

test/test_compute_service.py

@@ -6,7 +6,7 @@ from cloudbridge.cloud.interfaces \
     import InvalidConfigurationException
 from cloudbridge.cloud.interfaces import InstanceState
 from cloudbridge.cloud.interfaces.resources import InstanceType
-from cloudbridge.cloud.interfaces.resources import WaitStateException
+from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 from test.helpers import ProviderTestBase
 import test.helpers as helpers
 
@@ -287,14 +287,13 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             lc.add_ephemeral_device()
 
         net, _ = helpers.create_test_network(self.provider, name)
-        lc.add_network_interface(net.id)
 
         inst = helpers.create_test_instance(
             self.provider,
             name,
-            zone=helpers.get_provider_test_data(
-                self.provider,
-                'placement'),
+            network=net,
+            # We don't have a way to match the test net placement and this zone
+            # zone=helpers.get_provider_test_data(self.provider, 'placement'),
             launch_config=lc)
 
         def cleanup(instance, net):

test/test_interface.py

@@ -1,6 +1,10 @@
+import unittest
 import cloudbridge
 from cloudbridge.cloud import interfaces
 from test.helpers import ProviderTestBase
+from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
+from cloudbridge.cloud.factory import CloudProviderFactory
 
 
 class CloudInterfaceTestCase(ProviderTestBase):
@@ -38,3 +42,25 @@ class CloudInterfaceTestCase(ProviderTestBase):
         """
         self.assertIsNotNone(cloudbridge.get_version(),
                              "Did not get library version.")
+
+    def test_authenticate_success(self):
+        self.assertTrue(self.provider.authenticate())
+
+    def test_authenticate_failure(self):
+        if isinstance(self.provider, TestMockHelperMixin):
+            raise unittest.SkipTest(
+                "Mock providers are not expected to"
+                " authenticate correctly")
+
+        cloned_provider = CloudProviderFactory().create_provider(
+            self.provider.PROVIDER_ID, self.provider.config)
+
+        with self.assertRaises(ProviderConnectionException):
+            # Mock up test by clearing credentials on a per provider basis
+            if cloned_provider.PROVIDER_ID == 'aws':
+                cloned_provider.a_key = "dummy_a_key"
+                cloned_provider.s_key = "dummy_s_key"
+            elif cloned_provider.PROVIDER_ID == 'openstack':
+                cloned_provider.username = "cb_dummy"
+                cloned_provider.password = "cb_dummy"
+            cloned_provider.authenticate()

test/test_object_life_cycle.py

@@ -1,7 +1,7 @@
 import uuid
 
 from cloudbridge.cloud.interfaces import VolumeState
-from cloudbridge.cloud.interfaces.resources import WaitStateException
+from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 from test.helpers import ProviderTestBase
 import test.helpers as helpers
 

test/test_security_service.py

@@ -1,3 +1,4 @@
+"""Test cloudbridge.security modules."""
 import json
 from test.helpers import ProviderTestBase
 import time
@@ -13,7 +14,7 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             methodName=methodName, provider=provider)
 
     def test_crud_key_pair_service(self):
-        name = 'cbtestkeypair-a'
+        name = 'cbtestkeypairA-{0}'.format(uuid.uuid4()).lower()
         kp = self.provider.security.key_pairs.create(name=name)
         with helpers.cleanup_action(
             lambda:
@@ -64,7 +65,7 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             "Found a key pair {0} that should not exist?".format(no_kp))
 
     def test_key_pair(self):
-        name = 'cbtestkeypair-b'
+        name = 'cbtestkeypairB-{0}'.format(uuid.uuid4()).lower()
         kp = self.provider.security.key_pairs.create(name=name)
         with helpers.cleanup_action(lambda: kp.delete()):
             kpl = self.provider.security.key_pairs.list()
@@ -94,16 +95,19 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             "Key pair {0} should have been deleted but still exists."
             .format(name))
 
+    def cleanup_sg(self, sg, net):
+        with helpers.cleanup_action(
+                lambda: self.provider.network.delete(network_id=net.id)):
+            self.provider.security.security_groups.delete(group_id=sg.id)
+
     def test_crud_security_group_service(self):
-        name = 'cbtestsecuritygroup-a'
+        name = 'cbtestsecuritygroupA-{0}'.format(uuid.uuid4()).lower()
+        net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
-            name=name, description=name)
+            name=name, description=name, network_id=net.id)
         #Empty security groups don't exist in GCE. Let's add a dummy rule.
         sg.add_rule(ip_protocol='tcp', cidr_ip='0.0.0.0/0')
-        with helpers.cleanup_action(
-            lambda:
-                self.provider.security.security_groups.delete(group_id=sg.id)
-        ):
+        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
             self.assertEqual(name, sg.description)
 
             # test list method
@@ -153,10 +157,11 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
 
     def test_security_group(self):
         """Test for proper creation of a security group."""
-        name = 'cbtestsecuritygroup-b'
+        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4()).lower()
+        net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
-            name=name, description=name)
-        with helpers.cleanup_action(lambda: sg.delete()):
+            name=name, description=name, network_id=net.id)
+        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
             rule = sg.add_rule(ip_protocol='tcp', from_port=1111, to_port=1111,
                                cidr_ip='0.0.0.0/0')
             found_rule = sg.get_rule(ip_protocol='tcp', from_port=1111,
@@ -181,16 +186,17 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             self.assertFalse(
                 sg != sg,
                 "The same security groups should still be equal?")
-            json_repr = json.dumps(
-                {"description": name, "name": name, "id": sg.id, "rules":
-                 [{"from_port": 1111, "group": "", "cidr_ip": "0.0.0.0/0",
-                   "parent": sg.id, "to_port": 1111, "ip_protocol": "tcp",
-                   "id": sg.rules[0].id}]},
-                sort_keys=True)
-            self.assertTrue(
-                sg.to_json() == json_repr,
-                "JSON sec group representation {0} does not match expected {1}"
-                .format(sg.to_json(), json_repr))
+#             json_repr = json.dumps(
+#                 {"description": name, "name": name, "id": sg.id,
+#                  "rules":
+#                     [{"from_port": 1111, "group": "", "cidr_ip": "0.0.0.0/0",
+#                       "parent": sg.id, "to_port": 1111, "ip_protocol": "tcp",
+#                       "id": sg.rules[0].id}]},
+#                 sort_keys=True)
+#             self.assertTrue(
+#                 sg.to_json() == json_repr,
+#                 "JSON sec group representation {0} does not match expected {1}"
+#                 .format(sg.to_json(), json_repr))
 
         sgl = self.provider.security.security_groups.list()
         found_sg = [g for g in sgl if g.name == name]
@@ -201,10 +207,11 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
 
     def test_security_group_rule_add_twice(self):
         """Test whether adding the same rule twice succeeds."""
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4())
+        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4()).lower()
+        net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
-            name=name, description=name)
-        with helpers.cleanup_action(lambda: sg.delete()):
+            name=name, description=name, network_id=net.id)
+        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
             rule = sg.add_rule(ip_protocol='tcp', from_port=1111, to_port=1111,
                                cidr_ip='0.0.0.0/0')
             # attempting to add the same rule twice should succeed
@@ -217,16 +224,17 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
 
     def test_security_group_group_rule(self):
         """Test for proper creation of a security group rule."""
-        name = 'cbtestsecuritygroup-c'
+        name = 'cbtestsecuritygroupC-{0}'.format(uuid.uuid4()).lower()
+        net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
-            name=name, description=name)
-        with helpers.cleanup_action(
-                lambda: None if sg is None else sg.delete()):
+            name=name, description=name, network_id=net.id)
+        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
             self.assertTrue(
                 len(sg.rules) == 0,
                 "Expected no security group group rule. Got {0}."
                 .format(sg.rules))
-            rule = sg.add_rule(ip_protocol='tcp', src_group=sg)
+            rule = sg.add_rule(src_group=sg, ip_protocol='tcp', from_port=0,
+                               to_port=65535)
             self.assertTrue(
                 rule.group.name == name,
                 "Expected security group rule name {0}. Got {1}."