Merge branch 'master' into nuwan_gce_changes

# Conflicts:
#	test/test_network_service.py
#	test/test_security_service.py
#	tox.ini

.gitignore

@@ -57,3 +57,4 @@ docs/_build/
 target/
 
 *.DS_Store
+/venv/

.travis.yml

@@ -1,12 +1,15 @@
 language: python
-python: 3.5
+python: 3.6
 os:
   - linux
 #  - osx
 env:
-  - TOX_ENV=py27
-  - TOX_ENV=py35
-  - TOX_ENV=pypy
+  - TOX_ENV=py27-aws
+  - TOX_ENV=py27-openstack
+  - TOX_ENV=py36-aws
+  - TOX_ENV=py36-openstack
+  - TOX_ENV=pypy-aws
+  - TOX_ENV=pypy-openstack
 matrix:
   fast_finish: true
   allow_failures:
@@ -19,4 +22,4 @@ script:
   - tox -e $TOX_ENV
 after_success:
   - coveralls
-  - codecov 
+  - codecov

CHANGELOG.rst

@@ -1,4 +1,20 @@
-0.1.1 - Aug 10, 2016.
+0.2.0 - March 23, 2017. (sha a442d96b829ea2c721728520b01981fa61774625)
+-------
+
+* Reworked the instance launch method to require subnet vs. network. This
+  removed the option of adding network interface to a launch config object.
+* Added object store methods: upload from file path, list objects with a
+  prefix, check if an object exists, (AWS only) get an accessible URL for an
+  object (thanks @VJalili)
+* Modified `get_ec2_credentials()` method to `get_or_create_ec2_credentials()`
+* Added an option to read provider config values from a file
+  (`~/.cloudbridge` or `/etc/cloudbridge`)
+* Replaced py35 with py36 for running tests
+* Added logging configuration for the library
+* General documentation updates
+
+
+0.1.1 - Aug 10, 2016. (sha 0122fb1173c88ae64e40140ffd35ff3797e9e4ad)
 -------
 
 * For AWS, always launch instances into private networking (i.e., VPC)

README.rst

@@ -1,7 +1,6 @@
 CloudBridge aims to provide a simple layer of abstraction over
 different cloud providers, reducing or eliminating the need to write
-conditional code for each cloud. It is currently under development and is in
-an Alpha state.
+conditional code for each cloud.
 
 .. image:: https://landscape.io/github/gvlproject/cloudbridge/master/landscape.svg?style=flat
    :target: https://landscape.io/github/gvlproject/cloudbridge/master

cloudbridge/__init__.py

@@ -2,7 +2,7 @@
 import logging
 
 # Current version of the library
-__version__ = '0.1.1'
+__version__ = '0.2.0'
 
 
 def get_version():

cloudbridge/cloud/base/provider.py

@@ -5,6 +5,7 @@ try:
 except ImportError:  # Python 2
     from ConfigParser import SafeConfigParser
 from os.path import expanduser
+import functools
 
 from cloudbridge.cloud.interfaces import CloudProvider
 from cloudbridge.cloud.interfaces.resources import Configuration
@@ -95,6 +96,10 @@ class BaseCloudProvider(CloudProvider):
             raise ProviderConnectionException(
                 "Authentication with cloud provider failed: %s" % (e,))
 
+    def _deepgetattr(self, obj, attr):
+        """Recurses through an attribute chain to get the ultimate value."""
+        return functools.reduce(getattr, attr.split('.'), obj)
+
     def has_service(self, service_type):
         """
         Checks whether this provider supports a given service.
@@ -106,10 +111,12 @@ class BaseCloudProvider(CloudProvider):
         :return: ``True`` if the service type is supported.
         """
         try:
-            if getattr(self, service_type):
+            if self._deepgetattr(self, service_type):
                 return True
         except AttributeError:
             pass  # Undefined service type
+        except NotImplementedError:
+            pass  # service not implemented
         return False
 
     def _get_config_value(self, key, default_value):

cloudbridge/cloud/base/resources.py

@@ -667,6 +667,9 @@ class BaseNetwork(BaseCloudResource, Network, BaseObjectLifeCycleMixin):
 
 class BaseSubnet(Subnet, BaseCloudResource):
 
+    CB_DEFAULT_SUBNET_NAME = os.environ.get('CB_DEFAULT_SUBNET_NAME',
+                                            'CloudBridgeSubnet')
+
     def __init__(self, provider):
         super(BaseSubnet, self).__init__(provider)
 
@@ -699,6 +702,9 @@ class BaseFloatingIP(FloatingIP, BaseCloudResource):
 
 class BaseRouter(Router, BaseCloudResource):
 
+    CB_DEFAULT_ROUTER_NAME = os.environ.get('CB_DEFAULT_ROUTER_NAME',
+                                            'CloudBridgeRouter')
+
     def __init__(self, provider):
         super(BaseRouter, self).__init__(provider)
 

cloudbridge/cloud/factory.py

@@ -48,19 +48,19 @@ class CloudProviderFactory(object):
                 if issubclass(cls, TestMockHelperMixin):
                     if self.provider_list.get(provider_id, {}).get(
                             'mock_class'):
-                        log.warn("Mock provider with id: %s is already "
-                                 "registered. Overriding with class: %s",
-                                 provider_id, cls)
+                        log.warning("Mock provider with id: %s is already "
+                                    "registered. Overriding with class: %s",
+                                    provider_id, cls)
                     self.provider_list[provider_id]['mock_class'] = cls
                 else:
                     if self.provider_list.get(provider_id, {}).get('class'):
-                        log.warn("Provider with id: %s is already "
-                                 "registered. Overriding with class: %s",
-                                 provider_id, cls)
+                        log.warning("Provider with id: %s is already "
+                                    "registered. Overriding with class: %s",
+                                    provider_id, cls)
                     self.provider_list[provider_id]['class'] = cls
             else:
-                log.warn("Provider class: %s implements CloudProvider but"
-                         " does not define PROVIDER_ID. Ignoring...", cls)
+                log.warning("Provider class: %s implements CloudProvider but"
+                            " does not define PROVIDER_ID. Ignoring...", cls)
         else:
             log.debug("Class: %s does not implement the CloudProvider"
                       "  interface. Ignoring...", cls)

cloudbridge/cloud/interfaces/resources.py

@@ -487,10 +487,6 @@ class Instance(ObjectLifeCycleMixin, CloudResource):
     def terminate(self):
         """
         Permanently terminate this instance.
-
-        :rtype: ``bool``
-        :return: ``True`` if the termination of the instance was successfully
-                 initiated; ``False`` otherwise.
         """
         pass
 
@@ -856,7 +852,7 @@ class Network(CloudResource):
         pass
 
     @abstractmethod
-    def create_subnet(self, cidr_block, name=None):
+    def create_subnet(self, cidr_block, name=None, zone=None):
         """
         Create a new network subnet and associate it with this Network.
 
@@ -868,6 +864,11 @@ class Network(CloudResource):
         :param name: An optional subnet name. The name will be set if the
                      provider supports it.
 
+        :type zone: ``str``
+        :param zone: Placement zone where to create the subnet. Some providers
+                     may not support subnet zones, in which case the value is
+                     ignored.
+
         :rtype: ``object`` of :class:`.Subnet`
         :return:  A Subnet object
         """
@@ -921,6 +922,18 @@ class Subnet(CloudResource):
         """
         pass
 
+    @abstractproperty
+    def zone(self):
+        """
+        Placement zone of the subnet.
+
+        If the provider does not support subnet placement, return ``None``.
+
+        :rtype: :class:`.PlacementZone` object
+        :return: Placement zone of the subnet, or ``None`` if not defined.
+        """
+        pass
+
     @abstractmethod
     def delete(self):
         """
@@ -1826,8 +1839,8 @@ class SecurityGroup(CloudResource):
         Create a security group rule. If the rule already exists, simply
         returns it.
 
-        You need to pass in either ``src_group`` OR ``ip_protocol``,
-        ``from_port``, ``to_port``, and ``cidr_ip``. In other words, either
+        You need to pass in either ``src_group`` OR ``ip_protocol`` AND
+        ``from_port``, ``to_port``, ``cidr_ip``. In other words, either
         you are authorizing another group or you are authorizing some
         ip-based rule.
 
@@ -1856,7 +1869,7 @@ class SecurityGroup(CloudResource):
         """
         Get a security group rule with the specified parameters.
 
-        You need to pass in either ``src_group`` OR ``ip_protocol``,
+        You need to pass in either ``src_group`` OR ``ip_protocol`` AND
         ``from_port``, ``to_port``, and ``cidr_ip``. Note that when retrieving
         a group rule, this method will return only one rule although possibly
         several rules exist for the group rule. In that case, use the
@@ -2037,6 +2050,16 @@ class BucketObject(CloudResource):
         """
         pass
 
+    @abstractmethod
+    def upload_from_file(self, path):
+        """
+        Store the contents of the file pointed by the "path" variable.
+
+        :type path: ``str``
+        :param path: Absolute path to the file to be uploaded to S3.
+        """
+        pass
+
     @abstractmethod
     def delete(self):
         """
@@ -2047,6 +2070,23 @@ class BucketObject(CloudResource):
         """
         pass
 
+    @abstractmethod
+    def generate_url(self, expires_in=0):
+        """
+        Generate a URL to this object.
+
+        If the object is public, `expires_in` argument is not necessary, but if
+        the object is private, the lifetime of URL is set using `expires_in`
+        argument.
+
+        :type expires_in: ``int``
+        :param expires_in: Time to live of the generated URL in seconds.
+
+        :rtype: ``str``
+        :return: A URL to access the object.
+        """
+        pass
+
 
 class Bucket(PageableObjectMixin, CloudResource):
 
@@ -2073,12 +2113,12 @@ class Bucket(PageableObjectMixin, CloudResource):
         pass
 
     @abstractmethod
-    def get(self, key):
+    def get(self, name):
         """
         Retrieve a given object from this bucket.
 
-        :type key: ``str``
-        :param key: the identifier of the object to retrieve
+        :type name: ``str``
+        :param name: The identifier of the object to retrieve
 
         :rtype: :class:``.BucketObject``
         :return: The BucketObject or ``None`` if it cannot be found.
@@ -2086,9 +2126,18 @@ class Bucket(PageableObjectMixin, CloudResource):
         pass
 
     @abstractmethod
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, prefix=None):
         """
-        List all objects within this bucket.
+        List objects in this bucket.
+
+        :type limit: ``int``
+        :param limit: Maximum number of elements to return.
+
+        :type marker: ``int``
+        :param marker: Fetch results after this offset.
+
+        :type prefix: ``str``
+        :param prefix: Prefix criteria by which to filter listed objects.
 
         :rtype: :class:``.BucketObject``
         :return: List of all available BucketObjects within this bucket.
@@ -2112,9 +2161,10 @@ class Bucket(PageableObjectMixin, CloudResource):
     @abstractmethod
     def create_object(self, name):
         """
-        Creates a new object within this bucket.
+        Create a new object within this bucket.
 
         :rtype: :class:``.BucketObject``
         :return: The newly created bucket object
         """
         pass
+

cloudbridge/cloud/interfaces/services.py

@@ -187,7 +187,12 @@ class InstanceService(PageableObjectMixin, CloudService):
                 print("Instance Data: {0}", instance)
 
         :type  limit: ``int``
-        :param limit: The maximum number of objects to return
+        :param limit: The maximum number of objects to return. Note that the
+                      maximum is not guaranteed to be honoured, and a lower
+                      maximum may be enforced depending on the provider. In
+                      such a case, the returned ResultList's is_truncated
+                      property can be used to determine whether more records
+                      are available.
 
         :type  marker: ``str``
         :param marker: The marker is an opaque identifier used to assist
@@ -200,7 +205,7 @@ class InstanceService(PageableObjectMixin, CloudService):
         pass
 
     @abstractmethod
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None,
                **kwargs):
@@ -218,33 +223,40 @@ class InstanceService(PageableObjectMixin, CloudService):
         :param instance_type: The InstanceType or name, specifying the size of
                               the instance to boot into
 
-        :type  network:  ``Network`` or ``str``
-        :param network:  The Network or an ID with which the instance should
-                         be associated. If no network was specified, this
-                         method will attempt to find a 'default' one and launch
-                         the instance using that network. A 'default' network
-                         is one tagged as such by the native API. If such tag
-                         or functionality does not exist, an attempt to create
-                         a new network (by default called 'CloudBridgeNet')
-                         will be made. If that falls through, an attempt will
-                         be made to launch the instance without specifying the
-                         network parameter (this is under the assumption the
-                         private networking functionality is not available on
-                         the provider).
+        :type  subnet:  ``Subnet`` or ``str``
+        :param subnet: The subnet object or a subnet string ID with which the
+                       instance should be associated. The subnet is a mandatory
+                       parameter, and must be provided when launching an
+                       instance.
+
+                       Note: Older clouds (with classic networking), may not
+                       have proper subnet support and are not guaranteed to
+                       work. Some providers (e.g. OpenStack) support a null
+                       value but the behaviour is implementation specific.
 
         :type  zone: ``Zone`` or ``str``
         :param zone: The Zone or its name, where the instance should be placed.
+                     This parameter is provided for legacy compatibility (with
+                     classic networks).
+
+                     The subnet's placement zone will take precedence over this
+                     parameter, but in its absence, this value will be used.
 
         :type  key_pair: ``KeyPair`` or ``str``
         :param key_pair: The KeyPair object or its name, to set for the
                          instance.
 
         :type  security_groups: A ``list`` of ``SecurityGroup`` objects or a
-                                list of ``str`` names
+                                list of ``str`` object IDs
         :param security_groups: A list of ``SecurityGroup`` objects or a list
-                                of ``SecurityGroup`` names, which should be
+                                of ``SecurityGroup`` IDs, which should be
                                 assigned to this instance.
 
+                                The security groups must be associated with the
+                                same network as the supplied subnet. Use
+                                ``network.security_groups`` to retrieve a list
+                                of security groups belonging to a network.
+
         :type  user_data: ``str``
         :param user_data: An extra userdata object which is compatible with
                           the provider.
@@ -655,7 +667,7 @@ class SubnetService(PageableObjectMixin, CloudService):
         pass
 
     @abstractmethod
-    def create(self, network_id, cidr_block, name=None):
+    def create(self, network_id, cidr_block, name=None, zone=None):
         """
         Create a new subnet within the supplied network.
 
@@ -670,11 +682,35 @@ class SubnetService(PageableObjectMixin, CloudService):
         :param name: An optional subnet name. The name will be set if the
                      provider supports it.
 
+        :type zone: ``str``
+        :param zone: An optional placement zone for the subnet. Some providers
+                     may not support this, in which case the value is ignored.
+
         :rtype: ``object`` of :class:`.Subnet`
         :return:  A Subnet object
         """
         pass
 
+    @abstractmethod
+    def get_or_create_default(self, zone=None):
+        """
+        Return a default subnet for the account or create one if not found.
+
+        A default network is one marked as such by the provider or matches the
+        default name used by this library (e.g., CloudBridgeNet).
+
+        If this method creates a new subnet, it will create one in each zone
+        available from the provider.
+
+        :type zone: ``str``
+        :param zone: Placement zone where to look for the subnet. If not
+                     supplied, a subnet from random zone will be selected.
+
+        :rtype: ``object`` of :class:`.Subnet`
+        :return: A Subnet object
+        """
+        pass
+
     @abstractmethod
     def delete(self, subnet):
         """
@@ -970,7 +1006,7 @@ class SecurityGroupService(PageableObjectMixin, CloudService):
     @abstractmethod
     def find(self, name, limit=None, marker=None):
         """
-        Get all security groups associated with your account.
+        Get security groups associated with your account filtered by name.
 
         :type name: str
         :param name: The name of the security group to retrieve.

cloudbridge/cloud/providers/aws/provider.py

@@ -38,6 +38,7 @@ class AWSCloudProvider(BaseCloudProvider):
             'aws_access_key', os.environ.get('AWS_ACCESS_KEY', None))
         self.s_key = self._get_config_value(
             'aws_secret_key', os.environ.get('AWS_SECRET_KEY', None))
+        self.session_token = self._get_config_value('aws_session_token', None)
         # EC2 connection fields
         self.ec2_is_secure = self._get_config_value('ec2_is_secure', True)
         self.region_name = self._get_config_value(
@@ -133,6 +134,7 @@ class AWSCloudProvider(BaseCloudProvider):
         vpc_conn = boto.connect_vpc(
             aws_access_key_id=self.a_key,
             aws_secret_access_key=self.s_key,
+            security_token=self.session_token,
             is_secure=self.ec2_is_secure,
             region=r,
             port=self.ec2_port,
@@ -147,6 +149,7 @@ class AWSCloudProvider(BaseCloudProvider):
         """
         s3_conn = boto.connect_s3(aws_access_key_id=self.a_key,
                                   aws_secret_access_key=self.s_key,
+                                  security_token=self.session_token,
                                   is_secure=self.s3_is_secure,
                                   port=self.s3_port,
                                   host=self.s3_host,
@@ -172,7 +175,7 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
         HTTPretty.register_uri(
             method="GET",
             uri="https://d168wakzal7fp0.cloudfront.net/aws_instance_data.json",
-            body="""
+            body=u"""
 [
   {
     "family": "General Purpose",
@@ -196,13 +199,12 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
     "storage": null,
     "max_bandwidth": 0,
     "instance_type": "t2.nano",
-    "ECU": "variable,
+    "ECU": "variable",
     "memory": 0.5,
     "ebs_max_bandwidth": 0
   }
 ]
-"""
-        )
+""")
 
     def tearDownMock(self):
         """

cloudbridge/cloud/providers/aws/resources.py

@@ -27,6 +27,7 @@ from cloudbridge.cloud.interfaces.resources import NetworkState
 from cloudbridge.cloud.interfaces.resources import RouterState
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
+
 from datetime import datetime
 import hashlib
 import inspect
@@ -34,6 +35,7 @@ import json
 
 from boto.exception import EC2ResponseError
 from boto.s3.key import Key
+
 from retrying import retry
 
 
@@ -646,7 +648,7 @@ class AWSSecurityGroup(BaseSecurityGroup):
         :return: Rule object if successful or ``None``.
         """
         try:
-            if not isinstance(src_group, SecurityGroup):
+            if src_group and not isinstance(src_group, SecurityGroup):
                 src_group = self._provider.security.security_groups.get(
                     src_group)
 
@@ -672,17 +674,17 @@ class AWSSecurityGroup(BaseSecurityGroup):
                  cidr_ip=None, src_group=None):
         for rule in self._security_group.rules:
             if (rule.ip_protocol == ip_protocol and
-               rule.from_port == from_port and
-               rule.to_port == to_port and
-               rule.grants[0].cidr_ip == cidr_ip) or \
-               (rule.grants[0].group_id == src_group.id if src_group and
-               hasattr(rule.grants[0], 'group_id') else False):
+                rule.from_port == from_port and
+                rule.to_port == to_port and
+                rule.grants[0].cidr_ip == cidr_ip) or \
+                    (rule.grants[0].group_id == src_group.id if src_group and
+                        hasattr(rule.grants[0], 'group_id') else False):
                 return AWSSecurityGroupRule(self._provider, rule, self)
         return None
 
     def to_json(self):
-        attr = inspect.getmembers(self, lambda a: not(inspect.isroutine(a)))
-        js = {k: v for(k, v) in attr if not k.startswith('_')}
+        attr = inspect.getmembers(self, lambda a: not (inspect.isroutine(a)))
+        js = {k: v for (k, v) in attr if not k.startswith('_')}
         json_rules = [r.to_json() for r in self.rules]
         js['rules'] = [json.loads(r) for r in json_rules]
         if js.get('network_id'):
@@ -738,8 +740,8 @@ class AWSSecurityGroupRule(BaseSecurityGroupRule):
         return None
 
     def to_json(self):
-        attr = inspect.getmembers(self, lambda a: not(inspect.isroutine(a)))
-        js = {k: v for(k, v) in attr if not k.startswith('_')}
+        attr = inspect.getmembers(self, lambda a: not (inspect.isroutine(a)))
+        js = {k: v for (k, v) in attr if not k.startswith('_')}
         js['group'] = self.group.id if self.group else ''
         js['parent'] = self.parent.id if self.parent else ''
         return json.dumps(js, sort_keys=True)
@@ -807,6 +809,12 @@ class AWSBucketObject(BaseBucketObject):
         """
         self._key.set_contents_from_string(data)
 
+    def upload_from_file(self, path):
+        """
+        Store the contents of the file pointed by the "path" variable.
+        """
+        self._key.set_contents_from_filename(path)
+
     def delete(self):
         """
         Delete this object.
@@ -816,6 +824,12 @@ class AWSBucketObject(BaseBucketObject):
         """
         self._key.delete()
 
+    def generate_url(self, expires_in=0):
+        """
+        Generate a URL to this object.
+        """
+        return self._key.generate_url(expires_in=expires_in)
+
 
 class AWSBucket(BaseBucket):
 
@@ -834,16 +848,16 @@ class AWSBucket(BaseBucket):
         """
         return self._bucket.name
 
-    def get(self, key):
+    def get(self, name):
         """
         Retrieve a given object from this bucket.
         """
-        key = Key(self._bucket, key)
-        if key.exists():
+        key = Key(self._bucket, name)
+        if key and key.exists():
             return AWSBucketObject(self._provider, key)
         return None
 
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, prefix=None):
         """
         List all objects within this bucket.
 
@@ -851,7 +865,8 @@ class AWSBucket(BaseBucket):
         :return: List of all available BucketObjects within this bucket.
         """
         objects = [AWSBucketObject(self._provider, obj)
-                   for obj in self._bucket.list()]
+                   for obj in self._bucket.list(prefix=prefix)]
+
         return ClientPagedResultList(self._provider, objects,
                                      limit=limit, marker=marker)
 
@@ -959,8 +974,9 @@ class AWSNetwork(BaseNetwork):
         subnets = self._provider.vpc_conn.get_all_subnets(filters=flter)
         return [AWSSubnet(self._provider, subnet) for subnet in subnets]
 
-    def create_subnet(self, cidr_block, name=None):
-        subnet = self._provider.vpc_conn.create_subnet(self.id, cidr_block)
+    def create_subnet(self, cidr_block, name=None, zone=None):
+        subnet = self._provider.vpc_conn.create_subnet(self.id, cidr_block,
+                                                       availability_zone=zone)
         cb_subnet = AWSSubnet(self._provider, subnet)
         if name:
             cb_subnet.name = name
@@ -1009,6 +1025,11 @@ class AWSSubnet(BaseSubnet):
     def network_id(self):
         return self._subnet.vpc_id
 
+    @property
+    def zone(self):
+        return AWSPlacementZone(self._provider, self._subnet.availability_zone,
+                                self._provider.region_name)
+
     def delete(self):
         return self._provider.vpc_conn.delete_subnet(subnet_id=self.id)
 

cloudbridge/cloud/providers/aws/services.py

@@ -4,7 +4,7 @@ import string
 
 from boto.ec2.blockdevicemapping import BlockDeviceMapping
 from boto.ec2.blockdevicemapping import BlockDeviceType
-from boto.exception import EC2ResponseError
+from boto.exception import EC2ResponseError, S3ResponseError
 
 from cloudbridge.cloud.base.resources import ClientPagedResultList
 from cloudbridge.cloud.base.resources import ServerPagedResultList
@@ -27,7 +27,6 @@ from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
-from cloudbridge.cloud.interfaces.resources import Network
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
@@ -50,8 +49,8 @@ from .resources import AWSSnapshot
 from .resources import AWSSubnet
 from .resources import AWSVolume
 
-import cloudbridge as cb
 # Uncomment to enable logging by default for this module
+# import cloudbridge as cb
 # cb.set_stream_logger(__name__)
 
 
@@ -363,11 +362,26 @@ class AWSObjectStoreService(BaseObjectStoreService):
         Returns a bucket given its ID. Returns ``None`` if the bucket
         does not exist.
         """
-        bucket = self.provider.s3_conn.lookup(bucket_id)
-        if bucket:
+        try:
+            # Make a call to make sure the bucket exists. While this would
+            # normally return a Bucket instance, there's an edge case where a
+            # 403 response can occur when the bucket exists but the
+            # user simply does not have permissions to access it. See below.
+            bucket = self.provider.s3_conn.get_bucket(bucket_id)
             return AWSBucket(self.provider, bucket)
-        else:
-            return None
+        except S3ResponseError as e:
+            # If 403, it means the bucket exists, but the user does not have
+            # permissions to access the bucket. However, limited operations
+            # may be permitted (with a session token for example), so return a
+            # Bucket instance to allow further operations.
+            # http://stackoverflow.com/questions/32331456/using-boto-upload-file-to-s3-
+            # sub-folder-when-i-have-no-permissions-on-listing-fo
+            if e.status == 403:
+                bucket = self.provider.s3_conn.get_bucket(bucket_id,
+                                                          validate=False)
+                return AWSBucket(self.provider, bucket)
+        # For all other responses, it's assumed that the bucket does not exist.
+        return None
 
     def find(self, name, limit=None, marker=None):
         """
@@ -467,13 +481,14 @@ class AWSInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(AWSInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None, **kwargs):
         image_id = image.id if isinstance(image, MachineImage) else image
         instance_size = instance_type.id if \
             isinstance(instance_type, InstanceType) else instance_type
-        network_id = network.id if isinstance(network, Network) else network
+        subnet = (self.provider.network.subnets.get(subnet)
+                  if isinstance(subnet, str) else subnet)
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         key_pair_name = key_pair.name if isinstance(
             key_pair,
@@ -484,7 +499,7 @@ class AWSInstanceService(BaseInstanceService):
             bdm = None
 
         subnet_id, zone_id, security_group_ids = \
-            self._resolve_launch_options(zone_id, network_id, security_groups)
+            self._resolve_launch_options(subnet, zone_id, security_groups)
 
         reservation = self.provider.ec2_conn.run_instances(
             image_id=image_id, instance_type=instance_size,
@@ -498,202 +513,37 @@ class AWSInstanceService(BaseInstanceService):
             instance.name = name
         return instance
 
-    def _resolve_launch_options(self, zone_id=None, vpc_id=None,
+    def _resolve_launch_options(self, subnet=None, zone_id=None,
                                 security_groups=None):
         """
         Work out interdependent launch options.
 
-        Some launch options are required and interdependent so work through
-        those constraints. There are 8 subsets of options so the logic works
-        through each of those combinations to figure out the proper launch
-        options.
+        Some launch options are required and interdependent so make sure
+        they conform to the interface contract.
+
+        :type subnet: ``Subnet``
+        :param subnet: Subnet object within which to launch.
 
         :type zone_id: ``str``
         :param zone_id: ID of the zone where the launch should happen.
 
-        :type vpc_id: ``str``
-        :param vpc_id: ID of the network within which to launch.
-
-        :type security_groups: ``list`` of ``str``
-        :param zone_id: List of security group names.
+        :type security_groups: ``list`` of ``id``
+        :param zone_id: List of security group IDs.
 
         :rtype: triplet of ``str``
         :return: Subnet ID, zone ID and security group IDs for launch.
 
-        :raise ValueError: In case a conflicting combination is found or the
-                           method cannot infer the defaults, raise.
-        """
-        def _get_default_vpc(vpcs, exc="No default network found."):
-            """
-            Inspect supplied VPCs to figure out a default one or create one.
-
-            Default VPC either has ``is_default`` property set or matches the
-            default network name used by this library. If a default network
-            is not found, an attempt to create one is made.
-
-            :type vpcs: ``list``
-            :param vpcs: A list of boto VPC objects.
-
-            :type exc: ``str``
-            :type exc: A string value to use if/when raising ValueError.
-
-            :rtype: ``str``
-            :return: Default VPC ID.
-            """
-            default_vpc = None
-            for vpc in vpcs:
-                if vpc.is_default:
-                    default_vpc = vpc.id
-                    break
-            if not default_vpc:
-                for vpc in vpcs:
-                    if vpc.tags.get('Name', '') == \
-                       AWSNetwork.CB_DEFAULT_NETWORK_NAME:
-                        default_vpc = vpc.id
-                        break
-            if not default_vpc:
-                net = self.provider.network.create(
-                    name=AWSNetwork.CB_DEFAULT_NETWORK_NAME)
-                default_vpc = net.id
-            return default_vpc
-
-        def _get_potential_subnets(filters, exc):
-            """
-            Query existing subnets through supplied filters.
-
-            :type filters: ``dict``
-            :param filters: A dictionary supplying desired filters.
-
-            :type exc: ``str``
-            :type exc: A string value to use if/when raising ValueError.
-
-            :rtype: tuple of ``str``
-            :return: A tuple with a random subnet that matches supplied
-                     filters and an availability zone where the given subnet
-                     is defined.
-
-            :raise ValueError: If no subnet can be found, raise a ValueError.
-            """
-            potential_subnets = self.provider.vpc_conn.get_all_subnets(
-                filters=filters)
-            if potential_subnets and len(potential_subnets) > 0:
-                sn_id = potential_subnets[0].id
-                zone_id = potential_subnets[0].availability_zone
-            else:
-                raise ValueError(exc)
-            return sn_id, zone_id
-
-        def _get_security_groups(security_groups, vpc_id=None, obj=False):
-            """
-            Resolve exact security groups to use.
-
-            :type security_groups: A ``list`` of ``SecurityGroup`` objects or
-                                   a list of ``str`` names.
-            :param security_groups: A list of ``SecurityGroup`` objects or a
-                                    list of ``SecurityGroup`` names, which
-                                    should be resolved.
-
-            :type vpc_id: ``str``
-            :param vpc_id: ID of the network within which to launch.
-
-            :type obj: ``bool``
-            :param obj: If True, return provider-native security group objects.
-                        Otherwise, return the IDs.
-
-            :rtype: list
-            :return: provider-native security group objects or the IDs (see
-                    ``obj`` param).
-            """
-            if isinstance(security_groups, list) and \
-               isinstance(security_groups[0], SecurityGroup):
-                return [sg._security_group if obj else sg.id
-                        for sg in security_groups]
-            else:
-                flters = {'group_name': security_groups}
-                if vpc_id:
-                    flters['vpc_id'] = vpc_id
-                sgs = self.provider.ec2_conn.get_all_security_groups(
-                    filters=flters)
-                return list(set([sg if obj else sg.id for sg in sgs]))
-
-        if zone_id and vpc_id and security_groups:
-            exc = "No subnets found in zone {0} for network {1}.".format(
-                zone_id, vpc_id)
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpcId': vpc_id}
-            sn_id, _ = _get_potential_subnets(flters, exc)
-            sg_ids = _get_security_groups(security_groups, vpc_id)
-        elif vpc_id and security_groups:
-            sg_ids = _get_security_groups(security_groups, vpc_id)
-            exc = "No subnets found in network {0}.".format(vpc_id)
-            flters = {'state': 'available', 'vpcId': vpc_id}
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
-        elif vpc_id and zone_id:
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpcId': vpc_id}
-            exc = "No subnets found in zone {0} for network {1}.".format(
-                zone_id, vpc_id)
-            sn_id, _ = _get_potential_subnets(flters, exc)
-            sg_ids = None
-        elif zone_id and security_groups:
-            sgs = _get_security_groups(security_groups, obj=True)
-            # Get VPCs the supplied SGs belong to
-            vpc_ids = list(set([sg.vpc_id for sg in sgs if sg.vpc_id]))
-            vpcs = []
-            if vpc_ids:
-                vpcs = self.provider.vpc_conn.get_all_vpcs(vpc_ids=vpc_ids)
-            exc = ("No default network found for zone {0} and security groups "
-                   "{1}".format(zone_id, security_groups))
-            default_vpc = _get_default_vpc(vpcs, exc)
-            # Filter only the SGs within the default VPC
-            sg_ids = _get_security_groups(security_groups, default_vpc)
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpc_id': default_vpc}
-            exc = "No subnets found in zone {0} for default network {1}." \
-                .format(zone_id, default_vpc)
-            sn_id, _ = _get_potential_subnets(flters, exc)
-        elif vpc_id:
-            flters = {'state': 'available', 'vpcId': vpc_id}
-            exc = "No subnets found for network {0}.".format(vpc_id)
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
-            sg_ids = None
-        elif zone_id:
-            vpcs = self.provider.vpc_conn.get_all_vpcs()
-            exc = "No default network exists for security zone {0}.".format(
-                zone_id)
-            default_vpc = _get_default_vpc(vpcs, exc)
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpcId': default_vpc}
-            exc = "No subnets found in zone {0} for default network {1}." \
-                .format(zone_id, default_vpc)
-            sn_id, _ = _get_potential_subnets(flters, exc)
-            sg_ids = None
-        elif security_groups:
-            sgs = _get_security_groups(security_groups, obj=True)
-            # Get VPCs the supplied SGs belong to
-            vpc_ids = list(set([sg.vpc_id for sg in sgs if sg.vpc_id]))
-            vpcs = []
-            if vpc_ids:
-                vpcs = self.provider.vpc_conn.get_all_vpcs(vpc_ids=vpc_ids)
-            exc = "No default network exists for security groups {0}.".format(
-                security_groups)
-            default_vpc = _get_default_vpc(vpcs, exc)
-            # Filter only the SGs within the default VPC
-            sg_ids = _get_security_groups(security_groups, default_vpc)
-            flters = {'state': 'available', 'vpcId': default_vpc}
-            exc = "No subnets found in network {0}.".format(default_vpc)
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
+        :raise ValueError: In case a conflicting combination is found.
+        """
+        if subnet:
+            # subnet's zone takes precedence
+            zone_id = subnet.zone.id
+        if isinstance(security_groups, list) and isinstance(
+                security_groups[0], SecurityGroup):
+            security_group_ids = [sg.id for sg in security_groups]
         else:
-            # Nothing was defined, use all defaults
-            vpcs = self.provider.vpc_conn.get_all_vpcs()
-            default_vpc = _get_default_vpc(vpcs)
-            flters = {'state': 'available', 'vpcId': default_vpc}
-            exc = "No subnets found for default network {1}.".format(
-                default_vpc)
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
-            sg_ids = None
-
-        return sn_id, zone_id, sg_ids
+            security_group_ids = security_groups
+        return subnet.id, zone_id, security_group_ids
 
     def _process_block_device_mappings(self, launch_config, zone=None):
         """
@@ -929,15 +779,43 @@ class AWSSubnetService(BaseSubnetService):
         subnets = self.provider.vpc_conn.get_all_subnets(filters=fltr)
         return [AWSSubnet(self.provider, subnet) for subnet in subnets]
 
-    def create(self, network, cidr_block, name=None):
+    def create(self, network, cidr_block, name=None, zone=None):
         network_id = network.id if isinstance(network, AWSNetwork) else network
-        subnet = self.provider.vpc_conn.create_subnet(network_id, cidr_block)
+        subnet = self.provider.vpc_conn.create_subnet(network_id, cidr_block,
+                                                      availability_zone=zone)
         cb_subnet = AWSSubnet(self.provider, subnet)
         if name:
             time.sleep(2)  # The subnet does not always get created in time
             cb_subnet.name = name
         return cb_subnet
 
+    def get_or_create_default(self, zone=None):
+        filtr = {'availabilityZone': zone} if zone else None
+        sns = self.provider.vpc_conn.get_all_subnets(filters=filtr)
+        for sn in sns:
+            if sn.defaultForAz:
+                return AWSSubnet(self.provider, sn)
+        # No provider-default Subnet exists, look for a library-default one
+        for sn in sns:
+            if sn.tags.get('Name') == AWSSubnet.CB_DEFAULT_SUBNET_NAME:
+                return AWSSubnet(self.provider, sn)
+        # No provider-default Subnet exists, try to create it (net + subnets)
+        default_net = self.provider.network.create(
+            name=AWSNetwork.CB_DEFAULT_NETWORK_NAME)
+        # Create a subnet in each of the region's zones
+        region = self.provider.compute.regions.get(
+            self.provider.vpc_conn.region.name)
+        default_sn = None
+        for i, z in enumerate(region.zones):
+            sn = self.create(default_net, '10.0.{0}.0/24'.format(i),
+                             AWSSubnet.CB_DEFAULT_SUBNET_NAME, z.name)
+            if zone and zone == z.name:
+                default_sn = sn
+        # No specific zone was supplied; return the last created subnet
+        if not default_sn:
+            default_sn = sn
+        return default_sn
+
     def delete(self, subnet):
         subnet_id = subnet.id if isinstance(subnet, AWSSubnet) else subnet
         return self.provider.vpc_conn.delete_subnet(subnet_id=subnet_id)

cloudbridge/cloud/providers/openstack/provider.py

@@ -45,9 +45,6 @@ class OpenStackCloudProvider(BaseCloudProvider):
             os.environ.get('OS_PROJECT_DOMAIN_NAME', None))
         self.user_domain_name = self._get_config_value(
             'os_user_domain_name', os.environ.get('OS_USER_DOMAIN_NAME', None))
-        self.identity_api_version = self._get_config_value(
-            'os_identity_api_version',
-            os.environ.get('OS_IDENTITY_API_VERSION', None))
 
         # Service connections, lazily initialized
         self._nova = None
@@ -235,9 +232,18 @@ class OpenStackCloudProvider(BaseCloudProvider):
 #                                     session=self.keystone.session)
 
     def _connect_swift(self):
+        storage_url = self._get_config_value(
+            'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
+        auth_token = self._get_config_value(
+            'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
+
         """Get an OpenStack Swift (object store) client object cloud."""
-        return swift_client.Connection(authurl=self.auth_url,
-                                       session=self._keystone_session)
+        if storage_url and auth_token:
+            return swift_client.Connection(preauthurl=storage_url,
+                                           preauthtoken=auth_token)
+        else:
+            return swift_client.Connection(authurl=self.auth_url,
+                                           session=self._keystone_session)
 
     def _connect_neutron(self):
         """Get an OpenStack Neutron (networking) client object cloud."""

cloudbridge/cloud/providers/openstack/resources.py

@@ -26,13 +26,16 @@ from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.providers.openstack import helpers as oshelpers
+
 import inspect
 import json
 
 import ipaddress
 
 from keystoneclient.v3.regions import Region
+
 import novaclient.exceptions as novaex
+
 import swiftclient.exceptions as swiftex
 
 
@@ -399,8 +402,8 @@ class OpenStackRegion(BaseRegion):
 
     @property
     def zones(self):
-        # detailed must be set to ``False`` because the (default) ``True``
-        # value requires Admin privileges
+        # ``detailed`` param must be set to ``False`` because the (default)
+        # ``True`` value requires Admin privileges
         if self.name == self._provider.region_name:  # optimisation
             zones = self._provider.nova.availability_zones.list(detailed=False)
         else:
@@ -412,8 +415,7 @@ class OpenStackRegion(BaseRegion):
                 # return an empty list
                 zones = []
 
-        return [OpenStackPlacementZone(self._provider, z.zoneName,
-                                       self._os_region)
+        return [OpenStackPlacementZone(self._provider, z.zoneName, self.name)
                 for z in zones]
 
 
@@ -691,7 +693,8 @@ class OpenStackNetwork(BaseNetwork):
                    .get('subnets', []))
         return [OpenStackSubnet(self._provider, subnet) for subnet in subnets]
 
-    def create_subnet(self, cidr_block, name=''):
+    def create_subnet(self, cidr_block, name='', zone=None):
+        """OpenStack has no support for subnet zones so the value is ignored"""
         subnet_info = {'name': name, 'network_id': self.id,
                        'cidr': cidr_block, 'ip_version': 4}
         subnet = (self._provider.neutron.create_subnet({'subnet': subnet_info})
@@ -728,6 +731,15 @@ class OpenStackSubnet(BaseSubnet):
     def network_id(self):
         return self._subnet.get('network_id', None)
 
+    @property
+    def zone(self):
+        """
+        OpenStack does not have a notion of placement zone for subnets.
+
+        Default to None.
+        """
+        return None
+
     def delete(self):
         if self.id in str(self._provider.neutron.list_subnets()):
             self._provider.neutron.delete_subnet(self.id)
@@ -875,8 +887,8 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
         """
         Create a security group rule.
 
-        You need to pass in either ``src_group`` OR ``ip_protocol``,
-        ``from_port``, ``to_port``, and ``cidr_ip``.  In other words, either
+        You need to pass in either ``src_group`` OR ``ip_protocol`` AND
+        ``from_port``, ``to_port``, ``cidr_ip``.  In other words, either
         you are authorizing another group or you are authorizing some
         ip-based rule.
 
@@ -902,20 +914,19 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
             if not isinstance(src_group, SecurityGroup):
                 src_group = self._provider.security.security_groups.get(
                     src_group)
-            for protocol in ['udp', 'tcp']:
-                existing_rule = self.get_rule(ip_protocol=ip_protocol,
-                                              from_port=1,
-                                              to_port=65535,
-                                              src_group=src_group)
-                if existing_rule:
-                    return existing_rule
-
-                rule = self._provider.nova.security_group_rules.create(
-                    parent_group_id=self._security_group.id,
-                    ip_protocol=protocol,
-                    from_port=1,
-                    to_port=65535,
-                    group_id=src_group.id)
+            existing_rule = self.get_rule(ip_protocol=ip_protocol,
+                                          from_port=from_port,
+                                          to_port=to_port,
+                                          src_group=src_group)
+            if existing_rule:
+                return existing_rule
+
+            rule = self._provider.nova.security_group_rules.create(
+                parent_group_id=self._security_group.id,
+                ip_protocol=ip_protocol,
+                from_port=from_port,
+                to_port=to_port,
+                group_id=src_group.id)
             if rule:
                 # We can only return one Rule so default to TCP (ie, last in
                 # the for loop above).
@@ -942,16 +953,16 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
 
     def get_rule(self, ip_protocol=None, from_port=None, to_port=None,
                  cidr_ip=None, src_group=None):
-        # Update SG object; otherwise, recently added rules do now show
+        # Update SG object; otherwise, recently added rules do not show
         self._security_group = self._provider.nova.security_groups.get(
             self._security_group)
         for rule in self._security_group.rules:
             if (rule['ip_protocol'] == ip_protocol and
                 rule['from_port'] == from_port and
                 rule['to_port'] == to_port and
-                rule['ip_range'].get('cidr') == cidr_ip) or \
-               (rule['group'].get('name') == src_group.name if src_group
-                    else False):
+                (rule['ip_range'].get('cidr') == cidr_ip or
+                 (rule['group'].get('name') == src_group.name if src_group
+                  else False))):
                 return OpenStackSecurityGroupRule(self._provider, rule, self)
         return None
 
@@ -1023,9 +1034,7 @@ class OpenStackBucketObject(BaseBucketObject):
 
     @property
     def name(self):
-        """
-        Get this object's name.
-        """
+        """Get this object's name."""
         return self._obj.get("name")
 
     @property
@@ -1037,10 +1046,7 @@ class OpenStackBucketObject(BaseBucketObject):
         return self._obj.get("last_modified")
 
     def iter_content(self):
-        """
-        Returns this object's content as an
-        iterable.
-        """
+        """Returns this object's content as an iterable."""
         _, content = self._provider.swift.get_object(
             self.cbcontainer.name, self.name, resp_chunk_size=65536)
         return content
@@ -1053,11 +1059,18 @@ class OpenStackBucketObject(BaseBucketObject):
         self._provider.swift.put_object(self.cbcontainer.name, self.name,
                                         data)
 
+    def upload_from_file(self, path):
+        """
+        Stores the contents of the file pointed by the "path" variable.
+        """
+        with open(path, 'r') as f:
+            self.upload(f.read())
+
     def delete(self):
         """
         Delete this object.
 
-        :rtype: bool
+        :rtype: ``bool``
         :return: True if successful
         """
         try:
@@ -1068,6 +1081,19 @@ class OpenStackBucketObject(BaseBucketObject):
                 return True
         return False
 
+    def generate_url(self, expires_in=0):
+        """
+        Generates a URL to this object.
+
+        If the object is public, `expires_in` argument is not necessary, but if
+        the object is private, the life time of URL is set using `expires_in`
+        argument.
+
+        See here for implementation details:
+        http://stackoverflow.com/a/37057172
+        """
+        raise NotImplementedError("This functionality is not implemented yet.")
+
 
 class OpenStackBucket(BaseBucket):
 
@@ -1086,19 +1112,23 @@ class OpenStackBucket(BaseBucket):
         """
         return self._bucket.get("name")
 
-    def get(self, key):
+    def get(self, name):
         """
         Retrieve a given object from this bucket.
+
+        FIXME: If multiple objects match the name as their name prefix,
+        all will be returned by the provider but this method will only
+        return the first element.
         """
         _, object_list = self._provider.swift.get_container(
-            self.name, prefix=key)
+            self.name, prefix=name)
         if object_list:
             return OpenStackBucketObject(self._provider, self,
                                          object_list[0])
         else:
             return None
 
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, prefix=None):
         """
         List all objects within this bucket.
 
@@ -1107,7 +1137,7 @@ class OpenStackBucket(BaseBucket):
         """
         _, object_list = self._provider.swift.get_container(
             self.name, limit=oshelpers.os_result_limit(self._provider, limit),
-            marker=marker)
+            marker=marker, prefix=prefix)
         cb_objects = [OpenStackBucketObject(
             self._provider, self, obj) for obj in object_list]
 

cloudbridge/cloud/providers/openstack/services.py

@@ -26,15 +26,17 @@ from cloudbridge.cloud.base.services import BaseVolumeService
 from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
-from cloudbridge.cloud.interfaces.resources import Network
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
+from cloudbridge.cloud.interfaces.resources import Subnet
 from cloudbridge.cloud.interfaces.resources import Volume
 from cloudbridge.cloud.providers.openstack import helpers as oshelpers
 
 from novaclient.exceptions import NotFound as NovaNotFound
 
+from neutronclient.common.exceptions import NeutronClientException
+
 from .resources import OpenStackBucket
 from .resources import OpenStackFloatingIP
 from .resources import OpenStackInstance
@@ -81,10 +83,10 @@ class OpenStackSecurityService(BaseSecurityService):
         """
         return self._security_groups
 
-    def get_ec2_credentials(self):
+    def get_or_create_ec2_credentials(self):
         """
         A provider specific method than returns the ec2 credentials for the
-        current user.
+        current user, or creates a new pair if one doesn't exist.
         """
         keystone = self.provider.keystone
         if hasattr(keystone, 'ec2'):
@@ -92,6 +94,10 @@ class OpenStackSecurityService(BaseSecurityService):
                           if cred.tenant_id == keystone.tenant_id]
             if user_creds:
                 return user_creds[0]
+            else:
+                return keystone.ec2.create(keystone.user_id,
+                                           keystone.tenant_id)
+
         return None
 
     def get_ec2_endpoints(self):
@@ -558,7 +564,7 @@ class OpenStackInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(OpenStackInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None,
                **kwargs):
@@ -568,7 +574,10 @@ class OpenStackInstanceService(BaseInstanceService):
             isinstance(instance_type, InstanceType) else \
             self.provider.compute.instance_types.find(
                 name=instance_type)[0].id
-        network_id = network.id if isinstance(network, Network) else network
+        network_id = subnet.network_id if isinstance(subnet, Subnet) else None
+        if not network_id and subnet:
+            network_id = (self.provider.network.subnets.get(subnet).network_id
+                          if isinstance(subnet, str) else None)
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         key_pair_name = key_pair.name if \
             isinstance(key_pair, KeyPair) else key_pair
@@ -583,8 +592,8 @@ class OpenStackInstanceService(BaseInstanceService):
         bdm = None
         if launch_config:
             bdm = self._to_block_device_mapping(launch_config)
-        net = self._get_network(network_id)
 
+        log.debug("Launching in network %s" % network_id)
         os_instance = self.provider.nova.servers.create(
             name,
             None if self._has_root_device(launch_config) else image_id,
@@ -596,7 +605,7 @@ class OpenStackInstanceService(BaseInstanceService):
             security_groups=security_groups_list,
             userdata=user_data,
             block_device_mapping_v2=bdm,
-            nics=net)
+            nics=[{'net-id': network_id}] if network_id else None)
         return OpenStackInstance(self.provider, os_instance)
 
     def _to_block_device_mapping(self, launch_config):
@@ -649,43 +658,6 @@ class OpenStackInstanceService(BaseInstanceService):
                 return True
         return False
 
-    def _get_network(self, network_id=None):
-        """
-        Format the network ID for the API call, figuring out a default network.
-
-        If a network_id is not supplied, figure out which is the default
-        network and use it. A default network is either marked as such by the
-        provider or matches the default network name defined within this
-        library (by default CloudBridgeNet). If a default network cannot be
-        found, attempt to create a new one.
-        """
-        if network_id:
-            return [{'net-id': network_id}]
-        for net in self.provider.network.list():
-            if net.name == OpenStackNetwork.CB_DEFAULT_NETWORK_NAME:
-                return [{'net-id': net.id}]
-        try:
-            # Try to create a complete, Internet-connected new network
-            net = self.provider.network.create(
-                name=OpenStackNetwork.CB_DEFAULT_NETWORK_NAME)
-            sn = net.create_subnet('10.0.0.0/16', '{0}Subnet'.format(
-                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME))
-            router = self.provider.network.create_router('{0}Router'.format(
-                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME))
-            for n in self.provider.network.list():
-                if n.external:
-                    external_net = n
-                    break
-            router.attach_network(external_net.id)
-            router.add_route(sn.id)
-            return [{'net-id': net.id}]
-        except Exception as exc:
-            # At this point we assume the provider does support user-defined
-            # networks so return None
-            log.warn("Exception occurred trying to create a default "
-                     "CloudBridge network: {0}".format(exc))
-            return None
-
     def create_launch_config(self):
         return BaseLaunchConfig(self.provider)
 
@@ -796,7 +768,8 @@ class OpenStackSubnetService(BaseSubnetService):
         subnets = self.provider.neutron.list_subnets().get('subnets', [])
         return [OpenStackSubnet(self.provider, subnet) for subnet in subnets]
 
-    def create(self, network, cidr_block, name=''):
+    def create(self, network, cidr_block, name='', zone=None):
+        """zone param is ignored."""
         network_id = (network.id if isinstance(network, OpenStackNetwork)
                       else network)
         subnet_info = {'name': name, 'network_id': network_id,
@@ -805,6 +778,31 @@ class OpenStackSubnetService(BaseSubnetService):
                   .get('subnet'))
         return OpenStackSubnet(self.provider, subnet)
 
+    def get_or_create_default(self, zone=None):
+        """
+        Subnet zone is not supported by OpenStack and is thus ignored.
+        """
+        try:
+            for sn in self.list():
+                if sn.name == OpenStackSubnet.CB_DEFAULT_SUBNET_NAME:
+                    return sn
+            # No default; create one
+            net = self.provider.network.create(
+                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME)
+            sn = net.create_subnet(cidr_block='10.0.0.0/24',
+                                   name=OpenStackSubnet.CB_DEFAULT_SUBNET_NAME)
+            router = self.provider.network.create_router(
+                OpenStackRouter.CB_DEFAULT_ROUTER_NAME)
+            for n in self.provider.network.list():
+                if n.external:
+                    external_net = n
+                    break
+            router.attach_network(external_net.id)
+            router.add_route(sn.id)
+            return sn
+        except NeutronClientException:
+            return None
+
     def delete(self, subnet):
         subnet_id = (subnet.id if isinstance(subnet, OpenStackSubnet)
                      else subnet)

docs/api_docs/cloud/exceptions.rst

@@ -0,0 +1,19 @@
+Exceptions
+==========
+
+.. contents:: :local:
+
+CloudBridgeBaseException
+------------------------
+.. autoclass:: cloudbridge.cloud.interfaces.exceptions.CloudBridgeBaseException
+    :members:
+
+WaitStateException
+------------------
+.. autoclass:: cloudbridge.cloud.interfaces.exceptions.WaitStateException
+    :members:
+
+InvalidConfigurationException
+-----------------------------
+.. autoclass:: cloudbridge.cloud.interfaces.exceptions.InvalidConfigurationException
+    :members:

docs/api_docs/cloud/resources.rst

@@ -8,21 +8,6 @@ CloudServiceType
 .. autoclass:: cloudbridge.cloud.interfaces.resources.CloudServiceType
     :members:
 
-CloudBridgeBaseException
-------------------------
-.. autoclass:: cloudbridge.cloud.interfaces.resources.CloudBridgeBaseException
-    :members:
-
-WaitStateException
-------------------
-.. autoclass:: cloudbridge.cloud.interfaces.resources.WaitStateException
-    :members:
-
-InvalidConfigurationException
------------------------------
-.. autoclass:: cloudbridge.cloud.interfaces.resources.InvalidConfigurationException
-    :members:
-
 ObjectLifeCycleMixin
 --------------------
 .. autoclass:: cloudbridge.cloud.interfaces.resources.ObjectLifeCycleMixin

docs/api_docs/ref.rst

@@ -10,3 +10,4 @@ This section includes the API documentation for the reference interface.
    cloud/providers.rst
    cloud/services.rst
    cloud/resources.rst
+   cloud/exceptions.rst

docs/getting_started.rst

@@ -94,9 +94,13 @@ on disk as a read-only file.
 Create a security group
 -----------------------
 Next, we need to create a security group and add a rule to allow ssh access.
+A security group needs to be associated with a private network, so we'll also
+need to fetch it.
 
 .. code-block:: python
 
+    provider.network.list()  # Find a desired network ID
+    net = provider.network.get('desired network ID')
     sg = provider.security.security_groups.create(
         'cloudbridge_intro', 'A security group used by CloudBridge', net.id)
     sg.add_rule('tcp', 22, 22, '0.0.0.0/0')
@@ -115,7 +119,7 @@ also add the network interface as a launch argument.
                        key=lambda x: x.vcpus*x.ram)[0]
     inst = provider.compute.instances.create(
         name='CloudBridge-intro', image=img, instance_type=inst_type,
-        key_pair=kp, security_groups=[sg])
+        network=net, key_pair=kp, security_groups=[sg])
     # Wait until ready
     inst.wait_till_ready()  # This is a blocking call
     # Show instance state

docs/topics/launch.rst

@@ -14,7 +14,7 @@ and 4 GB RAM.
 
 .. code-block:: python
 
-    img = provider.compute.images.get('ami-5ac2cd4d')  # Ubuntu 14.04 on AWS
+    img = provider.compute.images.get('ami-f4cc1de2')  # Ubuntu 16.04 on AWS
     inst_type = sorted([t for t in provider.compute.instance_types.list()
                         if t.vcpus >= 2 and t.ram >= 4],
                        key=lambda x: x.vcpus*x.ram)[0]
@@ -45,16 +45,17 @@ Private networking
 ~~~~~~~~~~~~~~~~~~
 Private networking gives you control over the networking setup for your
 instance(s) and is considered the preferred method for launching instances. To
-launch an instance with an explicit private network, just supply it as an
-additional argument to the ``create`` method:
+launch an instance with an explicit private network, supply a subnet within
+a network as an additional argument to the ``create`` method:
 
 .. code-block:: python
 
     provider.network.list()  # Find a desired network ID
     net = provider.network.get('desired network ID')
+    sn = net.subnets()[0]  # Get a handle on the desired subnet to launch with
     inst = provider.compute.instances.create(
         name='CloudBridge-VPC', image=img, instance_type=inst_type,
-        network=net, key_pair=kp, security_groups=[sg])
+        subnet=sn, key_pair=kp, security_groups=[sg])
 
 For more information on how to create and setup a private network, take a look
 at `Networking <./networking.html>`_.

docs/topics/networking.rst

@@ -15,9 +15,9 @@ Create a new private network
 ----------------------------
 Creating a private network is a simple, one-line command but appropriately
 connecting it so it has Internet access is a multi-step process:
-(1) create a network; (2) create a subnet within the network; (3) create a
+(1) create a network; (2) create a subnet within this network; (3) create a
 router; (4) attach the router to an external network; and (5) add a route to
-the router that links with with a subnet. For some providers, any network can
+the router that links with a subnet. For some providers, any network can
 be external (ie, connected to the Internet) while for others it's a specific,
 pre-defined one that exists in the an account by default. In order to properly
 connect the router, we need to ensure we're using an external network.
@@ -35,9 +35,9 @@ the block and allow up to 16 IP addresses into the subnet (``/28``).
     if not net.external:
         for n in self.provider.network.list():
             if n.external:
-                external_net = n
+                net = n
                 break
-    router.attach_network(external_net.id)
+    router.attach_network(net.id)
     router.add_route(sn.id)
 
 Retrieve an existing private network

docs/topics/setup.rst

@@ -2,10 +2,11 @@ Setup
 -----
 To initialize a connection to a cloud and get a provider object, you will
 need to provide the cloud's access credentials to CloudBridge. These may
-be provided in one of two ways:
+be provided in one of following ways:
 
 1. Environment variables
 2. A dictionary
+3. Configuration file
 
 Providing access credentials through environment variables
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -28,8 +29,8 @@ Mandatory variables  Optional Variables
 OS_AUTH_URL			 NOVA_SERVICE_NAME
 OS_USERNAME			 OS_COMPUTE_API_VERSION
 OS_PASSWORD			 OS_VOLUME_API_VERSION
-OS_PROJECT_NAME
-OS_REGION_NAME
+OS_PROJECT_NAME      OS_STORAGE_URL
+OS_REGION_NAME       OS_AUTH_TOKEN
 ===================  ==================
 
 
@@ -74,6 +75,8 @@ default_result_limit  Number of results that a ``.list()`` method should return.
 ====================  ==================
 Variable		      Description
 ====================  ==================
+aws_session_token     Session key for your AWS account (if using temporary
+                      credentials).
 ec2_is_secure         True to use an SSL connection. Default is ``True``.
 ec2_region_name       Default region name. Defaults to ``us-east-1``.
 ec2_region_endpoint   Endpoint to use. Default is ``ec2.us-east-1.amazonaws.com``.
@@ -92,20 +95,51 @@ s3_validate_certs     Whether to use SSL certificate verification. Default is
 ====================  ==================
 
 
+Providing access credentials in a file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+CloudBridge can also read credentials from a file on your local file system.
+The file should be placed in one of two locations: ``/etc/cloudbridge.ini`` or
+``~/.cloudbridge``. Each set of credentials should be delineated with the
+provider ID (e.g., ``openstack``, ``aws``) with the necessary credentials
+being supplied in YAML format. Note that only one set of credentials per
+cloud provider type can be supplied (i.e., via this method, it is not possible
+to provide credentials for two different OpenStack clouds).
+
+.. code-block:: bash
+
+    [openstack]
+    os_username: username
+    os_password: password
+    os_auth_url: auth url
+    os_user_domain_name: user domain name
+    os_project_domain_name: project domain name
+    os_project_name: project name
+
+    [aws]
+    aws_access_key: access key
+    aws_secret_key: secret key
+
+
 Other configuration variables
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In addition to the provider specific configuration variables above, there are
 some general configuration environment variables that apply to CloudBridge as
 a whole
 
-=====================  ==================
-Variable		       Description
-=====================  ==================
-CB_DEBUG               Setting ``CB_DEBUG=True`` will cause detailed debug
-                       output to be printed for each provider (including HTTP
-                       traces).
-CB_USE_MOCK_PROVIDERS  Setting this to ``True`` will cause the CloudBridge test
-                       suite to use mock drivers when available.
-CB_TEST_PROVIDER       Set this value to a valid :class:`.ProviderList` value
-                       such as ``aws``, to limit tests to that provider only.
-=====================  ==================
+======================  ==================
+Variable		            Description
+======================  ==================
+CB_DEBUG                Setting ``CB_DEBUG=True`` will cause detailed debug
+                        output to be printed for each provider (including HTTP
+                        traces).
+CB_USE_MOCK_PROVIDERS   Setting this to ``True`` will cause the CloudBridge test
+                        suite to use mock drivers when available.
+CB_TEST_PROVIDER        Set this value to a valid :class:`.ProviderList` value
+                        such as ``aws``, to limit tests to that provider only.
+CB_DEFAULT_SUBNET_NAME  Name to be used for a subnet that will be considered
+                        the 'default' by the library. This default will be used
+                        only in cases there is no subnet marked as the default by the provider.
+CB_DEFAULT_NETWORK_NAME Name to be used for a network that will be considered
+                        the 'default' by the library. This default will be used
+                        only in cases there is no network marked as the default by the provider.
+======================= ==================

docs/topics/testing.rst

@@ -38,18 +38,31 @@ This will run all the tests for all the environments defined in file
 ``tox.ini``.
 
 
-Specific environment
-~~~~~~~~~~~~~~~~~~~~
+Specific environment and infrastructure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you’d like to run the tests on a specific environment only, say Python 2.7,
-use a command like this: ``tox -e py27``. Alternativley, to use your default
-python, you can also run the test command directly ``python setup.py test``.
+against a specific infrastructure, say aws, use a command like this:
+``tox -e py27-aws``. The available provider names are listed in the
+`ProviderList`_ class (e.g., ``aws`` or ``openstack``).  
 
-Select infrastructure
-~~~~~~~~~~~~~~~~~~~~~
-You can also run the tests on a specific cloud only. To do so, export an
-environment variable ``CB_TEST_PROVIDER`` and specify the desired provider
-name. The available provider names are listed in the `ProviderList`_ class
-(e.g., ``aws`` or ``openstack``). Then, run the ``tox`` command.
+Specific test cases
+~~~~~~~~~~~~~~~~~~~~
+You can run a specific test case, as follows:
+``tox -- -s test.test_cloud_factory.CloudFactoryTestCase``
+
+It can also be restricted to a particular environment as follows:
+``tox -e "py27-aws" -- -s test.test_cloud_factory.CloudFactoryTestCase``
+
+Using unittest directly
+~~~~~~~~~~~~~~~~~~~~~~~
+You can also run the tests against your active virtual environment directly
+with ``python setup.py test``. You will need to set the ``CB_TEST_PROVIDER``
+and ``CB_USE_MOCK_PROVIDERS`` environment variables prior to running the tests,
+or they will default to ``CB_TEST_PROVIDER=aws`` and
+``CB_USE_MOCK_PROVIDERS=True``.
+
+You can also run a specific test case, as follows:
+``python setup.py test -s test.test_cloud_factory.CloudFactoryTestCase``
 
 Using a mock provider
 ~~~~~~~~~~~~~~~~~~~~~

setup.py

@@ -42,7 +42,7 @@ setup(name='cloudbridge',
       packages=find_packages(),
       license='MIT',
       classifiers=[
-          'Development Status :: 3 - Alpha',
+          'Development Status :: 4 - Beta',
           'Environment :: Console',
           'Intended Audience :: Developers',
           'Intended Audience :: System Administrators',
@@ -54,6 +54,7 @@ setup(name='cloudbridge',
           'Programming Language :: Python :: 3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
           'Programming Language :: Python :: Implementation :: CPython',
           'Programming Language :: Python :: Implementation :: PyPy'],
       test_suite="test"

test/__init__.py

@@ -1,64 +1,9 @@
 """
-Tests the functionality of each provider implementation against registered
-test cases. These tests require that provider credentials are set as
-environment variables, as required for each provider (see `tox.ini` for a list
-of env variables).
-
-Since the tests exercise the ``cloudbridge`` interfaces, and there are multiple
-implementations of these interfaces, for m interfaces and n implementation,
-exercising all interfaces means that m*n test case classes are needed.
-Otherwise, the standard test runners such as unittest and nose2 do not
-correctly pick up the tests.
-
-To avoid an explosion of repetitive test cases, the
-``ProviderTestCaseGenerator`` class will automatically generate a new Python
-class for each combination of test and provider. The ``load_tests`` protocol
-(https://docs.python.org/2/library/unittest.html#load-tests-protocol)
-is used to aid test discovery.
-
 Use ``python setup.py test`` to run these unit tests (alternatively, use
 ``python -m unittest test``).
 
-All test cases need to be registered below, and available providers will be
-discovered through the ``ProviderFactory``. Test Cases must not inherit from
-``unittest.TestCase``, to avoid confusing unittest and nose2's automatic
-discovery. (The test generator will automatically add ``unittest.TestCase``
-as a base class to each combination).
+You must set the CB_TEST_PROVIDER environment variable before running the
+tests. Otherwise, the test suite will default to running against the mock
+aws provider. Alternatively, use tox, which will run tests for all available
+provider combinations.
 """
-import cloudbridge
-from test.helpers import ProviderTestCaseGenerator
-from test.test_block_store_service import CloudBlockStoreServiceTestCase
-from test.test_cloud_helpers import CloudHelpersTestCase
-from test.test_compute_service import CloudComputeServiceTestCase
-from test.test_image_service import CloudImageServiceTestCase
-from test.test_instance_types_service import CloudInstanceTypesServiceTestCase
-from test.test_interface import CloudInterfaceTestCase
-from test.test_network_service import CloudNetworkServiceTestCase
-from test.test_object_life_cycle import CloudObjectLifeCycleTestCase
-from test.test_object_store_service import CloudObjectStoreServiceTestCase
-from test.test_region_service import CloudRegionServiceTestCase
-from test.test_security_service import CloudSecurityServiceTestCase
-
-
-PROVIDER_TESTS = [
-    CloudHelpersTestCase,
-    CloudInterfaceTestCase,
-    CloudObjectLifeCycleTestCase,
-    CloudSecurityServiceTestCase,
-    CloudNetworkServiceTestCase,
-    CloudInstanceTypesServiceTestCase,
-    CloudBlockStoreServiceTestCase,
-    CloudObjectStoreServiceTestCase,
-    CloudComputeServiceTestCase,
-    CloudRegionServiceTestCase,
-    CloudImageServiceTestCase
-]
-
-
-def load_tests(loader=None, tests=None, pattern=None):
-    """
-    This function is required to aid the load_tests protocol
-    (https://docs.python.org/2/library/unittest.html#load-tests-protocol)
-    """
-    cloudbridge.init_logging()
-    return ProviderTestCaseGenerator(PROVIDER_TESTS).generate_tests()

test/helpers.py

@@ -2,7 +2,7 @@ from contextlib import contextmanager
 import os
 import sys
 import unittest
-
+import functools
 from six import reraise
 
 from cloudbridge.cloud.factory import CloudProviderFactory
@@ -50,6 +50,28 @@ def cleanup_action(cleanup_func):
         print("Error during cleanup: {0}".format(e))
 
 
+def skipIfNoService(services):
+    """
+    A decorator for skipping tests if the provider
+    does not implement a given service.
+    """
+    def wrap(func):
+        """
+        The actual wrapper
+        """
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            provider = getattr(self, 'provider')
+            if provider:
+                for service in services:
+                    if not provider.has_service(service):
+                        self.skipTest("Skipping test because '%s' service is"
+                                      " not implemented" % (service,))
+            func(self, *args, **kwargs)
+        return wrapper
+    return wrap
+
+
 TEST_DATA_CONFIG = {
     "AWSCloudProvider": {
         "image": os.environ.get('CB_IMAGE_AWS', 'ami-5ac2cd4d'),
@@ -58,7 +80,7 @@ TEST_DATA_CONFIG = {
     },
     "OpenStackCloudProvider": {
         "image": os.environ.get('CB_IMAGE_OS',
-                                'a471339a-bd0e-41e2-9406-4f308267ed0f'),
+                                '842b949c-ea76-48df-998d-8a41f2626243'),
         "instance_type": os.environ.get('CB_INSTANCE_TYPE_OS', 'm1.tiny'),
         "placement": os.environ.get('CB_PLACEMENT_OS', 'nova'),
     },
@@ -85,7 +107,8 @@ def create_test_network(provider, name):
     """
     net = provider.network.create(name=name)
     cidr_block = (net.cidr_block).split('/')[0] or '10.0.0.1'
-    sn = net.create_subnet(cidr_block='{0}/28'.format(cidr_block, name=name))
+    sn = net.create_subnet(cidr_block='{0}/28'.format(cidr_block), name=name,
+                           zone=get_provider_test_data(provider, 'placement'))
     return net, sn
 
 
@@ -99,13 +122,13 @@ def delete_test_network(network):
 
 
 def create_test_instance(
-        provider, instance_name, network, zone=None, launch_config=None,
+        provider, instance_name, subnet, zone=None, launch_config=None,
         key_pair=None, security_groups=None):
     return provider.compute.instances.create(
         instance_name,
         get_provider_test_data(provider, 'image'),
         get_provider_test_data(provider, 'instance_type'),
-        network=network,
+        subnet=subnet,
         zone=zone,
         key_pair=key_pair,
         security_groups=security_groups,
@@ -113,12 +136,12 @@ def create_test_instance(
 
 
 def get_test_instance(provider, name, key_pair=None, security_groups=None,
-                      network=None):
+                      subnet=None):
     launch_config = None
     instance = create_test_instance(
         provider,
         name,
-        network=network,
+        subnet=subnet,
         key_pair=key_pair,
         security_groups=security_groups,
         launch_config=launch_config)
@@ -128,27 +151,20 @@ def get_test_instance(provider, name, key_pair=None, security_groups=None,
 
 def cleanup_test_resources(instance=None, network=None, security_group=None,
                            key_pair=None):
-    with cleanup_action(lambda: delete_test_network(network)):
-        with cleanup_action(lambda: key_pair.delete()):
-            with cleanup_action(lambda: security_group.delete()):
+    with cleanup_action(lambda: delete_test_network(network)
+                        if network else None):
+        with cleanup_action(lambda: key_pair.delete() if key_pair else None):
+            with cleanup_action(lambda: security_group.delete()
+                                if security_group else None):
                 instance.terminate()
                 instance.wait_for(
                     [InstanceState.TERMINATED, InstanceState.UNKNOWN],
                     terminal_states=[InstanceState.ERROR])
 
 
-class ProviderTestBase(object):
-
-    """
-    A dummy base class for Test Cases. Does not inherit from unittest.TestCase
-    to avoid confusing test discovery by unittest and nose2. unittest.TestCase
-    is injected as a base class by the generator, so calling the unittest
-    constructor works correctly.
-    """
+class ProviderTestBase(unittest.TestCase):
 
-    def __init__(self, methodName, provider):
-        unittest.TestCase.__init__(self, methodName=methodName)
-        self.provider = provider
+    _provider = None
 
     def setUp(self):
         if isinstance(self.provider, TestMockHelperMixin):
@@ -157,17 +173,7 @@ class ProviderTestBase(object):
     def tearDown(self):
         if isinstance(self.provider, TestMockHelperMixin):
             self.provider.tearDownMock()
-
-
-class ProviderTestCaseGenerator():
-
-    """
-    Generates test cases for all provider - testcase combinations.
-    Detailed docs at test/__init__.py
-    """
-
-    def __init__(self, test_classes):
-        self.all_test_classes = test_classes
+        self._provider = None
 
     def get_provider_wait_interval(self, provider_class):
         if issubclass(provider_class, TestMockHelperMixin):
@@ -175,80 +181,19 @@ class ProviderTestCaseGenerator():
         else:
             return 1
 
-    def create_provider_instance(self, provider_class):
-        """
-        Instantiate a default provider instance. All required connection
-        settings are expected to be set as environment variables.
-        """
+    def create_provider_instance(self):
+        provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
+        use_mock_drivers = parse_bool(
+            os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
+        factory = CloudProviderFactory()
+        provider_class = factory.get_provider_class(provider_name,
+                                                    get_mock=use_mock_drivers)
         config = {'default_wait_interval':
                   self.get_provider_wait_interval(provider_class)}
         return provider_class(config)
 
-    def generate_new_test_class(self, name, testcase_class):
-        """
-        Generates a new type which inherits from the given testcase_class and
-        unittest.TestCase
-        """
-        class_name = "{0}{1}".format(name, testcase_class.__name__)
-        return type(class_name, (testcase_class, unittest.TestCase), {})
-
-    def generate_test_suite_for_provider_testcase(
-            self, provider_class, testcase_class):
-        """
-        Generate and return a suite of tests for a specific provider class and
-        testcase combination
-        """
-        testloader = unittest.TestLoader()
-        testnames = testloader.getTestCaseNames(testcase_class)
-        suite = unittest.TestSuite()
-        for name in testnames:
-            generated_cls = self.generate_new_test_class(
-                provider_class.__name__,
-                testcase_class)
-            suite.addTest(
-                generated_cls(
-                    name,
-                    self.create_provider_instance(provider_class)))
-        return suite
-
-    def generate_test_suite_for_provider(self, provider_class):
-        """
-        Generate and return a suite of all available tests for a given provider
-        class
-        """
-        suite = unittest.TestSuite()
-        suites = [
-            self.generate_test_suite_for_provider_testcase(
-                provider_class, test_class)
-            for test_class in self.all_test_classes]
-        for s in suites:
-            suite.addTest(s)
-        return suite
-
-    def generate_tests(self):
-        """
-        Generate and return a suite of tests for all provider and test class
-        combinations
-        """
-        factory = CloudProviderFactory()
-        use_mock_drivers = parse_bool(
-            os.environ.get("CB_USE_MOCK_PROVIDERS", True))
-        provider_name = os.environ.get("CB_TEST_PROVIDER", None)
-        if provider_name:
-            provider_classes = [
-                factory.get_provider_class(
-                    provider_name,
-                    get_mock=use_mock_drivers)]
-            if not provider_classes[0]:
-                raise ValueError(
-                    "Could not find specified test provider %s" %
-                    provider_name)
-        else:
-            provider_classes = factory.get_all_provider_classes(
-                get_mock=use_mock_drivers)
-        suite = unittest.TestSuite()
-        suites = [
-            self.generate_test_suite_for_provider(p) for p in provider_classes]
-        for s in suites:
-            suite.addTest(s)
-        return suite
+    @property
+    def provider(self):
+        if not self._provider:
+            self._provider = self.create_provider_instance()
+        return self._provider

test/test_block_store_service.py

@@ -1,3 +1,4 @@
+import time
 import uuid
 
 import six
@@ -5,16 +6,14 @@ import six
 from cloudbridge.cloud.interfaces import SnapshotState
 from cloudbridge.cloud.interfaces import VolumeState
 from cloudbridge.cloud.interfaces.resources import AttachmentInfo
+
 from test.helpers import ProviderTestBase
 import test.helpers as helpers
 
 
 class CloudBlockStoreServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudBlockStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_crud_volume(self):
         """
         Create a new volume, check whether the expected values are set,
@@ -91,6 +90,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
             "Volume %s should have been deleted but still exists." %
             name)
 
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_attach_detach_volume(self):
         """
         Create a new volume, and attempt to attach it to an instance
@@ -98,9 +98,9 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
         instance_name = "CBVolOps-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, instance_name)
+        net, subnet = helpers.create_test_network(self.provider, instance_name)
         test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  network=net)
+                                                  subnet=subnet)
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
             name = "CBUnitTestAttachVol-{0}".format(uuid.uuid4())
@@ -117,6 +117,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                     [VolumeState.AVAILABLE],
                     terminal_states=[VolumeState.ERROR, VolumeState.DELETED])
 
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_volume_properties(self):
         """
         Test volume properties
@@ -125,9 +126,9 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
             self.provider.name,
             uuid.uuid4())
         vol_desc = 'newvoldesc1'
-        net, _ = helpers.create_test_network(self.provider, instance_name)
+        net, subnet = helpers.create_test_network(self.provider, instance_name)
         test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  network=net)
+                                                  subnet=subnet)
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
             name = "CBUnitTestVolProps-{0}".format(uuid.uuid4())
@@ -163,8 +164,9 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                                  "/dev/sda2")
                 test_vol.detach()
                 test_vol.name = 'newvolname1'
-                # Force a refresh before checking attachment status
-                test_vol.refresh()
+                test_vol.wait_for(
+                    [VolumeState.AVAILABLE],
+                    terminal_states=[VolumeState.ERROR, VolumeState.DELETED])
                 self.assertEqual(test_vol.name, 'newvolname1')
                 self.assertEqual(test_vol.description, vol_desc)
                 self.assertIsNone(test_vol.attachments)
@@ -172,6 +174,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                     [VolumeState.AVAILABLE],
                     terminal_states=[VolumeState.ERROR, VolumeState.DELETED])
 
+    @helpers.skipIfNoService(['block_store.snapshots'])
     def test_crud_snapshot(self):
         """
         Create a new volume, create a snapshot of the volume, and check
@@ -280,6 +283,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
 
             # Test creation of a snap via SnapshotService
             snap_too_name = "CBSnapToo-{0}".format(name)
+            time.sleep(15)  # Or get SnapshotCreationPerVolumeRateExceeded
             test_snap_too = self.provider.block_store.snapshots.create(
                 name=snap_too_name, volume=test_vol, description=snap_too_name)
             with helpers.cleanup_action(lambda: cleanup_snap(test_snap_too)):
@@ -289,6 +293,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                     "repr(obj) should contain the object id so that the object"
                     " can be reconstructed, but does not.")
 
+    @helpers.skipIfNoService(['block_store.snapshots'])
     def test_snapshot_properties(self):
         """
         Test snapshot properties

test/test_cloud_helpers.py

@@ -17,10 +17,6 @@ class DummyResult(object):
 
 class CloudHelpersTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudHelpersTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def setUp(self):
         super(CloudHelpersTestCase, self).setUp()
         self.objects = [DummyResult(1, "One"),

test/test_compute_service.py

@@ -1,28 +1,26 @@
+import ipaddress
 import uuid
 
-import ipaddress
 import six
-from cloudbridge.cloud.interfaces \
-    import InvalidConfigurationException
+
+from cloudbridge.cloud.interfaces import InvalidConfigurationException
 from cloudbridge.cloud.interfaces import InstanceState
 from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.exceptions import WaitStateException
+
 from test.helpers import ProviderTestBase
 import test.helpers as helpers
 
 
 class CloudComputeServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudComputeServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['compute.instances', 'network'])
     def test_crud_instance(self):
         name = "CBInstCrud-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, name)
-        inst = helpers.get_test_instance(self.provider, name, network=net)
+        net, subnet = helpers.create_test_network(self.provider, name)
+        inst = helpers.get_test_instance(self.provider, name, subnet=subnet)
 
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 inst, net)):
@@ -88,18 +86,21 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             return False
         return True
 
+    @helpers.skipIfNoService(['compute.instances', 'network',
+                              'security.security_groups',
+                              'security.key_pairs'])
     def test_instance_properties(self):
         name = "CBInstProps-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, name)
+        net, subnet = helpers.create_test_network(self.provider, name)
         kp = self.provider.security.key_pairs.create(name=name)
         sg = self.provider.security.security_groups.create(
             name=name, description=name, network_id=net.id)
         test_instance = helpers.get_test_instance(self.provider,
                                                   name, key_pair=kp,
                                                   security_groups=[sg],
-                                                  network=net)
+                                                  subnet=subnet)
 
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net, sg, kp)):
@@ -168,6 +169,8 @@ class CloudComputeServiceTestCase(ProviderTestBase):
                 "Instance type {0} does not match expected type {1}".format(
                     itype.name, expected_type))
 
+    @helpers.skipIfNoService(['compute.instances', 'compute.images',
+                              'compute.instance_types'])
     def test_block_device_mapping_launch_config(self):
         lc = self.provider.compute.instances.create_launch_config()
 
@@ -227,6 +230,8 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             "Expected %d total block devices bit found %d" %
             (2 + inst_type.num_ephemeral_disks, len(lc.block_devices)))
 
+    @helpers.skipIfNoService(['compute.instances', 'compute.images',
+                              'compute.instance_types', 'block_store.volumes'])
     def test_block_device_mapping_attachments(self):
         name = "CBInstBlkAttch-{0}-{1}".format(
             self.provider.name,
@@ -274,7 +279,7 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             # TODO: This should be greater than the ami size or tests
             # will fail on actual infrastructure. Needs an image.size
             # method
-            size=2,
+            size=8,
             delete_on_terminate=True)
 
         # Add all available ephemeral devices
@@ -286,14 +291,13 @@ class CloudComputeServiceTestCase(ProviderTestBase):
         for _ in range(inst_type.num_ephemeral_disks):
             lc.add_ephemeral_device()
 
-        net, _ = helpers.create_test_network(self.provider, name)
+        net, subnet = helpers.create_test_network(self.provider, name)
 
         inst = helpers.create_test_instance(
             self.provider,
             name,
-            network=net,
-            # We don't have a way to match the test net placement and this zone
-            # zone=helpers.get_provider_test_data(self.provider, 'placement'),
+            subnet=subnet,
+            zone=helpers.get_provider_test_data(self.provider, 'placement'),
             launch_config=lc)
 
         def cleanup(instance, net):

test/test_image_service.py

@@ -3,16 +3,15 @@ import uuid
 import six
 
 from cloudbridge.cloud.interfaces import MachineImageState
+
 from test.helpers import ProviderTestBase
 import test.helpers as helpers
 
 
 class CloudImageServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudImageServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['compute.images', 'network',
+                              'compute.instances'])
     def test_create_and_list_image(self):
         """
         Create a new image and check whether that image can be listed.
@@ -22,9 +21,9 @@ class CloudImageServiceTestCase(ProviderTestBase):
         instance_name = "CBImageTest-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, instance_name)
+        net, subnet = helpers.create_test_network(self.provider, instance_name)
         test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  network=net)
+                                                  subnet=subnet)
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
             name = "CBUnitTestListImg-{0}".format(uuid.uuid4())
@@ -48,26 +47,27 @@ class CloudImageServiceTestCase(ProviderTestBase):
                         test_image.description, six.string_types),
                     "Image description must be None or a string")
 
-                images = self.provider.compute.images.list()
-                list_images = [image for image in images
-                               if image.name == name]
-                self.assertTrue(
-                    len(list_images) == 1,
-                    "List images does not return the expected image %s" %
-                    name)
+                # This check won't work when >50 images are available
+                # images = self.provider.compute.images.list()
+                # list_images = [image for image in images
+                #                if image.name == name]
+                # self.assertTrue(
+                #     len(list_images) == 1,
+                #     "List images does not return the expected image %s" %
+                #     name)
 
                 # check iteration
                 iter_images = [image for image in self.provider.compute.images
                                if image.name == name]
                 self.assertTrue(
-                    len(iter_images) == 1,
-                    "Iter images does not return the expected image %s" %
-                    name)
+                    name in [ii.name for ii in iter_images],
+                    "Iter images (%s) does not contain the expected image %s" %
+                    (iter_images, name))
 
                 # find image
                 found_images = self.provider.compute.images.find(name=name)
                 self.assertTrue(
-                    len(found_images) == 1,
+                    name in [fi.name for fi in found_images],
                     "Find images error: expected image %s but found: %s" %
                     (name, found_images))
 
@@ -82,17 +82,15 @@ class CloudImageServiceTestCase(ProviderTestBase):
                 get_img = self.provider.compute.images.get(
                     test_image.id)
                 self.assertTrue(
-                    found_images[0] == iter_images[0] == get_img == test_image,
+                    found_images[0] == get_img == test_image,
                     "Objects returned by list: {0} and get: {1} are not as "
                     " expected: {2}" .format(found_images[0].id,
                                              get_img.id,
                                              test_image.id))
                 self.assertTrue(
-                    list_images[0].name == found_images[0].name ==
-                    get_img.name == test_image.name,
-                    "Names returned by list: {0}, find: {1} and get: {2} are"
-                    " not as expected: {3}" .format(list_images[0].name,
-                                                    found_images[0].name,
+                    found_images[0].name == get_img.name == test_image.name,
+                    "Names returned by find: {0} and get: {1} are"
+                    " not as expected: {2}" .format(found_images[0].name,
                                                     get_img.name,
                                                     test_image.name))
             # TODO: Images take a long time to deregister on EC2. Needs

test/test_instance_types_service.py

@@ -8,14 +8,11 @@ from test.helpers import ProviderTestBase
 
 class CloudInstanceTypesServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudInstanceTypesServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types(self):
         instance_types = self.provider.compute.instance_types.list()
-        # check iteration
-        iter_instance_types = list(self.provider.compute.instance_types)
+        # Check iteration, keeping the first 50 entries (the .list() default)
+        iter_instance_types = list(self.provider.compute.instance_types)[:50]
         self.assertListEqual(iter_instance_types, instance_types)
 
         for inst_type in instance_types:
@@ -69,6 +66,7 @@ class CloudInstanceTypesServiceTestCase(ProviderTestBase):
                     inst_type.extra_data, dict),
                 "InstanceType extra_data must be None or a dict")
 
+    @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types_find(self):
         """
         Searching for an instance by name should return an
@@ -91,6 +89,7 @@ class CloudInstanceTypesServiceTestCase(ProviderTestBase):
             self.provider.compute.instance_types.find(
                 non_existent_param="random_value")
 
+    @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types_get(self):
         """
         Searching for an instance by id should return an

test/test_interface.py

@@ -9,10 +9,6 @@ from cloudbridge.cloud.factory import CloudProviderFactory
 
 class CloudInterfaceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudInterfaceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def test_name_property(self):
         """
         Name should always return a value and should not raise an exception

test/test_network_service.py

@@ -1,17 +1,16 @@
 import test.helpers as helpers
 import uuid
 from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.interfaces.resources import RouterState
 
 
 class CloudNetworkServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudNetworkServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['network'])
     def test_crud_network_service(self):
         name = 'cbtestnetworkservice-{0}'.format(uuid.uuid4())
+        subnet_name = 'cbtestsubnetservice-{0}'.format(uuid.uuid4())
         net = self.provider.network.create(name=name)
         with helpers.cleanup_action(
             lambda:
@@ -32,21 +31,7 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 "Get network did not return the expected network {0}."
                 .format(name))
 
-        netl = self.provider.network.list()
-        found_net = [n for n in netl if n.name == name]
-        self.assertEqual(
-            len(found_net), 0,
-            "Network {0} should have been deleted but still exists."
-            .format(name))
-
-    def test_crud_subnet_service(self):
-        name = 'cbtestnetworkservice-{0}'.format(uuid.uuid4())
-        subnet_name = 'cbtestsubnetservice-{0}'.format(uuid.uuid4())
-        net = self.provider.network.create(name=name)
-        with helpers.cleanup_action(
-            lambda:
-                self.provider.network.delete(network_id=net.id)
-        ):
+            # check subnet
             subnet = self.provider.network.subnets.create(
                 network=net, cidr_block="10.0.0.1/24", name=subnet_name)
             with helpers.cleanup_action(
@@ -73,6 +58,38 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 "Subnet {0} should have been deleted but still exists."
                 .format(subnet_name))
 
+            # Check floating IP address
+            ip = self.provider.network.create_floating_ip()
+            ip_id = ip.id
+            with helpers.cleanup_action(lambda: ip.delete()):
+                ipl = self.provider.network.floating_ips()
+                self.assertTrue(
+                    ip in ipl,
+                    "Floating IP address {0} should exist in the list {1}"
+                    .format(ip.id, ipl))
+                # 2016-08: address filtering not implemented in moto
+                # empty_ipl = self.provider.network.floating_ips('dummy-net')
+                # self.assertFalse(
+                #     empty_ipl,
+                #     "Bogus network should not have any floating IPs: {0}"
+                #     .format(empty_ipl))
+                self.assertIn(
+                    ip.public_ip, repr(ip),
+                    "repr(obj) should contain the address public IP value.")
+                self.assertFalse(
+                    ip.private_ip,
+                    "Floating IP should not have a private IP value ({0})."
+                    .format(ip.private_ip))
+                self.assertFalse(
+                    ip.in_use(),
+                    "Newly created floating IP address should not be in use.")
+            ipl = self.provider.network.floating_ips()
+            found_ip = [a for a in ipl if a.id == ip_id]
+            self.assertTrue(
+                len(found_ip) == 0,
+                "Floating IP {0} should have been deleted but still exists."
+                .format(ip_id))
+
         netl = self.provider.network.list()
         found_net = [n for n in netl if n.name == name]
         self.assertEqual(
@@ -112,6 +129,9 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
             "Floating IP {0} should have been deleted but still exists."
             .format(ip_id))
 
+    @helpers.skipIfNoService(['network'])
+    @helpers.skipIfNoService(['network'])
+    @helpers.skipIfNoService(['network'])
     def test_crud_network(self):
         name = 'cbtestnetwork-{0}'.format(uuid.uuid4())
         subnet_name = 'cbtestsubnet-{0}'.format(uuid.uuid4())
@@ -135,7 +155,10 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 % net.cidr_block)
 
             cidr = '10.0.1.0/24'
-            sn = net.create_subnet(cidr_block=cidr, name=subnet_name)
+            sn = net.create_subnet(
+                cidr_block=cidr, name=subnet_name,
+                zone=helpers.get_provider_test_data(self.provider,
+                                                    'placement'))
             with helpers.cleanup_action(lambda: sn.delete()):
                 self.assertTrue(
                     sn.id in [s.id for s in net.subnets()],
@@ -152,6 +175,7 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                     "Subnet's CIDR %s should match the specified one %s." % (
                         sn.cidr_block, cidr))
 
+    @helpers.skipIfNoService(['network.routers'])
     def test_crud_router(self):
 
         def _cleanup(net, subnet, router):
@@ -165,7 +189,9 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
         router = self.provider.network.create_router(name=name)
         net = self.provider.network.create(name=name)
         cidr = '10.0.1.0/24'
-        sn = net.create_subnet(cidr_block=cidr, name=name)
+        sn = net.create_subnet(
+            cidr_block=cidr, name=name,
+            zone=helpers.get_provider_test_data(self.provider, 'placement'))
         with helpers.cleanup_action(lambda: _cleanup(net, sn, router)):
             # Check basic router properties
             self.assertIn(

test/test_object_life_cycle.py

@@ -8,10 +8,7 @@ import test.helpers as helpers
 
 class CloudObjectLifeCycleTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudObjectLifeCycleTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_object_life_cycle(self):
         """
         Test object life cycle methods by using a volume.

test/test_object_store_service.py

@@ -1,18 +1,21 @@
 from datetime import datetime
 from io import BytesIO
+from unittest import skip
 import uuid
 
+import requests
+
+import tempfile
+
 from cloudbridge.cloud.interfaces.resources import BucketObject
+
 from test.helpers import ProviderTestBase
 import test.helpers as helpers
 
 
 class CloudObjectStoreServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudObjectStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['object_store'])
     def test_crud_bucket(self):
         """
         Create a new bucket, check whether the expected values are set,
@@ -66,6 +69,7 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
             "Bucket %s should have been deleted but still exists." %
             name)
 
+    @helpers.skipIfNoService(['object_store'])
     def test_crud_bucket_objects(self):
         """
         Create a new bucket, upload some contents into the bucket, and
@@ -80,7 +84,8 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
         self.assertEqual([], objects)
 
         with helpers.cleanup_action(lambda: test_bucket.delete()):
-            obj_name = "hello_world.txt"
+            obj_name_prefix = "hello"
+            obj_name = obj_name_prefix + "_world.txt"
             obj = test_bucket.create_object(obj_name)
 
             self.assertTrue(
@@ -101,7 +106,7 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                     "Object size property needs to be a int, not {0}".format(
                         type(objs[0].size)))
                 self.assertTrue(
-                    datetime.strptime(objs[0].last_modified,
+                    datetime.strptime(objs[0].last_modified[:23],
                                       "%Y-%m-%dT%H:%M:%S.%f"),
                     "Object's last_modified field format {0} not matching."
                     .format(objs[0].last_modified))
@@ -130,6 +135,13 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                     isinstance(obj_too, BucketObject),
                     "Did not get object {0} of expected type.".format(obj_too))
 
+                prefix_filtered_list = test_bucket.list(prefix=obj_name_prefix)
+                self.assertTrue(
+                    len(objs) == len(prefix_filtered_list) == 1,
+                    'The number of objects returned by list function, '
+                    'with and without a prefix, are expected to be equal, '
+                    'but its detected otherwise.')
+
             objs = test_bucket.list()
             found_objs = [o for o in objs if o.name == obj_name]
             self.assertTrue(
@@ -137,8 +149,8 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                 "Object %s should have been deleted but still exists." %
                 obj_name)
 
+    @helpers.skipIfNoService(['object_store'])
     def test_upload_download_bucket_content(self):
-
         name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
         test_bucket = self.provider.object_store.create(name)
 
@@ -159,3 +171,42 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                 for data in obj.iter_content():
                     target_stream2.write(data)
                 self.assertEqual(target_stream2.getvalue(), content)
+
+    @skip("Skip until OpenStack implementation is provided")
+    @helpers.skipIfNoService(['object_store'])
+    def test_generate_url(self):
+        name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
+        test_bucket = self.provider.object_store.create(name)
+
+        with helpers.cleanup_action(lambda: test_bucket.delete()):
+            obj_name = "hello_upload_download.txt"
+            obj = test_bucket.create_object(obj_name)
+
+            with helpers.cleanup_action(lambda: obj.delete()):
+                content = b"Hello World. Generate a url."
+                obj.upload(content)
+                target_stream = BytesIO()
+                obj.save_content(target_stream)
+
+                url = obj.generate_url(100)
+                self.assertEqual(requests.get(url).content, content)
+
+    @helpers.skipIfNoService(['object_store'])
+    def test_upload_download_bucket_content_from_file(self):
+        name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
+        test_bucket = self.provider.object_store.create(name)
+
+        with helpers.cleanup_action(lambda: test_bucket.delete()):
+            obj_name = "hello_upload_download.txt"
+            obj = test_bucket.create_object(obj_name)
+
+            with helpers.cleanup_action(lambda: obj.delete()):
+                content = b"Hello World. Upload from file."
+                with tempfile.NamedTemporaryFile() as tmpFile:
+                    tmpFile.write(content)
+                    tmpFile.flush()
+
+                    obj.upload_from_file(tmpFile.name)
+                    target_stream = BytesIO()
+                    obj.save_content(target_stream)
+                    self.assertEqual(target_stream.getvalue(), content)

test/test_region_service.py

@@ -7,10 +7,7 @@ import test.helpers as helpers
 
 class CloudRegionServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudRegionServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['compute.regions'])
     def test_get_and_list_regions(self):
         """
         Test whether the region listing methods work,
@@ -47,6 +44,7 @@ class CloudRegionServiceTestCase(ProviderTestBase):
             "Region name {0} not in JSON representation {1}".format(
                 region.name, region.to_json()))
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_regions_unique(self):
         """
         Regions should not return duplicate items
@@ -55,6 +53,7 @@ class CloudRegionServiceTestCase(ProviderTestBase):
         unique_regions = set([region.id for region in regions])
         self.assertTrue(len(regions) == len(list(unique_regions)))
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_current_region(self):
         """
         RegionService.current should return a valid region
@@ -63,6 +62,7 @@ class CloudRegionServiceTestCase(ProviderTestBase):
         self.assertIsInstance(current_region, Region)
         self.assertTrue(current_region in self.provider.compute.regions.list())
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_zones(self):
         """
         Test whether regions return the correct zone information

test/test_security_service.py

@@ -1,20 +1,19 @@
 """Test cloudbridge.security modules."""
 import json
-from test.helpers import ProviderTestBase
-import time
+import unittest
 import uuid
 
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
+
+from test.helpers import ProviderTestBase
 import test.helpers as helpers
 
 
 class CloudSecurityServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudSecurityServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['security.key_pairs'])
     def test_crud_key_pair_service(self):
-        name = 'cbtestkeypairA-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestkeypairA-{0}'.format(uuid.uuid4())
         kp = self.provider.security.key_pairs.create(name=name)
         with helpers.cleanup_action(
             lambda:
@@ -64,8 +63,9 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             no_kp,
             "Found a key pair {0} that should not exist?".format(no_kp))
 
+    @helpers.skipIfNoService(['security.key_pairs'])
     def test_key_pair(self):
-        name = 'cbtestkeypairB-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestkeypairB-{0}'.format(uuid.uuid4())
         kp = self.provider.security.key_pairs.create(name=name)
         with helpers.cleanup_action(lambda: kp.delete()):
             kpl = self.provider.security.key_pairs.list()
@@ -100,8 +100,9 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 lambda: self.provider.network.delete(network_id=net.id)):
             self.provider.security.security_groups.delete(group_id=sg.id)
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_crud_security_group_service(self):
-        name = 'cbtestsecuritygroupA-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestsecuritygroupA-{0}'.format(uuid.uuid4())
         net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
             name=name, description=name, network_id=net.id)
@@ -155,9 +156,10 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             len(no_sg) == 0,
             "Found a bogus security group?!?".format(no_sg))
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group(self):
         """Test for proper creation of a security group."""
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4())
         net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
             name=name, description=name, network_id=net.id)
@@ -195,7 +197,7 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
 #                 sort_keys=True)
 #             self.assertTrue(
 #                 sg.to_json() == json_repr,
-#                 "JSON sec group representation {0} does not match expected {1}"
+#                 "JSON SG representation {0} does not match expected {1}"
 #                 .format(sg.to_json(), json_repr))
 
         sgl = self.provider.security.security_groups.list()
@@ -205,9 +207,15 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             "Security group {0} should have been deleted but still exists."
             .format(name))
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group_rule_add_twice(self):
         """Test whether adding the same rule twice succeeds."""
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4()).lower()
+        if isinstance(self.provider, TestMockHelperMixin):
+            raise unittest.SkipTest(
+                "Mock provider returns InvalidParameterValue: "
+                "Value security_group is invalid for parameter.")
+
+        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4())
         net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
             name=name, description=name, network_id=net.id)
@@ -222,9 +230,10 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 "Expected rule {0} not found in security group: {1}".format(
                     same_rule, sg.rules))
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group_group_rule(self):
         """Test for proper creation of a security group rule."""
-        name = 'cbtestsecuritygroupC-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestsecuritygroupC-{0}'.format(uuid.uuid4())
         net = self.provider.network.create(name=name)
         sg = self.provider.security.security_groups.create(
             name=name, description=name, network_id=net.id)
@@ -233,7 +242,7 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 len(sg.rules) == 0,
                 "Expected no security group group rule. Got {0}."
                 .format(sg.rules))
-            rule = sg.add_rule(src_group=sg, ip_protocol='tcp', from_port=0,
+            rule = sg.add_rule(src_group=sg, ip_protocol='tcp', from_port=1,
                                to_port=65535)
             self.assertTrue(
                 rule.group.name == name,

tox.ini

@@ -1,14 +1,30 @@
 # Tox (http://tox.testrun.org/) is a tool for running tests
 # in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
+# test suite on all supported python versions and providers.
+# To use it, "pip install tox" and then run "tox" from this directory.
+# You will have to set all required environment variables (below) before
+# running the tests.
+#
+# Alternatively, to run mock tests only, run tox as follows:
+# CB_USE_MOCK_PROVIDERS=True tox -e py27-aws
+#
+# Simply running tox -e py27-aws also works, because the default is to use
+# mock providers.
 
 [tox]
-envlist = py27, py35, pypy
+envlist = {py27,py36,pypy}-{aws,openstack,gce}
 
 [testenv]
-commands = {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test
-passenv = AWS_ACCESS_KEY AWS_SECRET_KEY GCE_CLIENT_EMAIL GCE_PROJECT_NAME GCE_DEFAULT_ZONE GCE_SERVICE_CREDS_FILE OS_AUTH_URL OS_PASSWORD OS_TENANT_NAME OS_USERNAME OS_REGION_NAME NOVA_SERVICE_NAME CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS CB_TEST_PROVIDER CB_USE_MOCK_PROVIDERS
+commands = {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test {posargs}
+setenv =
+    aws: CB_TEST_PROVIDER=aws
+    openstack: CB_TEST_PROVIDER=openstack
+    gce: CB_TEST_PROVIDER=gce
+passenv =
+    CB_USE_MOCK_PROVIDERS
+    aws: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY
+    openstack:  CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS OS_AUTH_URL OS_PASSWORD OS_PROJECT_NAME OS_TENANT_NAME OS_USERNAME OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME NOVA_SERVICE_NAME
+    gce: CB_IMAGE_GCE CB_INSTANCE_TYPE_GCE CB_PLACEMENT_GCE GCE_CLIENT_EMAIL GCE_PROJECT_NAME GCE_DEFAULT_ZONE GCE_SERVICE_CREDS_FILE
 deps =
     -rrequirements.txt
-    coverage
+    coverage