Merge pull request #33 from gvlproject/nuwan_gce_changes

Merge latest master into GCE + other fixes

.gitignore

@@ -57,3 +57,4 @@ docs/_build/
 target/
 
 *.DS_Store
+/venv/

.travis.yml

@@ -1,12 +1,15 @@
 language: python
-python: 3.5
+python: 3.6
 os:
   - linux
 #  - osx
 env:
-  - TOX_ENV=py27
-  - TOX_ENV=py35
-  - TOX_ENV=pypy
+  - TOX_ENV=py27-aws
+  - TOX_ENV=py27-openstack
+  - TOX_ENV=py36-aws
+  - TOX_ENV=py36-openstack
+  - TOX_ENV=pypy-aws
+  - TOX_ENV=pypy-openstack
 matrix:
   fast_finish: true
   allow_failures:
@@ -19,4 +22,4 @@ script:
   - tox -e $TOX_ENV
 after_success:
   - coveralls
-  - codecov 
+  - codecov

CHANGELOG.rst

@@ -1,4 +1,46 @@
-0.1.1 - Aug 10, 2016.
+0.3.2 - June 10, 2017. (sha f07f3cbd758a0872b847b5537d9073c90f87c24d)
+-------
+
+* Patch release to support files>5GB with OpenStack (thanks @MartinPaulo)
+* Misc bug fixes
+
+0.3.1 - April 18, 2017. (sha f36a462e886d8444cb2818f6573677ecf0565315)
+-------
+
+* Patch for binary file handling in openstack
+
+0.3.0 - April 11, 2017. (sha 13539ccda9e4809082796574d18b1b9bb3f2c624)
+-------
+
+* Reworked test framework to rely on tox's test generation features. This
+  allows for individual test cases to be run on a per provider basis.
+* Added more OpenStack swift config options (OS_AUTH_TOKEN and OS_STORAGE_URL)
+* Added supports for accessing EC2 containers with restricted permissions.
+* Removed exists() method from object store interface. Use get()==None check
+  instead.
+* New method (img.min_disk) for geting size of machine image.
+* Test improvements (flake8 during build, more tests)
+* Misc bug fixes and improvements
+* Changed library to beta state
+* General documentation updates (testing, release process)
+
+0.2.0 - March 23, 2017. (sha a442d96b829ea2c721728520b01981fa61774625)
+-------
+
+* Reworked the instance launch method to require subnet vs. network. This
+  removed the option of adding network interface to a launch config object.
+* Added object store methods: upload from file path, list objects with a
+  prefix, check if an object exists, (AWS only) get an accessible URL for an
+  object (thanks @VJalili)
+* Modified `get_ec2_credentials()` method to `get_or_create_ec2_credentials()`
+* Added an option to read provider config values from a file
+  (`~/.cloudbridge` or `/etc/cloudbridge`)
+* Replaced py35 with py36 for running tests
+* Added logging configuration for the library
+* General documentation updates
+
+
+0.1.1 - Aug 10, 2016. (sha 0122fb1173c88ae64e40140ffd35ff3797e9e4ad)
 -------
 
 * For AWS, always launch instances into private networking (i.e., VPC)

README.rst

@@ -1,7 +1,6 @@
 CloudBridge aims to provide a simple layer of abstraction over
 different cloud providers, reducing or eliminating the need to write
-conditional code for each cloud. It is currently under development and is in
-an Alpha state.
+conditional code for each cloud.
 
 .. image:: https://landscape.io/github/gvlproject/cloudbridge/master/landscape.svg?style=flat
    :target: https://landscape.io/github/gvlproject/cloudbridge/master

cloudbridge/__init__.py

@@ -2,7 +2,7 @@
 import logging
 
 # Current version of the library
-__version__ = '0.1.1'
+__version__ = '0.3.2'
 
 
 def get_version():
@@ -32,6 +32,7 @@ class NullHandler(logging.Handler):
         """Don't emit a log."""
         pass
 
+
 TRACE = 5  # Lower than debug which is 10
 
 
@@ -46,13 +47,13 @@ class CBLogger(logging.Logger):
         """Add ``trace`` log level."""
         self.log(TRACE, msg, *args, **kwargs)
 
+
 # By default, do not force any logging by the library. If you want to see the
 # log messages in your scripts, add the following to the top of your script:
 #   import cloudbridge
 #   cloudbridge.set_stream_logger(__name__)
 #   OR
 #   cloudbridge.set_file_logger(__name__, '/tmp/cb.log')
-
 default_format_string = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
 logging.setLoggerClass(CBLogger)
 logging.addLevelName(TRACE, "TRACE")

cloudbridge/cloud/base/provider.py

@@ -1,14 +1,16 @@
 """Base implementation of a provider interface."""
+import functools
 import os
+from os.path import expanduser
 try:
     from configparser import SafeConfigParser
 except ImportError:  # Python 2
     from ConfigParser import SafeConfigParser
-from os.path import expanduser
 
 from cloudbridge.cloud.interfaces import CloudProvider
-from cloudbridge.cloud.interfaces.resources import Configuration
 from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
+from cloudbridge.cloud.interfaces.resources import Configuration
+
 
 DEFAULT_RESULT_LIMIT = 50
 DEFAULT_WAIT_TIMEOUT = 600
@@ -95,6 +97,10 @@ class BaseCloudProvider(CloudProvider):
             raise ProviderConnectionException(
                 "Authentication with cloud provider failed: %s" % (e,))
 
+    def _deepgetattr(self, obj, attr):
+        """Recurses through an attribute chain to get the ultimate value."""
+        return functools.reduce(getattr, attr.split('.'), obj)
+
     def has_service(self, service_type):
         """
         Checks whether this provider supports a given service.
@@ -106,10 +112,12 @@ class BaseCloudProvider(CloudProvider):
         :return: ``True`` if the service type is supported.
         """
         try:
-            if getattr(self, service_type):
+            if self._deepgetattr(self, service_type):
                 return True
         except AttributeError:
             pass  # Undefined service type
+        except NotImplementedError:
+            pass  # service not implemented
         return False
 
     def _get_config_value(self, key, default_value):

cloudbridge/cloud/base/resources.py

@@ -9,14 +9,14 @@ import os
 import shutil
 import time
 
-import six
-
 from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
+from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 from cloudbridge.cloud.interfaces.resources import AttachmentInfo
 from cloudbridge.cloud.interfaces.resources import Bucket
 from cloudbridge.cloud.interfaces.resources import BucketObject
 from cloudbridge.cloud.interfaces.resources import CloudResource
+from cloudbridge.cloud.interfaces.resources import FloatingIP
 from cloudbridge.cloud.interfaces.resources import Instance
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import InstanceType
@@ -30,18 +30,17 @@ from cloudbridge.cloud.interfaces.resources import ObjectLifeCycleMixin
 from cloudbridge.cloud.interfaces.resources import PageableObjectMixin
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import Region
-from cloudbridge.cloud.interfaces.resources import Router
 from cloudbridge.cloud.interfaces.resources import ResultList
+from cloudbridge.cloud.interfaces.resources import Router
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SecurityGroupRule
 from cloudbridge.cloud.interfaces.resources import Snapshot
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import Subnet
-from cloudbridge.cloud.interfaces.resources import FloatingIP
 from cloudbridge.cloud.interfaces.resources import Volume
 from cloudbridge.cloud.interfaces.resources import VolumeState
-from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 
+import six
 
 log = logging.getLogger(__name__)
 
@@ -667,6 +666,9 @@ class BaseNetwork(BaseCloudResource, Network, BaseObjectLifeCycleMixin):
 
 class BaseSubnet(Subnet, BaseCloudResource):
 
+    CB_DEFAULT_SUBNET_NAME = os.environ.get('CB_DEFAULT_SUBNET_NAME',
+                                            'CloudBridgeSubnet')
+
     def __init__(self, provider):
         super(BaseSubnet, self).__init__(provider)
 
@@ -699,6 +701,9 @@ class BaseFloatingIP(FloatingIP, BaseCloudResource):
 
 class BaseRouter(Router, BaseCloudResource):
 
+    CB_DEFAULT_ROUTER_NAME = os.environ.get('CB_DEFAULT_ROUTER_NAME',
+                                            'CloudBridgeRouter')
+
     def __init__(self, provider):
         super(BaseRouter, self).__init__(provider)
 

cloudbridge/cloud/base/services.py

@@ -16,6 +16,7 @@ from cloudbridge.cloud.interfaces.services import SecurityService
 from cloudbridge.cloud.interfaces.services import SnapshotService
 from cloudbridge.cloud.interfaces.services import SubnetService
 from cloudbridge.cloud.interfaces.services import VolumeService
+
 from .resources import BasePageableObjectMixin
 
 

cloudbridge/cloud/factory.py

@@ -1,11 +1,12 @@
-from cloudbridge.cloud import providers
-from cloudbridge.cloud.interfaces import CloudProvider
-from cloudbridge.cloud.interfaces import TestMockHelperMixin
-from collections import defaultdict
 import importlib
 import inspect
 import logging
 import pkgutil
+from collections import defaultdict
+
+from cloudbridge.cloud import providers
+from cloudbridge.cloud.interfaces import CloudProvider
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
 
 log = logging.getLogger(__name__)
@@ -48,19 +49,19 @@ class CloudProviderFactory(object):
                 if issubclass(cls, TestMockHelperMixin):
                     if self.provider_list.get(provider_id, {}).get(
                             'mock_class'):
-                        log.warn("Mock provider with id: %s is already "
-                                 "registered. Overriding with class: %s",
-                                 provider_id, cls)
+                        log.warning("Mock provider with id: %s is already "
+                                    "registered. Overriding with class: %s",
+                                    provider_id, cls)
                     self.provider_list[provider_id]['mock_class'] = cls
                 else:
                     if self.provider_list.get(provider_id, {}).get('class'):
-                        log.warn("Provider with id: %s is already "
-                                 "registered. Overriding with class: %s",
-                                 provider_id, cls)
+                        log.warning("Provider with id: %s is already "
+                                    "registered. Overriding with class: %s",
+                                    provider_id, cls)
                     self.provider_list[provider_id]['class'] = cls
             else:
-                log.warn("Provider class: %s implements CloudProvider but"
-                         " does not define PROVIDER_ID. Ignoring...", cls)
+                log.warning("Provider class: %s implements CloudProvider but"
+                            " does not define PROVIDER_ID. Ignoring...", cls)
         else:
             log.debug("Class: %s does not implement the CloudProvider"
                       "  interface. Ignoring...", cls)

cloudbridge/cloud/interfaces/resources.py

@@ -487,10 +487,6 @@ class Instance(ObjectLifeCycleMixin, CloudResource):
     def terminate(self):
         """
         Permanently terminate this instance.
-
-        :rtype: ``bool``
-        :return: ``True`` if the termination of the instance was successfully
-                 initiated; ``False`` otherwise.
         """
         pass
 
@@ -584,6 +580,26 @@ class Instance(ObjectLifeCycleMixin, CloudResource):
         """
         pass
 
+    @abstractmethod
+    def add_security_group(self, sg):
+        """
+        Add a security group to this instance
+
+        :type sg: ``SecurityGroup``
+        :param sg: The SecurityGroup to associate with the instance.
+        """
+        pass
+
+    @abstractmethod
+    def remove_security_group(self, sg):
+        """
+        Remove a security group from this instance
+
+        :type sg: ``SecurityGroup``
+        :param sg: The SecurityGroup to associate with the instance.
+        """
+        pass
+
 
 class MachineImageState(object):
 
@@ -746,6 +762,17 @@ class MachineImage(ObjectLifeCycleMixin, CloudResource):
         """
         pass
 
+    @abstractproperty
+    def min_disk(self):
+        """
+        Returns the minimum size of the disk that's required to
+        boot this image (in GB)
+
+        :rtype: ``int``
+        :return: The minimum disk size needed by this image
+        """
+        pass
+
     @abstractmethod
     def delete(self):
         """
@@ -856,7 +883,7 @@ class Network(CloudResource):
         pass
 
     @abstractmethod
-    def create_subnet(self, cidr_block, name=None):
+    def create_subnet(self, cidr_block, name=None, zone=None):
         """
         Create a new network subnet and associate it with this Network.
 
@@ -868,6 +895,11 @@ class Network(CloudResource):
         :param name: An optional subnet name. The name will be set if the
                      provider supports it.
 
+        :type zone: ``str``
+        :param zone: Placement zone where to create the subnet. Some providers
+                     may not support subnet zones, in which case the value is
+                     ignored.
+
         :rtype: ``object`` of :class:`.Subnet`
         :return:  A Subnet object
         """
@@ -921,6 +953,18 @@ class Subnet(CloudResource):
         """
         pass
 
+    @abstractproperty
+    def zone(self):
+        """
+        Placement zone of the subnet.
+
+        If the provider does not support subnet placement, return ``None``.
+
+        :rtype: :class:`.PlacementZone` object
+        :return: Placement zone of the subnet, or ``None`` if not defined.
+        """
+        pass
+
     @abstractmethod
     def delete(self):
         """
@@ -1826,8 +1870,8 @@ class SecurityGroup(CloudResource):
         Create a security group rule. If the rule already exists, simply
         returns it.
 
-        You need to pass in either ``src_group`` OR ``ip_protocol``,
-        ``from_port``, ``to_port``, and ``cidr_ip``. In other words, either
+        You need to pass in either ``src_group`` OR ``ip_protocol`` AND
+        ``from_port``, ``to_port``, ``cidr_ip``. In other words, either
         you are authorizing another group or you are authorizing some
         ip-based rule.
 
@@ -1856,7 +1900,7 @@ class SecurityGroup(CloudResource):
         """
         Get a security group rule with the specified parameters.
 
-        You need to pass in either ``src_group`` OR ``ip_protocol``,
+        You need to pass in either ``src_group`` OR ``ip_protocol`` AND
         ``from_port``, ``to_port``, and ``cidr_ip``. Note that when retrieving
         a group rule, this method will return only one rule although possibly
         several rules exist for the group rule. In that case, use the
@@ -2037,6 +2081,16 @@ class BucketObject(CloudResource):
         """
         pass
 
+    @abstractmethod
+    def upload_from_file(self, path):
+        """
+        Store the contents of the file pointed by the "path" variable.
+
+        :type path: ``str``
+        :param path: Absolute path to the file to be uploaded to S3.
+        """
+        pass
+
     @abstractmethod
     def delete(self):
         """
@@ -2047,6 +2101,23 @@ class BucketObject(CloudResource):
         """
         pass
 
+    @abstractmethod
+    def generate_url(self, expires_in=0):
+        """
+        Generate a URL to this object.
+
+        If the object is public, `expires_in` argument is not necessary, but if
+        the object is private, the lifetime of URL is set using `expires_in`
+        argument.
+
+        :type expires_in: ``int``
+        :param expires_in: Time to live of the generated URL in seconds.
+
+        :rtype: ``str``
+        :return: A URL to access the object.
+        """
+        pass
+
 
 class Bucket(PageableObjectMixin, CloudResource):
 
@@ -2073,12 +2144,12 @@ class Bucket(PageableObjectMixin, CloudResource):
         pass
 
     @abstractmethod
-    def get(self, key):
+    def get(self, name):
         """
         Retrieve a given object from this bucket.
 
-        :type key: ``str``
-        :param key: the identifier of the object to retrieve
+        :type name: ``str``
+        :param name: The identifier of the object to retrieve
 
         :rtype: :class:``.BucketObject``
         :return: The BucketObject or ``None`` if it cannot be found.
@@ -2086,9 +2157,18 @@ class Bucket(PageableObjectMixin, CloudResource):
         pass
 
     @abstractmethod
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, prefix=None):
         """
-        List all objects within this bucket.
+        List objects in this bucket.
+
+        :type limit: ``int``
+        :param limit: Maximum number of elements to return.
+
+        :type marker: ``int``
+        :param marker: Fetch results after this offset.
+
+        :type prefix: ``str``
+        :param prefix: Prefix criteria by which to filter listed objects.
 
         :rtype: :class:``.BucketObject``
         :return: List of all available BucketObjects within this bucket.
@@ -2112,7 +2192,7 @@ class Bucket(PageableObjectMixin, CloudResource):
     @abstractmethod
     def create_object(self, name):
         """
-        Creates a new object within this bucket.
+        Create a new object within this bucket.
 
         :rtype: :class:``.BucketObject``
         :return: The newly created bucket object

cloudbridge/cloud/interfaces/services.py

@@ -187,7 +187,12 @@ class InstanceService(PageableObjectMixin, CloudService):
                 print("Instance Data: {0}", instance)
 
         :type  limit: ``int``
-        :param limit: The maximum number of objects to return
+        :param limit: The maximum number of objects to return. Note that the
+                      maximum is not guaranteed to be honoured, and a lower
+                      maximum may be enforced depending on the provider. In
+                      such a case, the returned ResultList's is_truncated
+                      property can be used to determine whether more records
+                      are available.
 
         :type  marker: ``str``
         :param marker: The marker is an opaque identifier used to assist
@@ -200,7 +205,7 @@ class InstanceService(PageableObjectMixin, CloudService):
         pass
 
     @abstractmethod
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None,
                **kwargs):
@@ -218,33 +223,40 @@ class InstanceService(PageableObjectMixin, CloudService):
         :param instance_type: The InstanceType or name, specifying the size of
                               the instance to boot into
 
-        :type  network:  ``Network`` or ``str``
-        :param network:  The Network or an ID with which the instance should
-                         be associated. If no network was specified, this
-                         method will attempt to find a 'default' one and launch
-                         the instance using that network. A 'default' network
-                         is one tagged as such by the native API. If such tag
-                         or functionality does not exist, an attempt to create
-                         a new network (by default called 'CloudBridgeNet')
-                         will be made. If that falls through, an attempt will
-                         be made to launch the instance without specifying the
-                         network parameter (this is under the assumption the
-                         private networking functionality is not available on
-                         the provider).
+        :type  subnet:  ``Subnet`` or ``str``
+        :param subnet: The subnet object or a subnet string ID with which the
+                       instance should be associated. The subnet is a mandatory
+                       parameter, and must be provided when launching an
+                       instance.
+
+                       Note: Older clouds (with classic networking), may not
+                       have proper subnet support and are not guaranteed to
+                       work. Some providers (e.g. OpenStack) support a null
+                       value but the behaviour is implementation specific.
 
         :type  zone: ``Zone`` or ``str``
         :param zone: The Zone or its name, where the instance should be placed.
+                     This parameter is provided for legacy compatibility (with
+                     classic networks).
+
+                     The subnet's placement zone will take precedence over this
+                     parameter, but in its absence, this value will be used.
 
         :type  key_pair: ``KeyPair`` or ``str``
         :param key_pair: The KeyPair object or its name, to set for the
                          instance.
 
         :type  security_groups: A ``list`` of ``SecurityGroup`` objects or a
-                                list of ``str`` names
+                                list of ``str`` object IDs
         :param security_groups: A list of ``SecurityGroup`` objects or a list
-                                of ``SecurityGroup`` names, which should be
+                                of ``SecurityGroup`` IDs, which should be
                                 assigned to this instance.
 
+                                The security groups must be associated with the
+                                same network as the supplied subnet. Use
+                                ``network.security_groups`` to retrieve a list
+                                of security groups belonging to a network.
+
         :type  user_data: ``str``
         :param user_data: An extra userdata object which is compatible with
                           the provider.
@@ -655,7 +667,7 @@ class SubnetService(PageableObjectMixin, CloudService):
         pass
 
     @abstractmethod
-    def create(self, network_id, cidr_block, name=None):
+    def create(self, network_id, cidr_block, name=None, zone=None):
         """
         Create a new subnet within the supplied network.
 
@@ -670,11 +682,35 @@ class SubnetService(PageableObjectMixin, CloudService):
         :param name: An optional subnet name. The name will be set if the
                      provider supports it.
 
+        :type zone: ``str``
+        :param zone: An optional placement zone for the subnet. Some providers
+                     may not support this, in which case the value is ignored.
+
         :rtype: ``object`` of :class:`.Subnet`
         :return:  A Subnet object
         """
         pass
 
+    @abstractmethod
+    def get_or_create_default(self, zone=None):
+        """
+        Return a default subnet for the account or create one if not found.
+
+        A default network is one marked as such by the provider or matches the
+        default name used by this library (e.g., CloudBridgeNet).
+
+        If this method creates a new subnet, it will create one in each zone
+        available from the provider.
+
+        :type zone: ``str``
+        :param zone: Placement zone where to look for the subnet. If not
+                     supplied, a subnet from random zone will be selected.
+
+        :rtype: ``object`` of :class:`.Subnet`
+        :return: A Subnet object
+        """
+        pass
+
     @abstractmethod
     def delete(self, subnet):
         """
@@ -970,7 +1006,7 @@ class SecurityGroupService(PageableObjectMixin, CloudService):
     @abstractmethod
     def find(self, name, limit=None, marker=None):
         """
-        Get all security groups associated with your account.
+        Get security groups associated with your account filtered by name.
 
         :type name: str
         :param name: The name of the security group to retrieve.

cloudbridge/cloud/providers/aws/provider.py

@@ -1,6 +1,4 @@
-"""
-Provider implementation based on boto library for AWS-compatible clouds.
-"""
+"""Provider implementation based on boto library for AWS-compatible clouds."""
 
 import os
 
@@ -9,8 +7,8 @@ from boto.ec2.regioninfo import RegionInfo
 try:
     # These are installed only for the case of a dev instance
     from httpretty import HTTPretty
-    from moto.ec2 import mock_ec2
-    from moto.s3 import mock_s3
+    from moto import mock_ec2
+    from moto import mock_s3
 except ImportError:
     # TODO: Once library logging is configured, change this
     print("[aws provider] moto library not available!")
@@ -28,6 +26,8 @@ from .services import AWSSecurityService
 class AWSCloudProvider(BaseCloudProvider):
 
     PROVIDER_ID = 'aws'
+    AWS_INSTANCE_DATA_DEFAULT_URL = "https://d168wakzal7fp0.cloudfront.net/" \
+                                    "aws_instance_data.json"
 
     def __init__(self, config):
         super(AWSCloudProvider, self).__init__(config)
@@ -38,6 +38,7 @@ class AWSCloudProvider(BaseCloudProvider):
             'aws_access_key', os.environ.get('AWS_ACCESS_KEY', None))
         self.s_key = self._get_config_value(
             'aws_secret_key', os.environ.get('AWS_SECRET_KEY', None))
+        self.session_token = self._get_config_value('aws_session_token', None)
         # EC2 connection fields
         self.ec2_is_secure = self._get_config_value('ec2_is_secure', True)
         self.region_name = self._get_config_value(
@@ -133,6 +134,7 @@ class AWSCloudProvider(BaseCloudProvider):
         vpc_conn = boto.connect_vpc(
             aws_access_key_id=self.a_key,
             aws_secret_access_key=self.s_key,
+            security_token=self.session_token,
             is_secure=self.ec2_is_secure,
             region=r,
             port=self.ec2_port,
@@ -147,6 +149,7 @@ class AWSCloudProvider(BaseCloudProvider):
         """
         s3_conn = boto.connect_s3(aws_access_key_id=self.a_key,
                                   aws_secret_access_key=self.s_key,
+                                  security_token=self.session_token,
                                   is_secure=self.s3_is_secure,
                                   port=self.s3_port,
                                   host=self.s3_host,
@@ -170,9 +173,9 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
         self.s3mock = mock_s3()
         self.s3mock.start()
         HTTPretty.register_uri(
-            method="GET",
-            uri="https://d168wakzal7fp0.cloudfront.net/aws_instance_data.json",
-            body="""
+            HTTPretty.GET,
+            self.AWS_INSTANCE_DATA_DEFAULT_URL,
+            body=u"""
 [
   {
     "family": "General Purpose",
@@ -196,13 +199,12 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
     "storage": null,
     "max_bandwidth": 0,
     "instance_type": "t2.nano",
-    "ECU": "variable,
+    "ECU": "variable",
     "memory": 0.5,
     "ebs_max_bandwidth": 0
   }
 ]
-"""
-        )
+""")
 
     def tearDownMock(self):
         """

cloudbridge/cloud/providers/aws/resources.py

@@ -1,9 +1,19 @@
 """
 DataTypes used by this provider
 """
+import hashlib
+import inspect
+import json
+
+from datetime import datetime
+
+from boto.exception import EC2ResponseError
+from boto.s3.key import Key
+
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
 from cloudbridge.cloud.base.resources import BaseBucket
 from cloudbridge.cloud.base.resources import BaseBucketObject
+from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseInstance
 from cloudbridge.cloud.base.resources import BaseInstanceType
 from cloudbridge.cloud.base.resources import BaseKeyPair
@@ -17,23 +27,16 @@ from cloudbridge.cloud.base.resources import BaseSecurityGroup
 from cloudbridge.cloud.base.resources import BaseSecurityGroupRule
 from cloudbridge.cloud.base.resources import BaseSnapshot
 from cloudbridge.cloud.base.resources import BaseSubnet
-from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseVolume
 from cloudbridge.cloud.base.resources import ClientPagedResultList
-from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import MachineImageState
 from cloudbridge.cloud.interfaces.resources import NetworkState
 from cloudbridge.cloud.interfaces.resources import RouterState
+from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
-from datetime import datetime
-import hashlib
-import inspect
-import json
 
-from boto.exception import EC2ResponseError
-from boto.s3.key import Key
 from retrying import retry
 
 
@@ -83,6 +86,18 @@ class AWSMachineImage(BaseMachineImage):
         """
         return self._ec2_image.description
 
+    @property
+    def min_disk(self):
+        """
+        Returns the minimum size of the disk that's required to
+        boot this image (in GB)
+
+        :rtype: ``int``
+        :return: The minimum disk size needed by this image
+        """
+        bdm = self._ec2_image.block_device_mapping
+        return bdm[self._ec2_image.root_device_name].size
+
     def delete(self):
         """
         Delete this image
@@ -342,7 +357,7 @@ class AWSInstance(BaseInstance):
         """
         if self._ec2_instance.vpc_id:
             aid = self._provider._vpc_conn.get_all_addresses([ip_address])[0]
-            return self._provider._ec2_conn.associate_address(
+            return self._provider.ec2_conn.associate_address(
                 self._ec2_instance.id, allocation_id=aid.allocation_id)
         else:
             return self._ec2_instance.use_ip(ip_address)
@@ -351,8 +366,28 @@ class AWSInstance(BaseInstance):
         """
         Remove a elastic IP address from this instance.
         """
-        raise NotImplementedError(
-            'remove_floating_ip not implemented by this provider.')
+        ip_addr = self._provider._vpc_conn.get_all_addresses([ip_address])[0]
+        if self._ec2_instance.vpc_id:
+            return self._provider.ec2_conn.disassociate_address(
+                association_id=ip_addr.association_id)
+        else:
+            return self._provider.ec2_conn.disassociate_address(
+                public_ip=ip_addr.public_ip)
+
+    def add_security_group(self, sg):
+        """
+        Add a security group to this instance
+        """
+        self._ec2_instance.modify_attribute(
+            'groupSet', [g.id for g in self._ec2_instance.groups] + [sg.id])
+
+    def remove_security_group(self, sg):
+        """
+        Remove a security group from this instance
+        """
+        self._ec2_instance.modify_attribute(
+            'groupSet', [g.id for g in self._ec2_instance.groups
+                         if g.id != sg.id])
 
     @property
     def state(self):
@@ -646,7 +681,7 @@ class AWSSecurityGroup(BaseSecurityGroup):
         :return: Rule object if successful or ``None``.
         """
         try:
-            if not isinstance(src_group, SecurityGroup):
+            if src_group and not isinstance(src_group, SecurityGroup):
                 src_group = self._provider.security.security_groups.get(
                     src_group)
 
@@ -672,17 +707,17 @@ class AWSSecurityGroup(BaseSecurityGroup):
                  cidr_ip=None, src_group=None):
         for rule in self._security_group.rules:
             if (rule.ip_protocol == ip_protocol and
-               rule.from_port == from_port and
-               rule.to_port == to_port and
-               rule.grants[0].cidr_ip == cidr_ip) or \
-               (rule.grants[0].group_id == src_group.id if src_group and
-               hasattr(rule.grants[0], 'group_id') else False):
+                rule.from_port == from_port and
+                rule.to_port == to_port and
+                rule.grants[0].cidr_ip == cidr_ip) or \
+                    (rule.grants[0].group_id == src_group.id if src_group and
+                        hasattr(rule.grants[0], 'group_id') else False):
                 return AWSSecurityGroupRule(self._provider, rule, self)
         return None
 
     def to_json(self):
-        attr = inspect.getmembers(self, lambda a: not(inspect.isroutine(a)))
-        js = {k: v for(k, v) in attr if not k.startswith('_')}
+        attr = inspect.getmembers(self, lambda a: not (inspect.isroutine(a)))
+        js = {k: v for (k, v) in attr if not k.startswith('_')}
         json_rules = [r.to_json() for r in self.rules]
         js['rules'] = [json.loads(r) for r in json_rules]
         if js.get('network_id'):
@@ -738,8 +773,8 @@ class AWSSecurityGroupRule(BaseSecurityGroupRule):
         return None
 
     def to_json(self):
-        attr = inspect.getmembers(self, lambda a: not(inspect.isroutine(a)))
-        js = {k: v for(k, v) in attr if not k.startswith('_')}
+        attr = inspect.getmembers(self, lambda a: not (inspect.isroutine(a)))
+        js = {k: v for (k, v) in attr if not k.startswith('_')}
         js['group'] = self.group.id if self.group else ''
         js['parent'] = self.parent.id if self.parent else ''
         return json.dumps(js, sort_keys=True)
@@ -807,6 +842,12 @@ class AWSBucketObject(BaseBucketObject):
         """
         self._key.set_contents_from_string(data)
 
+    def upload_from_file(self, path):
+        """
+        Store the contents of the file pointed by the "path" variable.
+        """
+        self._key.set_contents_from_filename(path)
+
     def delete(self):
         """
         Delete this object.
@@ -816,6 +857,12 @@ class AWSBucketObject(BaseBucketObject):
         """
         self._key.delete()
 
+    def generate_url(self, expires_in=0):
+        """
+        Generate a URL to this object.
+        """
+        return self._key.generate_url(expires_in=expires_in)
+
 
 class AWSBucket(BaseBucket):
 
@@ -834,16 +881,16 @@ class AWSBucket(BaseBucket):
         """
         return self._bucket.name
 
-    def get(self, key):
+    def get(self, name):
         """
         Retrieve a given object from this bucket.
         """
-        key = Key(self._bucket, key)
-        if key.exists():
+        key = Key(self._bucket, name)
+        if key and key.exists():
             return AWSBucketObject(self._provider, key)
         return None
 
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, prefix=None):
         """
         List all objects within this bucket.
 
@@ -851,7 +898,8 @@ class AWSBucket(BaseBucket):
         :return: List of all available BucketObjects within this bucket.
         """
         objects = [AWSBucketObject(self._provider, obj)
-                   for obj in self._bucket.list()]
+                   for obj in self._bucket.list(prefix=prefix)]
+
         return ClientPagedResultList(self._provider, objects,
                                      limit=limit, marker=marker)
 
@@ -959,8 +1007,9 @@ class AWSNetwork(BaseNetwork):
         subnets = self._provider.vpc_conn.get_all_subnets(filters=flter)
         return [AWSSubnet(self._provider, subnet) for subnet in subnets]
 
-    def create_subnet(self, cidr_block, name=None):
-        subnet = self._provider.vpc_conn.create_subnet(self.id, cidr_block)
+    def create_subnet(self, cidr_block, name=None, zone=None):
+        subnet = self._provider.vpc_conn.create_subnet(self.id, cidr_block,
+                                                       availability_zone=zone)
         cb_subnet = AWSSubnet(self._provider, subnet)
         if name:
             cb_subnet.name = name
@@ -1009,6 +1058,11 @@ class AWSSubnet(BaseSubnet):
     def network_id(self):
         return self._subnet.vpc_id
 
+    @property
+    def zone(self):
+        return AWSPlacementZone(self._provider, self._subnet.availability_zone,
+                                self._provider.region_name)
+
     def delete(self):
         return self._provider.vpc_conn.delete_subnet(subnet_id=self.id)
 

cloudbridge/cloud/providers/aws/services.py

@@ -1,10 +1,10 @@
 """Services implemented by the AWS provider."""
-import time
 import string
+import time
 
 from boto.ec2.blockdevicemapping import BlockDeviceMapping
 from boto.ec2.blockdevicemapping import BlockDeviceType
-from boto.exception import EC2ResponseError
+from boto.exception import EC2ResponseError, S3ResponseError
 
 from cloudbridge.cloud.base.resources import ClientPagedResultList
 from cloudbridge.cloud.base.resources import ServerPagedResultList
@@ -22,12 +22,11 @@ from cloudbridge.cloud.base.services import BaseSecurityService
 from cloudbridge.cloud.base.services import BaseSnapshotService
 from cloudbridge.cloud.base.services import BaseSubnetService
 from cloudbridge.cloud.base.services import BaseVolumeService
-from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
+from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
-from cloudbridge.cloud.interfaces.resources import Network
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
@@ -50,8 +49,8 @@ from .resources import AWSSnapshot
 from .resources import AWSSubnet
 from .resources import AWSVolume
 
-import cloudbridge as cb
 # Uncomment to enable logging by default for this module
+# import cloudbridge as cb
 # cb.set_stream_logger(__name__)
 
 
@@ -363,11 +362,26 @@ class AWSObjectStoreService(BaseObjectStoreService):
         Returns a bucket given its ID. Returns ``None`` if the bucket
         does not exist.
         """
-        bucket = self.provider.s3_conn.lookup(bucket_id)
-        if bucket:
+        try:
+            # Make a call to make sure the bucket exists. While this would
+            # normally return a Bucket instance, there's an edge case where a
+            # 403 response can occur when the bucket exists but the
+            # user simply does not have permissions to access it. See below.
+            bucket = self.provider.s3_conn.get_bucket(bucket_id)
             return AWSBucket(self.provider, bucket)
-        else:
-            return None
+        except S3ResponseError as e:
+            # If 403, it means the bucket exists, but the user does not have
+            # permissions to access the bucket. However, limited operations
+            # may be permitted (with a session token for example), so return a
+            # Bucket instance to allow further operations.
+            # http://stackoverflow.com/questions/32331456/using-boto-upload-file-to-s3-
+            # sub-folder-when-i-have-no-permissions-on-listing-fo
+            if e.status == 403:
+                bucket = self.provider.s3_conn.get_bucket(bucket_id,
+                                                          validate=False)
+                return AWSBucket(self.provider, bucket)
+        # For all other responses, it's assumed that the bucket does not exist.
+        return None
 
     def find(self, name, limit=None, marker=None):
         """
@@ -467,13 +481,14 @@ class AWSInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(AWSInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None, **kwargs):
         image_id = image.id if isinstance(image, MachineImage) else image
         instance_size = instance_type.id if \
             isinstance(instance_type, InstanceType) else instance_type
-        network_id = network.id if isinstance(network, Network) else network
+        subnet = (self.provider.network.subnets.get(subnet)
+                  if isinstance(subnet, str) else subnet)
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         key_pair_name = key_pair.name if isinstance(
             key_pair,
@@ -484,7 +499,7 @@ class AWSInstanceService(BaseInstanceService):
             bdm = None
 
         subnet_id, zone_id, security_group_ids = \
-            self._resolve_launch_options(zone_id, network_id, security_groups)
+            self._resolve_launch_options(subnet, zone_id, security_groups)
 
         reservation = self.provider.ec2_conn.run_instances(
             image_id=image_id, instance_type=instance_size,
@@ -498,202 +513,37 @@ class AWSInstanceService(BaseInstanceService):
             instance.name = name
         return instance
 
-    def _resolve_launch_options(self, zone_id=None, vpc_id=None,
+    def _resolve_launch_options(self, subnet=None, zone_id=None,
                                 security_groups=None):
         """
         Work out interdependent launch options.
 
-        Some launch options are required and interdependent so work through
-        those constraints. There are 8 subsets of options so the logic works
-        through each of those combinations to figure out the proper launch
-        options.
+        Some launch options are required and interdependent so make sure
+        they conform to the interface contract.
+
+        :type subnet: ``Subnet``
+        :param subnet: Subnet object within which to launch.
 
         :type zone_id: ``str``
         :param zone_id: ID of the zone where the launch should happen.
 
-        :type vpc_id: ``str``
-        :param vpc_id: ID of the network within which to launch.
-
-        :type security_groups: ``list`` of ``str``
-        :param zone_id: List of security group names.
+        :type security_groups: ``list`` of ``id``
+        :param zone_id: List of security group IDs.
 
         :rtype: triplet of ``str``
         :return: Subnet ID, zone ID and security group IDs for launch.
 
-        :raise ValueError: In case a conflicting combination is found or the
-                           method cannot infer the defaults, raise.
-        """
-        def _get_default_vpc(vpcs, exc="No default network found."):
-            """
-            Inspect supplied VPCs to figure out a default one or create one.
-
-            Default VPC either has ``is_default`` property set or matches the
-            default network name used by this library. If a default network
-            is not found, an attempt to create one is made.
-
-            :type vpcs: ``list``
-            :param vpcs: A list of boto VPC objects.
-
-            :type exc: ``str``
-            :type exc: A string value to use if/when raising ValueError.
-
-            :rtype: ``str``
-            :return: Default VPC ID.
-            """
-            default_vpc = None
-            for vpc in vpcs:
-                if vpc.is_default:
-                    default_vpc = vpc.id
-                    break
-            if not default_vpc:
-                for vpc in vpcs:
-                    if vpc.tags.get('Name', '') == \
-                       AWSNetwork.CB_DEFAULT_NETWORK_NAME:
-                        default_vpc = vpc.id
-                        break
-            if not default_vpc:
-                net = self.provider.network.create(
-                    name=AWSNetwork.CB_DEFAULT_NETWORK_NAME)
-                default_vpc = net.id
-            return default_vpc
-
-        def _get_potential_subnets(filters, exc):
-            """
-            Query existing subnets through supplied filters.
-
-            :type filters: ``dict``
-            :param filters: A dictionary supplying desired filters.
-
-            :type exc: ``str``
-            :type exc: A string value to use if/when raising ValueError.
-
-            :rtype: tuple of ``str``
-            :return: A tuple with a random subnet that matches supplied
-                     filters and an availability zone where the given subnet
-                     is defined.
-
-            :raise ValueError: If no subnet can be found, raise a ValueError.
-            """
-            potential_subnets = self.provider.vpc_conn.get_all_subnets(
-                filters=filters)
-            if potential_subnets and len(potential_subnets) > 0:
-                sn_id = potential_subnets[0].id
-                zone_id = potential_subnets[0].availability_zone
-            else:
-                raise ValueError(exc)
-            return sn_id, zone_id
-
-        def _get_security_groups(security_groups, vpc_id=None, obj=False):
-            """
-            Resolve exact security groups to use.
-
-            :type security_groups: A ``list`` of ``SecurityGroup`` objects or
-                                   a list of ``str`` names.
-            :param security_groups: A list of ``SecurityGroup`` objects or a
-                                    list of ``SecurityGroup`` names, which
-                                    should be resolved.
-
-            :type vpc_id: ``str``
-            :param vpc_id: ID of the network within which to launch.
-
-            :type obj: ``bool``
-            :param obj: If True, return provider-native security group objects.
-                        Otherwise, return the IDs.
-
-            :rtype: list
-            :return: provider-native security group objects or the IDs (see
-                    ``obj`` param).
-            """
-            if isinstance(security_groups, list) and \
-               isinstance(security_groups[0], SecurityGroup):
-                return [sg._security_group if obj else sg.id
-                        for sg in security_groups]
-            else:
-                flters = {'group_name': security_groups}
-                if vpc_id:
-                    flters['vpc_id'] = vpc_id
-                sgs = self.provider.ec2_conn.get_all_security_groups(
-                    filters=flters)
-                return list(set([sg if obj else sg.id for sg in sgs]))
-
-        if zone_id and vpc_id and security_groups:
-            exc = "No subnets found in zone {0} for network {1}.".format(
-                zone_id, vpc_id)
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpcId': vpc_id}
-            sn_id, _ = _get_potential_subnets(flters, exc)
-            sg_ids = _get_security_groups(security_groups, vpc_id)
-        elif vpc_id and security_groups:
-            sg_ids = _get_security_groups(security_groups, vpc_id)
-            exc = "No subnets found in network {0}.".format(vpc_id)
-            flters = {'state': 'available', 'vpcId': vpc_id}
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
-        elif vpc_id and zone_id:
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpcId': vpc_id}
-            exc = "No subnets found in zone {0} for network {1}.".format(
-                zone_id, vpc_id)
-            sn_id, _ = _get_potential_subnets(flters, exc)
-            sg_ids = None
-        elif zone_id and security_groups:
-            sgs = _get_security_groups(security_groups, obj=True)
-            # Get VPCs the supplied SGs belong to
-            vpc_ids = list(set([sg.vpc_id for sg in sgs if sg.vpc_id]))
-            vpcs = []
-            if vpc_ids:
-                vpcs = self.provider.vpc_conn.get_all_vpcs(vpc_ids=vpc_ids)
-            exc = ("No default network found for zone {0} and security groups "
-                   "{1}".format(zone_id, security_groups))
-            default_vpc = _get_default_vpc(vpcs, exc)
-            # Filter only the SGs within the default VPC
-            sg_ids = _get_security_groups(security_groups, default_vpc)
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpc_id': default_vpc}
-            exc = "No subnets found in zone {0} for default network {1}." \
-                .format(zone_id, default_vpc)
-            sn_id, _ = _get_potential_subnets(flters, exc)
-        elif vpc_id:
-            flters = {'state': 'available', 'vpcId': vpc_id}
-            exc = "No subnets found for network {0}.".format(vpc_id)
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
-            sg_ids = None
-        elif zone_id:
-            vpcs = self.provider.vpc_conn.get_all_vpcs()
-            exc = "No default network exists for security zone {0}.".format(
-                zone_id)
-            default_vpc = _get_default_vpc(vpcs, exc)
-            flters = {'availabilityZone': zone_id, 'state': 'available',
-                      'vpcId': default_vpc}
-            exc = "No subnets found in zone {0} for default network {1}." \
-                .format(zone_id, default_vpc)
-            sn_id, _ = _get_potential_subnets(flters, exc)
-            sg_ids = None
-        elif security_groups:
-            sgs = _get_security_groups(security_groups, obj=True)
-            # Get VPCs the supplied SGs belong to
-            vpc_ids = list(set([sg.vpc_id for sg in sgs if sg.vpc_id]))
-            vpcs = []
-            if vpc_ids:
-                vpcs = self.provider.vpc_conn.get_all_vpcs(vpc_ids=vpc_ids)
-            exc = "No default network exists for security groups {0}.".format(
-                security_groups)
-            default_vpc = _get_default_vpc(vpcs, exc)
-            # Filter only the SGs within the default VPC
-            sg_ids = _get_security_groups(security_groups, default_vpc)
-            flters = {'state': 'available', 'vpcId': default_vpc}
-            exc = "No subnets found in network {0}.".format(default_vpc)
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
+        :raise ValueError: In case a conflicting combination is found.
+        """
+        if subnet:
+            # subnet's zone takes precedence
+            zone_id = subnet.zone.id
+        if isinstance(security_groups, list) and isinstance(
+                security_groups[0], SecurityGroup):
+            security_group_ids = [sg.id for sg in security_groups]
         else:
-            # Nothing was defined, use all defaults
-            vpcs = self.provider.vpc_conn.get_all_vpcs()
-            default_vpc = _get_default_vpc(vpcs)
-            flters = {'state': 'available', 'vpcId': default_vpc}
-            exc = "No subnets found for default network {1}.".format(
-                default_vpc)
-            sn_id, zone_id = _get_potential_subnets(flters, exc)
-            sg_ids = None
-
-        return sn_id, zone_id, sg_ids
+            security_group_ids = security_groups
+        return subnet.id, zone_id, security_group_ids
 
     def _process_block_device_mappings(self, launch_config, zone=None):
         """
@@ -794,9 +644,6 @@ class AWSInstanceService(BaseInstanceService):
                                      reservations.next_token,
                                      False, data=instances)
 
-AWS_INSTANCE_DATA_DEFAULT_URL = "https://d168wakzal7fp0.cloudfront.net/" \
-                                "aws_instance_data.json"
-
 
 class AWSInstanceTypesService(BaseInstanceTypesService):
 
@@ -809,8 +656,8 @@ class AWSInstanceTypesService(BaseInstanceTypesService):
         Fetch info about the available instances.
 
         To update this information, update the file pointed to by the
-        ``AWS_INSTANCE_DATA_DEFAULT_URL`` above. The content for this file
-        should be obtained from this repo
+        ``provider.AWS_INSTANCE_DATA_DEFAULT_URL`` above. The content for this
+        file should be obtained from this repo:
         https://github.com/powdahound/ec2instances.info (in particular, this
         file: https://raw.githubusercontent.com/powdahound/ec2instances.info/
         master/www/instances.json).
@@ -818,7 +665,8 @@ class AWSInstanceTypesService(BaseInstanceTypesService):
         TODO: Needs a caching function with timeout
         """
         r = requests.get(self.provider.config.get(
-            "aws_instance_info_url", AWS_INSTANCE_DATA_DEFAULT_URL))
+            "aws_instance_info_url",
+            self.provider.AWS_INSTANCE_DATA_DEFAULT_URL))
         return r.json()
 
     def list(self, limit=None, marker=None):
@@ -929,15 +777,43 @@ class AWSSubnetService(BaseSubnetService):
         subnets = self.provider.vpc_conn.get_all_subnets(filters=fltr)
         return [AWSSubnet(self.provider, subnet) for subnet in subnets]
 
-    def create(self, network, cidr_block, name=None):
+    def create(self, network, cidr_block, name=None, zone=None):
         network_id = network.id if isinstance(network, AWSNetwork) else network
-        subnet = self.provider.vpc_conn.create_subnet(network_id, cidr_block)
+        subnet = self.provider.vpc_conn.create_subnet(network_id, cidr_block,
+                                                      availability_zone=zone)
         cb_subnet = AWSSubnet(self.provider, subnet)
         if name:
             time.sleep(2)  # The subnet does not always get created in time
             cb_subnet.name = name
         return cb_subnet
 
+    def get_or_create_default(self, zone=None):
+        filtr = {'availabilityZone': zone} if zone else None
+        sns = self.provider.vpc_conn.get_all_subnets(filters=filtr)
+        for sn in sns:
+            if sn.defaultForAz:
+                return AWSSubnet(self.provider, sn)
+        # No provider-default Subnet exists, look for a library-default one
+        for sn in sns:
+            if sn.tags.get('Name') == AWSSubnet.CB_DEFAULT_SUBNET_NAME:
+                return AWSSubnet(self.provider, sn)
+        # No provider-default Subnet exists, try to create it (net + subnets)
+        default_net = self.provider.network.create(
+            name=AWSNetwork.CB_DEFAULT_NETWORK_NAME)
+        # Create a subnet in each of the region's zones
+        region = self.provider.compute.regions.get(
+            self.provider.vpc_conn.region.name)
+        default_sn = None
+        for i, z in enumerate(region.zones):
+            sn = self.create(default_net, '10.0.{0}.0/24'.format(i),
+                             AWSSubnet.CB_DEFAULT_SUBNET_NAME, z.name)
+            if zone and zone == z.name:
+                default_sn = sn
+        # No specific zone was supplied; return the last created subnet
+        if not default_sn:
+            default_sn = sn
+        return default_sn
+
     def delete(self, subnet):
         subnet_id = subnet.id if isinstance(subnet, AWSSubnet) else subnet
         return self.provider.vpc_conn.delete_subnet(subnet_id=subnet_id)

cloudbridge/cloud/providers/gce/helpers.py

@@ -1,9 +1,19 @@
-import os
-from Crypto.PublicKey import RSA
+# based on http://stackoverflow.com/a/39126754
+from cryptography.hazmat.primitives import serialization as crypt_serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.backends import default_backend
 
 
 def generate_key_pair():
-    kp = RSA.generate(2048, os.urandom)
-    public_key = kp.publickey().exportKey("OpenSSH").split(" ")[1]
-    private_key = kp.exportKey("PEM")
+    key_pair = rsa.generate_private_key(
+        backend=default_backend(),
+        public_exponent=65537,
+        key_size=2048)
+    private_key = key_pair.private_bytes(
+        crypt_serialization.Encoding.PEM,
+        crypt_serialization.PrivateFormat.PKCS8,
+        crypt_serialization.NoEncryption())
+    public_key = key_pair.public_key().public_bytes(
+        crypt_serialization.Encoding.OpenSSH,
+        crypt_serialization.PublicFormat.OpenSSH)
     return private_key, public_key

cloudbridge/cloud/providers/gce/provider.py

@@ -35,7 +35,7 @@ class GCPResourceUrl(object):
 
 
 class GCPResources(object):
-    
+
     def __init__(self, connection):
         self._connection = connection
 
@@ -112,7 +112,8 @@ class GCPResources(object):
         GCP servers.
         """
         url = url.strip()
-        if url.startswith(self._root_url): url = url[len(self._root_url):]
+        if url.startswith(self._root_url):
+            url = url[len(self._root_url):]
         if url.startswith(self._service_path):
             url = url[len(self._service_path):]
 
@@ -213,6 +214,7 @@ class GCECloudProvider(BaseCloudProvider):
         """
         http = httplib2.Http()
         http = self._credentials.authorize(http)
+
         def _postproc(*kwargs):
             if len(kwargs) >= 2:
                 # The first argument is request, and the second is response.

cloudbridge/cloud/providers/gce/services.py

@@ -119,7 +119,7 @@ class GCEKeyPairService(BaseKeyPairService):
             # elems should be "ssh-rsa <public_key> <email>"
             elems = key.split(" ")
             if elems and elems[0]:  # ignore blank lines
-                yield GCEKeyPairService.GCEKeyInfo(elems[0], elems[1],
+                yield GCEKeyPairService.GCEKeyInfo(elems[0], elems[1].encode('ascii'),
                                                    elems[2])
 
     def gce_metadata_save_op(self, callback):
@@ -432,7 +432,7 @@ class GCEInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(GCEInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None, **kwargs):
         """
@@ -441,9 +441,10 @@ class GCEInstanceService(BaseInstanceService):
         if not zone:
             zone = self.provider.default_zone
         if not launch_config:
-            if network:
+            if subnet:
+                network = self.provider.network.get(subnet.network_id)
                 network_url = (network.resource_url
-                               if isinstance(network, Network) else network)
+                               if isinstance(network, GCENetwork) else network)
             else:
                 network_url = 'global/networks/default'
             config = {
@@ -454,12 +455,14 @@ class GCEInstanceService(BaseInstanceService):
                            'initializeParams': {
                                'sourceImage': image.resource_url,
                            }
-                       }],
+                           }],
                 'networkInterfaces': [
-                    {'network': network_url,
-                     'accessConfigs': [{'type': 'ONE_TO_ONE_NAT',
-                                        'name': 'External NAT'}]
-                 }],
+                    {
+                        # TODO: Should replace network below with subnetwork
+                        'network': network_url,
+                        'accessConfigs': [{'type': 'ONE_TO_ONE_NAT',
+                                           'name': 'External NAT'}]
+                    }],
             }
             if security_groups and isinstance(security_groups, list):
                 sg_names = []

cloudbridge/cloud/providers/openstack/helpers.py

@@ -2,6 +2,7 @@
 Helper functions
 """
 import itertools
+
 from cloudbridge.cloud.base.resources import ServerPagedResultList
 
 

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,16 +1,23 @@
 """Provider implementation based on OpenStack Python clients for OpenStack."""
 
+import inspect
+
 import os
 
 from cinderclient import client as cinder_client
+
+from cloudbridge.cloud.base import BaseCloudProvider
+
 from keystoneauth1 import session
+
 from keystoneclient import client as keystone_client
+
 from neutronclient.v2_0 import client as neutron_client
+
 from novaclient import client as nova_client
 from novaclient import shell as nova_shell
-from swiftclient import client as swift_client
 
-from cloudbridge.cloud.base import BaseCloudProvider
+from swiftclient import client as swift_client
 
 from .services import OpenStackBlockStoreService
 from .services import OpenStackComputeService
@@ -45,9 +52,6 @@ class OpenStackCloudProvider(BaseCloudProvider):
             os.environ.get('OS_PROJECT_DOMAIN_NAME', None))
         self.user_domain_name = self._get_config_value(
             'os_user_domain_name', os.environ.get('OS_USER_DOMAIN_NAME', None))
-        self.identity_api_version = self._get_config_value(
-            'os_identity_api_version',
-            os.environ.get('OS_IDENTITY_API_VERSION', None))
 
         # Service connections, lazily initialized
         self._nova = None
@@ -234,10 +238,66 @@ class OpenStackCloudProvider(BaseCloudProvider):
 #         return glance_client.Client(version=api_version,
 #                                     session=self.keystone.session)
 
-    def _connect_swift(self):
-        """Get an OpenStack Swift (object store) client object cloud."""
-        return swift_client.Connection(authurl=self.auth_url,
-                                       session=self._keystone_session)
+    @staticmethod
+    def _clean_options(options, method_to_match):
+        """
+        Returns a **copy** of the source options with all keys that are not in
+        the ``method_to_match`` parameter list removed.
+
+        .. note:: If ``options`` has the ``os_options`` key it will have
+            both the key and its value removed. This is because any entries
+            in this dictionary value will override our settings. This
+            situation is only going to happen when the `_connect_swift`
+            method is called by the SwiftService to manufacture new
+            connections.
+
+        .. seealso::
+            https://docs.openstack.org/developer/python-swiftclient/swiftclient.html#module-swiftclient.client
+
+        :param options: The source options.
+        :type options: ``dict``
+        :param method_to_match: The method whose signature is to be matched
+        :type method_to_match: A callable
+        :return: A copy of the source options with all keys that are not in the
+            ``method_to_match`` parameter list removed. If options is ``None``
+            then this will be an empty dictionary
+        :rtype: ``dict``
+        """
+        result = {}
+        if options:
+            try:
+                method_signature = inspect.signature(method_to_match)
+                parameters = set(method_signature.parameters.keys())
+            except AttributeError:
+                parameters = set(inspect.getargspec(method_to_match).args)
+            result = {key: val for key, val in options.items() if
+                      key in parameters}
+            # Don't allow the options to override our authentication
+            result.pop('os_options', None)
+        return result
+
+    def _connect_swift(self, options=None):
+        """
+        Get an OpenStack Swift (object store) client connection.
+
+        :param options: A dictionary of options from which values will be
+            passed to the connection.
+        :return: A Swift client connection using the auth credentials held by
+            the OpenStackCloudProvider instance
+        """
+        clean_options = self._clean_options(options,
+                                            swift_client.Connection.__init__)
+        storage_url = self._get_config_value(
+            'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
+        auth_token = self._get_config_value(
+            'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
+        if storage_url and auth_token:
+            clean_options['preauthurl'] = storage_url
+            clean_options['preauthtoken'] = auth_token
+        else:
+            clean_options['authurl'] = self.auth_url
+            clean_options['session'] = self._keystone_session
+        return swift_client.Connection(**clean_options)
 
     def _connect_neutron(self):
         """Get an OpenStack Neutron (networking) client object cloud."""

cloudbridge/cloud/providers/openstack/resources.py

@@ -1,9 +1,16 @@
 """
 DataTypes used by this provider
 """
+import inspect
+import ipaddress
+import json
+
+import os
+
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
 from cloudbridge.cloud.base.resources import BaseBucket
 from cloudbridge.cloud.base.resources import BaseBucketObject
+from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseInstance
 from cloudbridge.cloud.base.resources import BaseInstanceType
 from cloudbridge.cloud.base.resources import BaseKeyPair
@@ -16,24 +23,26 @@ from cloudbridge.cloud.base.resources import BaseSecurityGroup
 from cloudbridge.cloud.base.resources import BaseSecurityGroupRule
 from cloudbridge.cloud.base.resources import BaseSnapshot
 from cloudbridge.cloud.base.resources import BaseSubnet
-from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseVolume
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import MachineImageState
 from cloudbridge.cloud.interfaces.resources import NetworkState
 from cloudbridge.cloud.interfaces.resources import RouterState
+from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
-from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.providers.openstack import helpers as oshelpers
-import inspect
-import json
-
-import ipaddress
 
 from keystoneclient.v3.regions import Region
+
 import novaclient.exceptions as novaex
-import swiftclient.exceptions as swiftex
+
+import swiftclient
+
+from swiftclient.service import SwiftService, SwiftUploadObject
+
+ONE_GIG = 1048576000  # in bytes
+FIVE_GIG = ONE_GIG * 5  # in bytes
 
 
 class OpenStackMachineImage(BaseMachineImage):
@@ -77,6 +86,17 @@ class OpenStackMachineImage(BaseMachineImage):
         """
         return None
 
+    @property
+    def min_disk(self):
+        """
+        Returns the minimum size of the disk that's required to
+        boot this image (in GB)
+
+        :rtype: ``int``
+        :return: The minimum disk size needed by this image
+        """
+        return self._os_image.minDisk
+
     def delete(self):
         """
         Delete this image
@@ -360,6 +380,18 @@ class OpenStackInstance(BaseInstance):
         """
         self._os_instance.remove_floating_ip(ip_address)
 
+    def add_security_group(self, sg):
+        """
+        Add a security group to this instance
+        """
+        self._os_instance.add_security_group(sg.id)
+
+    def remove_security_group(self, sg):
+        """
+        Remove a security group from this instance
+        """
+        self._os_instance.remove_security_group(sg.id)
+
     @property
     def state(self):
         return OpenStackInstance.INSTANCE_STATE_MAP.get(
@@ -399,8 +431,8 @@ class OpenStackRegion(BaseRegion):
 
     @property
     def zones(self):
-        # detailed must be set to ``False`` because the (default) ``True``
-        # value requires Admin privileges
+        # ``detailed`` param must be set to ``False`` because the (default)
+        # ``True`` value requires Admin privileges
         if self.name == self._provider.region_name:  # optimisation
             zones = self._provider.nova.availability_zones.list(detailed=False)
         else:
@@ -412,8 +444,7 @@ class OpenStackRegion(BaseRegion):
                 # return an empty list
                 zones = []
 
-        return [OpenStackPlacementZone(self._provider, z.zoneName,
-                                       self._os_region)
+        return [OpenStackPlacementZone(self._provider, z.zoneName, self.name)
                 for z in zones]
 
 
@@ -670,6 +701,7 @@ class OpenStackNetwork(BaseNetwork):
 
     @property
     def state(self):
+        self.refresh()
         return OpenStackNetwork._NETWORK_STATE_MAP.get(
             self._network.get('status', None),
             NetworkState.UNKNOWN)
@@ -691,7 +723,8 @@ class OpenStackNetwork(BaseNetwork):
                    .get('subnets', []))
         return [OpenStackSubnet(self._provider, subnet) for subnet in subnets]
 
-    def create_subnet(self, cidr_block, name=''):
+    def create_subnet(self, cidr_block, name='', zone=None):
+        """OpenStack has no support for subnet zones so the value is ignored"""
         subnet_info = {'name': name, 'network_id': self.id,
                        'cidr': cidr_block, 'ip_version': 4}
         subnet = (self._provider.neutron.create_subnet({'subnet': subnet_info})
@@ -699,11 +732,9 @@ class OpenStackNetwork(BaseNetwork):
         return OpenStackSubnet(self._provider, subnet)
 
     def refresh(self):
-        """
-        Refreshes the state of this network by re-querying the cloud provider
-        for its latest state.
-        """
-        return self.state
+        """Refresh the state of this network by re-querying the provider."""
+        net = self._provider.neutron.list_networks(id=self.id).get('networks')
+        self._network = net[0] if net else {}
 
 
 class OpenStackSubnet(BaseSubnet):
@@ -728,6 +759,15 @@ class OpenStackSubnet(BaseSubnet):
     def network_id(self):
         return self._subnet.get('network_id', None)
 
+    @property
+    def zone(self):
+        """
+        OpenStack does not have a notion of placement zone for subnets.
+
+        Default to None.
+        """
+        return None
+
     def delete(self):
         if self.id in str(self._provider.neutron.list_subnets()):
             self._provider.neutron.delete_subnet(self.id)
@@ -755,7 +795,7 @@ class OpenStackFloatingIP(BaseFloatingIP):
         return self._ip.get('fixed_ip_address', None)
 
     def in_use(self):
-        return True if self._ip.get('status', None) == 'ACTIVE' else False
+        return bool(self._ip.get('port_id', None))
 
     def delete(self):
         self._provider.neutron.delete_floatingip(self.id)
@@ -875,8 +915,8 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
         """
         Create a security group rule.
 
-        You need to pass in either ``src_group`` OR ``ip_protocol``,
-        ``from_port``, ``to_port``, and ``cidr_ip``.  In other words, either
+        You need to pass in either ``src_group`` OR ``ip_protocol`` AND
+        ``from_port``, ``to_port``, ``cidr_ip``.  In other words, either
         you are authorizing another group or you are authorizing some
         ip-based rule.
 
@@ -902,20 +942,19 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
             if not isinstance(src_group, SecurityGroup):
                 src_group = self._provider.security.security_groups.get(
                     src_group)
-            for protocol in ['udp', 'tcp']:
-                existing_rule = self.get_rule(ip_protocol=ip_protocol,
-                                              from_port=1,
-                                              to_port=65535,
-                                              src_group=src_group)
-                if existing_rule:
-                    return existing_rule
-
-                rule = self._provider.nova.security_group_rules.create(
-                    parent_group_id=self._security_group.id,
-                    ip_protocol=protocol,
-                    from_port=1,
-                    to_port=65535,
-                    group_id=src_group.id)
+            existing_rule = self.get_rule(ip_protocol=ip_protocol,
+                                          from_port=from_port,
+                                          to_port=to_port,
+                                          src_group=src_group)
+            if existing_rule:
+                return existing_rule
+
+            rule = self._provider.nova.security_group_rules.create(
+                parent_group_id=self._security_group.id,
+                ip_protocol=ip_protocol,
+                from_port=from_port,
+                to_port=to_port,
+                group_id=src_group.id)
             if rule:
                 # We can only return one Rule so default to TCP (ie, last in
                 # the for loop above).
@@ -942,16 +981,16 @@ class OpenStackSecurityGroup(BaseSecurityGroup):
 
     def get_rule(self, ip_protocol=None, from_port=None, to_port=None,
                  cidr_ip=None, src_group=None):
-        # Update SG object; otherwise, recently added rules do now show
+        # Update SG object; otherwise, recently added rules do not show
         self._security_group = self._provider.nova.security_groups.get(
             self._security_group)
         for rule in self._security_group.rules:
             if (rule['ip_protocol'] == ip_protocol and
                 rule['from_port'] == from_port and
                 rule['to_port'] == to_port and
-                rule['ip_range'].get('cidr') == cidr_ip) or \
-               (rule['group'].get('name') == src_group.name if src_group
-                    else False):
+                (rule['ip_range'].get('cidr') == cidr_ip or
+                 (rule['group'].get('name') == src_group.name if src_group
+                  else False))):
                 return OpenStackSecurityGroupRule(self._provider, rule, self)
         return None
 
@@ -1023,9 +1062,7 @@ class OpenStackBucketObject(BaseBucketObject):
 
     @property
     def name(self):
-        """
-        Get this object's name.
-        """
+        """Get this object's name."""
         return self._obj.get("name")
 
     @property
@@ -1037,10 +1074,7 @@ class OpenStackBucketObject(BaseBucketObject):
         return self._obj.get("last_modified")
 
     def iter_content(self):
-        """
-        Returns this object's content as an
-        iterable.
-        """
+        """Returns this object's content as an iterable."""
         _, content = self._provider.swift.get_object(
             self.cbcontainer.name, self.name, resp_chunk_size=65536)
         return content
@@ -1049,24 +1083,81 @@ class OpenStackBucketObject(BaseBucketObject):
         """
         Set the contents of this object to the data read from the source
         string.
+
+        .. warning:: Will fail if the data is larger than 5 Gig.
         """
         self._provider.swift.put_object(self.cbcontainer.name, self.name,
                                         data)
 
+    def upload_from_file(self, path):
+        """
+        Stores the contents of the file pointed by the ``path`` variable.
+        If the file is bigger than 5 Gig, it will be broken into segments.
+
+        :type path: ``str``
+        :param path: Absolute path to the file to be uploaded to Swift.
+        :rtype: ``bool``
+        :return: ``True`` if successful, ``False`` if not.
+
+        .. note::
+            * The size of the segments chosen (or any of the other upload
+              options) is not under user control.
+            * If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
+
+        .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
+        """
+        upload_options = {}
+        if 'segment_size' not in upload_options:
+            if os.path.getsize(path) >= FIVE_GIG:
+                upload_options['segment_size'] = FIVE_GIG
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            upload_object = SwiftUploadObject(path, object_name=self.name)
+            for up_res in swift.upload(self.cbcontainer.name,
+                                       [upload_object, ],
+                                       options=upload_options):
+                result = result and up_res['success']
+        return result
+
     def delete(self):
         """
         Delete this object.
 
-        :rtype: bool
+        :rtype: ``bool``
         :return: True if successful
+
+        .. note:: If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
         """
-        try:
-            self._provider.swift.delete_object(self.cbcontainer.name,
-                                               self.name)
-        except swiftex.ClientException as err:
-            if err.http_status == 404:
-                return True
-        return False
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            for del_res in swift.delete(self.cbcontainer.name, [self.name, ]):
+                result = result and del_res['success']
+        return result
+
+    def generate_url(self, expires_in=0):
+        """
+        Generates a URL to this object.
+
+        If the object is public, `expires_in` argument is not necessary, but if
+        the object is private, the life time of URL is set using `expires_in`
+        argument.
+
+        See here for implementation details:
+        http://stackoverflow.com/a/37057172
+        """
+        raise NotImplementedError("This functionality is not implemented yet.")
 
 
 class OpenStackBucket(BaseBucket):
@@ -1086,19 +1177,23 @@ class OpenStackBucket(BaseBucket):
         """
         return self._bucket.get("name")
 
-    def get(self, key):
+    def get(self, name):
         """
         Retrieve a given object from this bucket.
+
+        FIXME: If multiple objects match the name as their name prefix,
+        all will be returned by the provider but this method will only
+        return the first element.
         """
         _, object_list = self._provider.swift.get_container(
-            self.name, prefix=key)
+            self.name, prefix=name)
         if object_list:
             return OpenStackBucketObject(self._provider, self,
                                          object_list[0])
         else:
             return None
 
-    def list(self, limit=None, marker=None):
+    def list(self, limit=None, marker=None, prefix=None):
         """
         List all objects within this bucket.
 
@@ -1107,7 +1202,7 @@ class OpenStackBucket(BaseBucket):
         """
         _, object_list = self._provider.swift.get_container(
             self.name, limit=oshelpers.os_result_limit(self._provider, limit),
-            marker=marker)
+            marker=marker, prefix=prefix)
         cb_objects = [OpenStackBucketObject(
             self._provider, self, obj) for obj in object_list]
 

cloudbridge/cloud/providers/openstack/services.py

@@ -26,13 +26,15 @@ from cloudbridge.cloud.base.services import BaseVolumeService
 from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
-from cloudbridge.cloud.interfaces.resources import Network
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
+from cloudbridge.cloud.interfaces.resources import Subnet
 from cloudbridge.cloud.interfaces.resources import Volume
 from cloudbridge.cloud.providers.openstack import helpers as oshelpers
 
+from neutronclient.common.exceptions import NeutronClientException
+
 from novaclient.exceptions import NotFound as NovaNotFound
 
 from .resources import OpenStackBucket
@@ -81,10 +83,10 @@ class OpenStackSecurityService(BaseSecurityService):
         """
         return self._security_groups
 
-    def get_ec2_credentials(self):
+    def get_or_create_ec2_credentials(self):
         """
         A provider specific method than returns the ec2 credentials for the
-        current user.
+        current user, or creates a new pair if one doesn't exist.
         """
         keystone = self.provider.keystone
         if hasattr(keystone, 'ec2'):
@@ -92,6 +94,10 @@ class OpenStackSecurityService(BaseSecurityService):
                           if cred.tenant_id == keystone.tenant_id]
             if user_creds:
                 return user_creds[0]
+            else:
+                return keystone.ec2.create(keystone.user_id,
+                                           keystone.tenant_id)
+
         return None
 
     def get_ec2_endpoints(self):
@@ -558,7 +564,7 @@ class OpenStackInstanceService(BaseInstanceService):
     def __init__(self, provider):
         super(OpenStackInstanceService, self).__init__(provider)
 
-    def create(self, name, image, instance_type, network=None, zone=None,
+    def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
                launch_config=None,
                **kwargs):
@@ -568,7 +574,10 @@ class OpenStackInstanceService(BaseInstanceService):
             isinstance(instance_type, InstanceType) else \
             self.provider.compute.instance_types.find(
                 name=instance_type)[0].id
-        network_id = network.id if isinstance(network, Network) else network
+        network_id = subnet.network_id if isinstance(subnet, Subnet) else None
+        if not network_id and subnet:
+            network_id = (self.provider.network.subnets.get(subnet).network_id
+                          if isinstance(subnet, str) else None)
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         key_pair_name = key_pair.name if \
             isinstance(key_pair, KeyPair) else key_pair
@@ -583,8 +592,8 @@ class OpenStackInstanceService(BaseInstanceService):
         bdm = None
         if launch_config:
             bdm = self._to_block_device_mapping(launch_config)
-        net = self._get_network(network_id)
 
+        log.debug("Launching in network %s" % network_id)
         os_instance = self.provider.nova.servers.create(
             name,
             None if self._has_root_device(launch_config) else image_id,
@@ -596,7 +605,7 @@ class OpenStackInstanceService(BaseInstanceService):
             security_groups=security_groups_list,
             userdata=user_data,
             block_device_mapping_v2=bdm,
-            nics=net)
+            nics=[{'net-id': network_id}] if network_id else None)
         return OpenStackInstance(self.provider, os_instance)
 
     def _to_block_device_mapping(self, launch_config):
@@ -649,43 +658,6 @@ class OpenStackInstanceService(BaseInstanceService):
                 return True
         return False
 
-    def _get_network(self, network_id=None):
-        """
-        Format the network ID for the API call, figuring out a default network.
-
-        If a network_id is not supplied, figure out which is the default
-        network and use it. A default network is either marked as such by the
-        provider or matches the default network name defined within this
-        library (by default CloudBridgeNet). If a default network cannot be
-        found, attempt to create a new one.
-        """
-        if network_id:
-            return [{'net-id': network_id}]
-        for net in self.provider.network.list():
-            if net.name == OpenStackNetwork.CB_DEFAULT_NETWORK_NAME:
-                return [{'net-id': net.id}]
-        try:
-            # Try to create a complete, Internet-connected new network
-            net = self.provider.network.create(
-                name=OpenStackNetwork.CB_DEFAULT_NETWORK_NAME)
-            sn = net.create_subnet('10.0.0.0/16', '{0}Subnet'.format(
-                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME))
-            router = self.provider.network.create_router('{0}Router'.format(
-                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME))
-            for n in self.provider.network.list():
-                if n.external:
-                    external_net = n
-                    break
-            router.attach_network(external_net.id)
-            router.add_route(sn.id)
-            return [{'net-id': net.id}]
-        except Exception as exc:
-            # At this point we assume the provider does support user-defined
-            # networks so return None
-            log.warn("Exception occurred trying to create a default "
-                     "CloudBridge network: {0}".format(exc))
-            return None
-
     def create_launch_config(self):
         return BaseLaunchConfig(self.provider)
 
@@ -796,7 +768,8 @@ class OpenStackSubnetService(BaseSubnetService):
         subnets = self.provider.neutron.list_subnets().get('subnets', [])
         return [OpenStackSubnet(self.provider, subnet) for subnet in subnets]
 
-    def create(self, network, cidr_block, name=''):
+    def create(self, network, cidr_block, name='', zone=None):
+        """zone param is ignored."""
         network_id = (network.id if isinstance(network, OpenStackNetwork)
                       else network)
         subnet_info = {'name': name, 'network_id': network_id,
@@ -805,6 +778,31 @@ class OpenStackSubnetService(BaseSubnetService):
                   .get('subnet'))
         return OpenStackSubnet(self.provider, subnet)
 
+    def get_or_create_default(self, zone=None):
+        """
+        Subnet zone is not supported by OpenStack and is thus ignored.
+        """
+        try:
+            for sn in self.list():
+                if sn.name == OpenStackSubnet.CB_DEFAULT_SUBNET_NAME:
+                    return sn
+            # No default; create one
+            net = self.provider.network.create(
+                OpenStackNetwork.CB_DEFAULT_NETWORK_NAME)
+            sn = net.create_subnet(cidr_block='10.0.0.0/24',
+                                   name=OpenStackSubnet.CB_DEFAULT_SUBNET_NAME)
+            router = self.provider.network.create_router(
+                OpenStackRouter.CB_DEFAULT_ROUTER_NAME)
+            for n in self.provider.network.list():
+                if n.external:
+                    external_net = n
+                    break
+            router.attach_network(external_net.id)
+            router.add_route(sn.id)
+            return sn
+        except NeutronClientException:
+            return None
+
     def delete(self, subnet):
         subnet_id = (subnet.id if isinstance(subnet, OpenStackSubnet)
                      else subnet)

docs/api_docs/cloud/exceptions.rst

@@ -0,0 +1,19 @@
+Exceptions
+==========
+
+.. contents:: :local:
+
+CloudBridgeBaseException
+------------------------
+.. autoclass:: cloudbridge.cloud.interfaces.exceptions.CloudBridgeBaseException
+    :members:
+
+WaitStateException
+------------------
+.. autoclass:: cloudbridge.cloud.interfaces.exceptions.WaitStateException
+    :members:
+
+InvalidConfigurationException
+-----------------------------
+.. autoclass:: cloudbridge.cloud.interfaces.exceptions.InvalidConfigurationException
+    :members:

docs/api_docs/cloud/resources.rst

@@ -8,21 +8,6 @@ CloudServiceType
 .. autoclass:: cloudbridge.cloud.interfaces.resources.CloudServiceType
     :members:
 
-CloudBridgeBaseException
-------------------------
-.. autoclass:: cloudbridge.cloud.interfaces.resources.CloudBridgeBaseException
-    :members:
-
-WaitStateException
-------------------
-.. autoclass:: cloudbridge.cloud.interfaces.resources.WaitStateException
-    :members:
-
-InvalidConfigurationException
------------------------------
-.. autoclass:: cloudbridge.cloud.interfaces.resources.InvalidConfigurationException
-    :members:
-
 ObjectLifeCycleMixin
 --------------------
 .. autoclass:: cloudbridge.cloud.interfaces.resources.ObjectLifeCycleMixin

docs/api_docs/ref.rst

@@ -10,3 +10,4 @@ This section includes the API documentation for the reference interface.
    cloud/providers.rst
    cloud/services.rst
    cloud/resources.rst
+   cloud/exceptions.rst

docs/getting_started.rst

@@ -94,9 +94,13 @@ on disk as a read-only file.
 Create a security group
 -----------------------
 Next, we need to create a security group and add a rule to allow ssh access.
+A security group needs to be associated with a private network, so we'll also
+need to fetch it.
 
 .. code-block:: python
 
+    provider.network.list()  # Find a desired network ID
+    net = provider.network.get('desired network ID')
     sg = provider.security.security_groups.create(
         'cloudbridge_intro', 'A security group used by CloudBridge', net.id)
     sg.add_rule('tcp', 22, 22, '0.0.0.0/0')
@@ -115,7 +119,7 @@ also add the network interface as a launch argument.
                        key=lambda x: x.vcpus*x.ram)[0]
     inst = provider.compute.instances.create(
         name='CloudBridge-intro', image=img, instance_type=inst_type,
-        key_pair=kp, security_groups=[sg])
+        network=net, key_pair=kp, security_groups=[sg])
     # Wait until ready
     inst.wait_till_ready()  # This is a blocking call
     # Show instance state

docs/topics/contributor_guide.rst

@@ -9,5 +9,6 @@ CloudBridge Provider.
 
     Design Goals <design_goals.rst>
     Testing <testing.rst>
-    Provider development walkthrough <provider_development.rst>
+    Provider Development Walkthrough <provider_development.rst>
+    Release Process <release_process.rst>
 

docs/topics/launch.rst

@@ -14,7 +14,7 @@ and 4 GB RAM.
 
 .. code-block:: python
 
-    img = provider.compute.images.get('ami-5ac2cd4d')  # Ubuntu 14.04 on AWS
+    img = provider.compute.images.get('ami-f4cc1de2')  # Ubuntu 16.04 on AWS
     inst_type = sorted([t for t in provider.compute.instance_types.list()
                         if t.vcpus >= 2 and t.ram >= 4],
                        key=lambda x: x.vcpus*x.ram)[0]
@@ -45,16 +45,17 @@ Private networking
 ~~~~~~~~~~~~~~~~~~
 Private networking gives you control over the networking setup for your
 instance(s) and is considered the preferred method for launching instances. To
-launch an instance with an explicit private network, just supply it as an
-additional argument to the ``create`` method:
+launch an instance with an explicit private network, supply a subnet within
+a network as an additional argument to the ``create`` method:
 
 .. code-block:: python
 
     provider.network.list()  # Find a desired network ID
     net = provider.network.get('desired network ID')
+    sn = net.subnets()[0]  # Get a handle on the desired subnet to launch with
     inst = provider.compute.instances.create(
         name='CloudBridge-VPC', image=img, instance_type=inst_type,
-        network=net, key_pair=kp, security_groups=[sg])
+        subnet=sn, key_pair=kp, security_groups=[sg])
 
 For more information on how to create and setup a private network, take a look
 at `Networking <./networking.html>`_.

docs/topics/networking.rst

@@ -15,9 +15,9 @@ Create a new private network
 ----------------------------
 Creating a private network is a simple, one-line command but appropriately
 connecting it so it has Internet access is a multi-step process:
-(1) create a network; (2) create a subnet within the network; (3) create a
+(1) create a network; (2) create a subnet within this network; (3) create a
 router; (4) attach the router to an external network; and (5) add a route to
-the router that links with with a subnet. For some providers, any network can
+the router that links with a subnet. For some providers, any network can
 be external (ie, connected to the Internet) while for others it's a specific,
 pre-defined one that exists in the an account by default. In order to properly
 connect the router, we need to ensure we're using an external network.
@@ -35,9 +35,9 @@ the block and allow up to 16 IP addresses into the subnet (``/28``).
     if not net.external:
         for n in self.provider.network.list():
             if n.external:
-                external_net = n
+                net = n
                 break
-    router.attach_network(external_net.id)
+    router.attach_network(net.id)
     router.add_route(sn.id)
 
 Retrieve an existing private network

docs/topics/release_process.rst

@@ -0,0 +1,21 @@
+Release Process
+~~~~~~~~~~~~~~~
+
+1. Increment version number in cloudbridge/__init__.py as per semver rules.
+
+2. Freeze all library dependencies in setup.py. The version numbers can be a range
+   with the upper limit being the latest known working version, and the lowest being
+   the last known working version. 
+
+3. Run all tox tests.
+
+4. Add release notes to CHANGELOG.rst. Also add last commit hash to changelog.
+
+5. Release to PyPi
+
+.. code-block:: bash
+
+   python setup.py sdist upload
+   python setup.py bdist_wheel upload
+
+6. Tag release and make github release.

docs/topics/setup.rst

@@ -2,10 +2,11 @@ Setup
 -----
 To initialize a connection to a cloud and get a provider object, you will
 need to provide the cloud's access credentials to CloudBridge. These may
-be provided in one of two ways:
+be provided in one of following ways:
 
 1. Environment variables
 2. A dictionary
+3. Configuration file
 
 Providing access credentials through environment variables
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -28,8 +29,8 @@ Mandatory variables  Optional Variables
 OS_AUTH_URL			 NOVA_SERVICE_NAME
 OS_USERNAME			 OS_COMPUTE_API_VERSION
 OS_PASSWORD			 OS_VOLUME_API_VERSION
-OS_PROJECT_NAME
-OS_REGION_NAME
+OS_PROJECT_NAME      OS_STORAGE_URL
+OS_REGION_NAME       OS_AUTH_TOKEN
 ===================  ==================
 
 
@@ -74,6 +75,8 @@ default_result_limit  Number of results that a ``.list()`` method should return.
 ====================  ==================
 Variable		      Description
 ====================  ==================
+aws_session_token     Session key for your AWS account (if using temporary
+                      credentials).
 ec2_is_secure         True to use an SSL connection. Default is ``True``.
 ec2_region_name       Default region name. Defaults to ``us-east-1``.
 ec2_region_endpoint   Endpoint to use. Default is ``ec2.us-east-1.amazonaws.com``.
@@ -92,20 +95,51 @@ s3_validate_certs     Whether to use SSL certificate verification. Default is
 ====================  ==================
 
 
+Providing access credentials in a file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+CloudBridge can also read credentials from a file on your local file system.
+The file should be placed in one of two locations: ``/etc/cloudbridge.ini`` or
+``~/.cloudbridge``. Each set of credentials should be delineated with the
+provider ID (e.g., ``openstack``, ``aws``) with the necessary credentials
+being supplied in YAML format. Note that only one set of credentials per
+cloud provider type can be supplied (i.e., via this method, it is not possible
+to provide credentials for two different OpenStack clouds).
+
+.. code-block:: bash
+
+    [openstack]
+    os_username: username
+    os_password: password
+    os_auth_url: auth url
+    os_user_domain_name: user domain name
+    os_project_domain_name: project domain name
+    os_project_name: project name
+
+    [aws]
+    aws_access_key: access key
+    aws_secret_key: secret key
+
+
 Other configuration variables
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In addition to the provider specific configuration variables above, there are
 some general configuration environment variables that apply to CloudBridge as
 a whole
 
-=====================  ==================
-Variable		       Description
-=====================  ==================
-CB_DEBUG               Setting ``CB_DEBUG=True`` will cause detailed debug
-                       output to be printed for each provider (including HTTP
-                       traces).
-CB_USE_MOCK_PROVIDERS  Setting this to ``True`` will cause the CloudBridge test
-                       suite to use mock drivers when available.
-CB_TEST_PROVIDER       Set this value to a valid :class:`.ProviderList` value
-                       such as ``aws``, to limit tests to that provider only.
-=====================  ==================
+======================  ==================
+Variable		            Description
+======================  ==================
+CB_DEBUG                Setting ``CB_DEBUG=True`` will cause detailed debug
+                        output to be printed for each provider (including HTTP
+                        traces).
+CB_USE_MOCK_PROVIDERS   Setting this to ``True`` will cause the CloudBridge test
+                        suite to use mock drivers when available.
+CB_TEST_PROVIDER        Set this value to a valid :class:`.ProviderList` value
+                        such as ``aws``, to limit tests to that provider only.
+CB_DEFAULT_SUBNET_NAME  Name to be used for a subnet that will be considered
+                        the 'default' by the library. This default will be used
+                        only in cases there is no subnet marked as the default by the provider.
+CB_DEFAULT_NETWORK_NAME Name to be used for a network that will be considered
+                        the 'default' by the library. This default will be used
+                        only in cases there is no network marked as the default by the provider.
+======================= ==================

docs/topics/testing.rst

@@ -38,18 +38,31 @@ This will run all the tests for all the environments defined in file
 ``tox.ini``.
 
 
-Specific environment
-~~~~~~~~~~~~~~~~~~~~
+Specific environment and infrastructure
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you’d like to run the tests on a specific environment only, say Python 2.7,
-use a command like this: ``tox -e py27``. Alternativley, to use your default
-python, you can also run the test command directly ``python setup.py test``.
+against a specific infrastructure, say aws, use a command like this:
+``tox -e py27-aws``. The available provider names are listed in the
+`ProviderList`_ class (e.g., ``aws`` or ``openstack``).  
 
-Select infrastructure
-~~~~~~~~~~~~~~~~~~~~~
-You can also run the tests on a specific cloud only. To do so, export an
-environment variable ``CB_TEST_PROVIDER`` and specify the desired provider
-name. The available provider names are listed in the `ProviderList`_ class
-(e.g., ``aws`` or ``openstack``). Then, run the ``tox`` command.
+Specific test cases
+~~~~~~~~~~~~~~~~~~~~
+You can run a specific test case, as follows:
+``tox -- -s test.test_cloud_factory.CloudFactoryTestCase``
+
+It can also be restricted to a particular environment as follows:
+``tox -e "py27-aws" -- -s test.test_cloud_factory.CloudFactoryTestCase``
+
+Using unittest directly
+~~~~~~~~~~~~~~~~~~~~~~~
+You can also run the tests against your active virtual environment directly
+with ``python setup.py test``. You will need to set the ``CB_TEST_PROVIDER``
+and ``CB_USE_MOCK_PROVIDERS`` environment variables prior to running the tests,
+or they will default to ``CB_TEST_PROVIDER=aws`` and
+``CB_USE_MOCK_PROVIDERS=True``.
+
+You can also run a specific test case, as follows:
+``python setup.py test -s test.test_cloud_factory.CloudFactoryTestCase``
 
 Using a mock provider
 ~~~~~~~~~~~~~~~~~~~~~

setup.py

@@ -1,7 +1,9 @@
+"""Library install script for setuptools."""
 import ast
 import os
 import re
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 # Cannot use "from cloudbridge import get_version" because that would try to
 # import the six package which may not be installed yet.
@@ -14,16 +16,20 @@ with open(os.path.join('cloudbridge', '__init__.py')) as f:
             break
 
 base_reqs = ['bunch>=1.0.1', 'six>=1.10.0', 'retrying>=1.3.3']
-openstack_reqs = ['python-novaclient>=7.0.0',
-                  'python-glanceclient>=2.5.0',
-                  'python-cinderclient>=1.9.0',
-                  'python-swiftclient>=3.2.0',
-                  'python-neutronclient>=6.0.0',
-                  'python-keystoneclient>=3.8.0']
-aws_reqs = ['boto>=2.38.0']
-gce_reqs = ['google-api-python-client>=1.4.2', "pycrypto"]
+openstack_reqs = ['requests<2.13.0',
+                  'Babel>=2.3.4,<2.4.0',
+                  'python-novaclient==7.0.0',
+                  'python-glanceclient>=2.5.0,<=2.6.0',
+                  'python-cinderclient>=1.9.0,<=2.0.1',
+                  'python-swiftclient>=3.2.0,<=3.3.0',
+                  'python-neutronclient>=6.0.0,<=6.1.0',
+                  'python-keystoneclient>=3.8.0,<=3.10.0']
+aws_reqs = ['boto>=2.38.0,<=2.46.1']
+gce_reqs = ['google-api-python-client>=1.4.2', "cryptography>=1.4"]
 full_reqs = base_reqs + aws_reqs + openstack_reqs + gce_reqs
-dev_reqs = (['tox>=2.1.1', 'moto>=0.4.18', 'sphinx>=1.3.1'] + full_reqs)
+# httpretty is required with/for moto 1.0.0 or AWS tests fail
+dev_reqs = (['tox>=2.1.1', 'moto<1.0.0', 'sphinx>=1.3.1', 'flake8>=3.3.0',
+             'flake8-import-order>=0.12', 'httpretty==0.8.10'] + full_reqs)
 
 setup(name='cloudbridge',
       version=version,
@@ -42,7 +48,7 @@ setup(name='cloudbridge',
       packages=find_packages(),
       license='MIT',
       classifiers=[
-          'Development Status :: 3 - Alpha',
+          'Development Status :: 4 - Beta',
           'Environment :: Console',
           'Intended Audience :: Developers',
           'Intended Audience :: System Administrators',
@@ -54,6 +60,7 @@ setup(name='cloudbridge',
           'Programming Language :: Python :: 3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
           'Programming Language :: Python :: Implementation :: CPython',
           'Programming Language :: Python :: Implementation :: PyPy'],
       test_suite="test"

test/__init__.py

@@ -1,64 +1,9 @@
 """
-Tests the functionality of each provider implementation against registered
-test cases. These tests require that provider credentials are set as
-environment variables, as required for each provider (see `tox.ini` for a list
-of env variables).
-
-Since the tests exercise the ``cloudbridge`` interfaces, and there are multiple
-implementations of these interfaces, for m interfaces and n implementation,
-exercising all interfaces means that m*n test case classes are needed.
-Otherwise, the standard test runners such as unittest and nose2 do not
-correctly pick up the tests.
-
-To avoid an explosion of repetitive test cases, the
-``ProviderTestCaseGenerator`` class will automatically generate a new Python
-class for each combination of test and provider. The ``load_tests`` protocol
-(https://docs.python.org/2/library/unittest.html#load-tests-protocol)
-is used to aid test discovery.
-
 Use ``python setup.py test`` to run these unit tests (alternatively, use
 ``python -m unittest test``).
 
-All test cases need to be registered below, and available providers will be
-discovered through the ``ProviderFactory``. Test Cases must not inherit from
-``unittest.TestCase``, to avoid confusing unittest and nose2's automatic
-discovery. (The test generator will automatically add ``unittest.TestCase``
-as a base class to each combination).
+You must set the CB_TEST_PROVIDER environment variable before running the
+tests. Otherwise, the test suite will default to running against the mock
+aws provider. Alternatively, use tox, which will run tests for all available
+provider combinations.
 """
-import cloudbridge
-from test.helpers import ProviderTestCaseGenerator
-from test.test_block_store_service import CloudBlockStoreServiceTestCase
-from test.test_cloud_helpers import CloudHelpersTestCase
-from test.test_compute_service import CloudComputeServiceTestCase
-from test.test_image_service import CloudImageServiceTestCase
-from test.test_instance_types_service import CloudInstanceTypesServiceTestCase
-from test.test_interface import CloudInterfaceTestCase
-from test.test_network_service import CloudNetworkServiceTestCase
-from test.test_object_life_cycle import CloudObjectLifeCycleTestCase
-from test.test_object_store_service import CloudObjectStoreServiceTestCase
-from test.test_region_service import CloudRegionServiceTestCase
-from test.test_security_service import CloudSecurityServiceTestCase
-
-
-PROVIDER_TESTS = [
-    CloudHelpersTestCase,
-    CloudInterfaceTestCase,
-    CloudObjectLifeCycleTestCase,
-    CloudSecurityServiceTestCase,
-    CloudNetworkServiceTestCase,
-    CloudInstanceTypesServiceTestCase,
-    CloudBlockStoreServiceTestCase,
-    CloudObjectStoreServiceTestCase,
-    CloudComputeServiceTestCase,
-    CloudRegionServiceTestCase,
-    CloudImageServiceTestCase
-]
-
-
-def load_tests(loader=None, tests=None, pattern=None):
-    """
-    This function is required to aid the load_tests protocol
-    (https://docs.python.org/2/library/unittest.html#load-tests-protocol)
-    """
-    cloudbridge.init_logging()
-    return ProviderTestCaseGenerator(PROVIDER_TESTS).generate_tests()

test/helpers.py

@@ -1,14 +1,16 @@
-from contextlib import contextmanager
+import functools
 import os
 import sys
 import unittest
 
-from six import reraise
+from contextlib import contextmanager
 
 from cloudbridge.cloud.factory import CloudProviderFactory
 from cloudbridge.cloud.interfaces import InstanceState
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
+from six import reraise
+
 
 def parse_bool(val):
     if val:
@@ -50,6 +52,28 @@ def cleanup_action(cleanup_func):
         print("Error during cleanup: {0}".format(e))
 
 
+def skipIfNoService(services):
+    """
+    A decorator for skipping tests if the provider
+    does not implement a given service.
+    """
+    def wrap(func):
+        """
+        The actual wrapper
+        """
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            provider = getattr(self, 'provider')
+            if provider:
+                for service in services:
+                    if not provider.has_service(service):
+                        self.skipTest("Skipping test because '%s' service is"
+                                      " not implemented" % (service,))
+            func(self, *args, **kwargs)
+        return wrapper
+    return wrap
+
+
 TEST_DATA_CONFIG = {
     "AWSCloudProvider": {
         "image": os.environ.get('CB_IMAGE_AWS', 'ami-5ac2cd4d'),
@@ -58,7 +82,7 @@ TEST_DATA_CONFIG = {
     },
     "OpenStackCloudProvider": {
         "image": os.environ.get('CB_IMAGE_OS',
-                                'a471339a-bd0e-41e2-9406-4f308267ed0f'),
+                                '842b949c-ea76-48df-998d-8a41f2626243'),
         "instance_type": os.environ.get('CB_INSTANCE_TYPE_OS', 'm1.tiny'),
         "placement": os.environ.get('CB_PLACEMENT_OS', 'nova'),
     },
@@ -85,7 +109,8 @@ def create_test_network(provider, name):
     """
     net = provider.network.create(name=name)
     cidr_block = (net.cidr_block).split('/')[0] or '10.0.0.1'
-    sn = net.create_subnet(cidr_block='{0}/28'.format(cidr_block, name=name))
+    sn = net.create_subnet(cidr_block='{0}/28'.format(cidr_block), name=name,
+                           zone=get_provider_test_data(provider, 'placement'))
     return net, sn
 
 
@@ -99,26 +124,26 @@ def delete_test_network(network):
 
 
 def create_test_instance(
-        provider, instance_name, network, zone=None, launch_config=None,
+        provider, instance_name, subnet, launch_config=None,
         key_pair=None, security_groups=None):
     return provider.compute.instances.create(
         instance_name,
         get_provider_test_data(provider, 'image'),
         get_provider_test_data(provider, 'instance_type'),
-        network=network,
-        zone=zone,
+        subnet=subnet,
+        zone=get_provider_test_data(provider, 'placement'),
         key_pair=key_pair,
         security_groups=security_groups,
         launch_config=launch_config)
 
 
 def get_test_instance(provider, name, key_pair=None, security_groups=None,
-                      network=None):
+                      subnet=None):
     launch_config = None
     instance = create_test_instance(
         provider,
         name,
-        network=network,
+        subnet=subnet,
         key_pair=key_pair,
         security_groups=security_groups,
         launch_config=launch_config)
@@ -126,29 +151,31 @@ def get_test_instance(provider, name, key_pair=None, security_groups=None,
     return instance
 
 
+def get_test_fixtures_folder():
+    return os.path.join(os.path.dirname(__file__), 'fixtures/')
+
+
+def delete_test_instance(instance):
+    if instance:
+        instance.terminate()
+        instance.wait_for([InstanceState.TERMINATED, InstanceState.UNKNOWN],
+                          terminal_states=[InstanceState.ERROR])
+
+
 def cleanup_test_resources(instance=None, network=None, security_group=None,
                            key_pair=None):
-    with cleanup_action(lambda: delete_test_network(network)):
-        with cleanup_action(lambda: key_pair.delete()):
-            with cleanup_action(lambda: security_group.delete()):
-                instance.terminate()
-                instance.wait_for(
-                    [InstanceState.TERMINATED, InstanceState.UNKNOWN],
-                    terminal_states=[InstanceState.ERROR])
+    """Clean up any combination of supplied resources."""
+    with cleanup_action(lambda: delete_test_network(network)
+                        if network else None):
+        with cleanup_action(lambda: key_pair.delete() if key_pair else None):
+            with cleanup_action(lambda: security_group.delete()
+                                if security_group else None):
+                delete_test_instance(instance)
 
 
-class ProviderTestBase(object):
+class ProviderTestBase(unittest.TestCase):
 
-    """
-    A dummy base class for Test Cases. Does not inherit from unittest.TestCase
-    to avoid confusing test discovery by unittest and nose2. unittest.TestCase
-    is injected as a base class by the generator, so calling the unittest
-    constructor works correctly.
-    """
-
-    def __init__(self, methodName, provider):
-        unittest.TestCase.__init__(self, methodName=methodName)
-        self.provider = provider
+    _provider = None
 
     def setUp(self):
         if isinstance(self.provider, TestMockHelperMixin):
@@ -157,17 +184,7 @@ class ProviderTestBase(object):
     def tearDown(self):
         if isinstance(self.provider, TestMockHelperMixin):
             self.provider.tearDownMock()
-
-
-class ProviderTestCaseGenerator():
-
-    """
-    Generates test cases for all provider - testcase combinations.
-    Detailed docs at test/__init__.py
-    """
-
-    def __init__(self, test_classes):
-        self.all_test_classes = test_classes
+        self._provider = None
 
     def get_provider_wait_interval(self, provider_class):
         if issubclass(provider_class, TestMockHelperMixin):
@@ -175,80 +192,19 @@ class ProviderTestCaseGenerator():
         else:
             return 1
 
-    def create_provider_instance(self, provider_class):
-        """
-        Instantiate a default provider instance. All required connection
-        settings are expected to be set as environment variables.
-        """
+    def create_provider_instance(self):
+        provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
+        use_mock_drivers = parse_bool(
+            os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
+        factory = CloudProviderFactory()
+        provider_class = factory.get_provider_class(provider_name,
+                                                    get_mock=use_mock_drivers)
         config = {'default_wait_interval':
                   self.get_provider_wait_interval(provider_class)}
         return provider_class(config)
 
-    def generate_new_test_class(self, name, testcase_class):
-        """
-        Generates a new type which inherits from the given testcase_class and
-        unittest.TestCase
-        """
-        class_name = "{0}{1}".format(name, testcase_class.__name__)
-        return type(class_name, (testcase_class, unittest.TestCase), {})
-
-    def generate_test_suite_for_provider_testcase(
-            self, provider_class, testcase_class):
-        """
-        Generate and return a suite of tests for a specific provider class and
-        testcase combination
-        """
-        testloader = unittest.TestLoader()
-        testnames = testloader.getTestCaseNames(testcase_class)
-        suite = unittest.TestSuite()
-        for name in testnames:
-            generated_cls = self.generate_new_test_class(
-                provider_class.__name__,
-                testcase_class)
-            suite.addTest(
-                generated_cls(
-                    name,
-                    self.create_provider_instance(provider_class)))
-        return suite
-
-    def generate_test_suite_for_provider(self, provider_class):
-        """
-        Generate and return a suite of all available tests for a given provider
-        class
-        """
-        suite = unittest.TestSuite()
-        suites = [
-            self.generate_test_suite_for_provider_testcase(
-                provider_class, test_class)
-            for test_class in self.all_test_classes]
-        for s in suites:
-            suite.addTest(s)
-        return suite
-
-    def generate_tests(self):
-        """
-        Generate and return a suite of tests for all provider and test class
-        combinations
-        """
-        factory = CloudProviderFactory()
-        use_mock_drivers = parse_bool(
-            os.environ.get("CB_USE_MOCK_PROVIDERS", True))
-        provider_name = os.environ.get("CB_TEST_PROVIDER", None)
-        if provider_name:
-            provider_classes = [
-                factory.get_provider_class(
-                    provider_name,
-                    get_mock=use_mock_drivers)]
-            if not provider_classes[0]:
-                raise ValueError(
-                    "Could not find specified test provider %s" %
-                    provider_name)
-        else:
-            provider_classes = factory.get_all_provider_classes(
-                get_mock=use_mock_drivers)
-        suite = unittest.TestSuite()
-        suites = [
-            self.generate_test_suite_for_provider(p) for p in provider_classes]
-        for s in suites:
-            suite.addTest(s)
-        return suite
+    @property
+    def provider(self):
+        if not self._provider:
+            self._provider = self.create_provider_instance()
+        return self._provider

test/test_block_store_service.py

@@ -1,20 +1,19 @@
+import time
 import uuid
 
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces import SnapshotState
 from cloudbridge.cloud.interfaces import VolumeState
 from cloudbridge.cloud.interfaces.resources import AttachmentInfo
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
 
+import six
 
-class CloudBlockStoreServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudBlockStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudBlockStoreServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_crud_volume(self):
         """
         Create a new volume, check whether the expected values are set,
@@ -91,6 +90,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
             "Volume %s should have been deleted but still exists." %
             name)
 
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_attach_detach_volume(self):
         """
         Create a new volume, and attempt to attach it to an instance
@@ -98,11 +98,16 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
         instance_name = "CBVolOps-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, instance_name)
-        test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  network=net)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        test_instance = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
+            net, subnet = helpers.create_test_network(
+                self.provider, instance_name)
+            test_instance = helpers.get_test_instance(
+                self.provider, instance_name, subnet=subnet)
             name = "CBUnitTestAttachVol-{0}".format(uuid.uuid4())
             test_vol = self.provider.block_store.volumes.create(
                 name, 1, test_instance.zone_id)
@@ -117,6 +122,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                     [VolumeState.AVAILABLE],
                     terminal_states=[VolumeState.ERROR, VolumeState.DELETED])
 
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_volume_properties(self):
         """
         Test volume properties
@@ -125,11 +131,17 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
             self.provider.name,
             uuid.uuid4())
         vol_desc = 'newvoldesc1'
-        net, _ = helpers.create_test_network(self.provider, instance_name)
-        test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  network=net)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_instance = None
+        net = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
+            net, subnet = helpers.create_test_network(
+                self.provider, instance_name)
+            test_instance = helpers.get_test_instance(
+                self.provider, instance_name, subnet=subnet)
+
             name = "CBUnitTestVolProps-{0}".format(uuid.uuid4())
             test_vol = self.provider.block_store.volumes.create(
                 name, 1, test_instance.zone_id, description=vol_desc)
@@ -163,8 +175,9 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                                  "/dev/sda2")
                 test_vol.detach()
                 test_vol.name = 'newvolname1'
-                # Force a refresh before checking attachment status
-                test_vol.refresh()
+                test_vol.wait_for(
+                    [VolumeState.AVAILABLE],
+                    terminal_states=[VolumeState.ERROR, VolumeState.DELETED])
                 self.assertEqual(test_vol.name, 'newvolname1')
                 self.assertEqual(test_vol.description, vol_desc)
                 self.assertIsNone(test_vol.attachments)
@@ -172,6 +185,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                     [VolumeState.AVAILABLE],
                     terminal_states=[VolumeState.ERROR, VolumeState.DELETED])
 
+    @helpers.skipIfNoService(['block_store.snapshots'])
     def test_crud_snapshot(self):
         """
         Create a new volume, create a snapshot of the volume, and check
@@ -280,6 +294,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
 
             # Test creation of a snap via SnapshotService
             snap_too_name = "CBSnapToo-{0}".format(name)
+            time.sleep(15)  # Or get SnapshotCreationPerVolumeRateExceeded
             test_snap_too = self.provider.block_store.snapshots.create(
                 name=snap_too_name, volume=test_vol, description=snap_too_name)
             with helpers.cleanup_action(lambda: cleanup_snap(test_snap_too)):
@@ -289,6 +304,7 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
                     "repr(obj) should contain the object id so that the object"
                     " can be reconstructed, but does not.")
 
+    @helpers.skipIfNoService(['block_store.snapshots'])
     def test_snapshot_properties(self):
         """
         Test snapshot properties

test/test_cloud_factory.py

@@ -1,5 +1,7 @@
 import unittest
 
+from test import helpers
+
 from cloudbridge.cloud import factory
 from cloudbridge.cloud import interfaces
 from cloudbridge.cloud.factory import CloudProviderFactory
@@ -7,7 +9,6 @@ from cloudbridge.cloud.interfaces import TestMockHelperMixin
 from cloudbridge.cloud.interfaces.provider import CloudProvider
 from cloudbridge.cloud.providers.aws import AWSCloudProvider
 from cloudbridge.cloud.providers.aws.provider import MockAWSCloudProvider
-import test.helpers as helpers
 
 
 class CloudFactoryTestCase(unittest.TestCase):

test/test_cloud_helpers.py

@@ -1,8 +1,9 @@
 import itertools
 
+from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.base.resources import ClientPagedResultList
 from cloudbridge.cloud.base.resources import ServerPagedResultList
-from test.helpers import ProviderTestBase
 
 
 class DummyResult(object):
@@ -17,10 +18,6 @@ class DummyResult(object):
 
 class CloudHelpersTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudHelpersTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def setUp(self):
         super(CloudHelpersTestCase, self).setUp()
         self.objects = [DummyResult(1, "One"),
@@ -75,4 +72,4 @@ class CloudHelpersTestCase(ProviderTestBase):
         self.assertTrue(results.supports_server_paging, "Server paged result"
                         " lists should return True for server paging.")
         with self.assertRaises(NotImplementedError):
-            _ = results.data
+            results.data

test/test_compute_service.py

@@ -1,31 +1,36 @@
+import ipaddress
 import uuid
 
-import ipaddress
-import six
-from cloudbridge.cloud.interfaces \
-    import InvalidConfigurationException
+from test import helpers
+from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.interfaces import InstanceState
-from cloudbridge.cloud.interfaces.resources import InstanceType
+from cloudbridge.cloud.interfaces import InvalidConfigurationException
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 from cloudbridge.cloud.interfaces.exceptions import WaitStateException
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+from cloudbridge.cloud.interfaces.resources import InstanceType
+# from cloudbridge.cloud.interfaces.resources import SnapshotState
 
+import six
 
-class CloudComputeServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudComputeServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudComputeServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['compute.instances', 'network'])
     def test_crud_instance(self):
         name = "CBInstCrud-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, name)
-        inst = helpers.get_test_instance(self.provider, name, network=net)
-
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        inst = None
+        net = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 inst, net)):
+            net, subnet = helpers.create_test_network(self.provider, name)
+            inst = helpers.get_test_instance(self.provider, name,
+                                             subnet=subnet)
+
             all_instances = self.provider.compute.instances.list()
 
             list_instances = [i for i in all_instances if i.name == name]
@@ -88,21 +93,31 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             return False
         return True
 
+    @helpers.skipIfNoService(['compute.instances', 'network',
+                              'security.security_groups',
+                              'security.key_pairs'])
     def test_instance_properties(self):
         name = "CBInstProps-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, name)
-        kp = self.provider.security.key_pairs.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        test_instance = helpers.get_test_instance(self.provider,
-                                                  name, key_pair=kp,
-                                                  security_groups=[sg],
-                                                  network=net)
 
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_instance = None
+        net = None
+        sg = None
+        kp = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net, sg, kp)):
+            net, subnet = helpers.create_test_network(self.provider, name)
+            kp = self.provider.security.key_pairs.create(name=name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
+            test_instance = helpers.get_test_instance(self.provider,
+                                                      name, key_pair=kp,
+                                                      security_groups=[sg],
+                                                      subnet=subnet)
+
             self.assertTrue(
                 test_instance.id in repr(test_instance),
                 "repr(obj) should contain the object id so that the object"
@@ -168,6 +183,8 @@ class CloudComputeServiceTestCase(ProviderTestBase):
                 "Instance type {0} does not match expected type {1}".format(
                     itype.name, expected_type))
 
+    @helpers.skipIfNoService(['compute.instances', 'compute.images',
+                              'compute.instance_types'])
     def test_block_device_mapping_launch_config(self):
         lc = self.provider.compute.instances.create_launch_config()
 
@@ -193,12 +210,12 @@ class CloudComputeServiceTestCase(ProviderTestBase):
         # Override root volume size
         image_id = helpers.get_provider_test_data(self.provider, "image")
         img = self.provider.compute.images.get(image_id)
+        # The size should be greater then the ami size
+        # and therefore, img.min_disk is used.
         lc.add_volume_device(
             is_root=True,
             source=img,
-            # TODO: This should be greater than the ami size or tests will fail
-            # on actual infrastructure. Needs an image.size method
-            size=2,
+            size=img.min_disk if img and img.min_disk else 2,
             delete_on_terminate=True)
 
         # Attempting to add more than one root volume should raise an
@@ -227,87 +244,163 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             "Expected %d total block devices bit found %d" %
             (2 + inst_type.num_ephemeral_disks, len(lc.block_devices)))
 
+    @helpers.skipIfNoService(['compute.instances', 'compute.images',
+                              'compute.instance_types', 'block_store.volumes'])
     def test_block_device_mapping_attachments(self):
         name = "CBInstBlkAttch-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
 
-#         test_vol = self.provider.block_store.volumes.create(
-#             name,
-#             1,
-#             helpers.get_provider_test_data(self.provider, "placement"))
-#         with helpers.cleanup_action(lambda: test_vol.delete()):
-#             test_vol.wait_till_ready()
-#             test_snap = test_vol.create_snapshot(name=name,
-#                                                  description=name)
+        # Comment out BDM tests because OpenStack is not stable enough yet
+        if True:
+            if True:
+
+                # test_vol = self.provider.block_store.volumes.create(
+                #    name,
+                #    1,
+                #    helpers.get_provider_test_data(self.provider,
+                #                                   "placement"))
+                # with helpers.cleanup_action(lambda: test_vol.delete()):
+                #    test_vol.wait_till_ready()
+                #    test_snap = test_vol.create_snapshot(name=name,
+                #                                         description=name)
+                #
+                #    def cleanup_snap(snap):
+                #        snap.delete()
+                #        snap.wait_for(
+                #            [SnapshotState.UNKNOWN],
+                #            terminal_states=[SnapshotState.ERROR])
+                #
+                #    with helpers.cleanup_action(lambda:
+                #                                cleanup_snap(test_snap)):
+                #         test_snap.wait_till_ready()
+
+                lc = self.provider.compute.instances.create_launch_config()
+
+#                 # Add a new blank volume
+#                 lc.add_volume_device(size=1, delete_on_terminate=True)
 #
-#             def cleanup_snap(snap):
-#                 snap.delete()
-#                 snap.wait_for(
-#                     [SnapshotState.UNKNOWN],
-#                     terminal_states=[SnapshotState.ERROR])
-#
-#             with helpers.cleanup_action(lambda: cleanup_snap(test_snap)):
-#                 test_snap.wait_till_ready()
-
-        lc = self.provider.compute.instances.create_launch_config()
-
-        # Add a new blank volume
-#         lc.add_volume_device(size=1, delete_on_terminate=True)
-
-        # Attach an existing volume
+#                 # Attach an existing volume
 #                 lc.add_volume_device(size=1, source=test_vol,
 #                                      delete_on_terminate=True)
-
-        # Add a new volume based on a snapshot
+#
+#                 # Add a new volume based on a snapshot
 #                 lc.add_volume_device(size=1, source=test_snap,
 #                                      delete_on_terminate=True)
 
-        # Override root volume size
-        image_id = helpers.get_provider_test_data(
-            self.provider,
-            "image")
-        img = self.provider.compute.images.get(image_id)
-        lc.add_volume_device(
-            is_root=True,
-            source=img,
-            # TODO: This should be greater than the ami size or tests
-            # will fail on actual infrastructure. Needs an image.size
-            # method
-            size=2,
-            delete_on_terminate=True)
-
-        # Add all available ephemeral devices
-        instance_type_name = helpers.get_provider_test_data(
-            self.provider,
-            "instance_type")
-        inst_type = self.provider.compute.instance_types.find(
-            name=instance_type_name)[0]
-        for _ in range(inst_type.num_ephemeral_disks):
-            lc.add_ephemeral_device()
+                # Override root volume size
+                image_id = helpers.get_provider_test_data(
+                    self.provider,
+                    "image")
+                img = self.provider.compute.images.get(image_id)
+                # The size should be greater then the ami size
+                # and therefore, img.min_disk is used.
+                lc.add_volume_device(
+                    is_root=True,
+                    source=img,
+                    size=img.min_disk if img and img.min_disk else 2,
+                    delete_on_terminate=True)
+
+                # Add all available ephemeral devices
+                instance_type_name = helpers.get_provider_test_data(
+                    self.provider,
+                    "instance_type")
+                inst_type = self.provider.compute.instance_types.find(
+                    name=instance_type_name)[0]
+                for _ in range(inst_type.num_ephemeral_disks):
+                    lc.add_ephemeral_device()
+
+                net, subnet = helpers.create_test_network(self.provider, name)
+
+                with helpers.cleanup_action(lambda:
+                                            helpers.delete_test_network(net)):
+
+                    inst = helpers.create_test_instance(
+                        self.provider,
+                        name,
+                        subnet=subnet,
+                        launch_config=lc)
+
+                    with helpers.cleanup_action(lambda:
+                                                helpers.delete_test_instance(
+                                                    inst)):
+                        try:
+                            inst.wait_till_ready()
+                        except WaitStateException as e:
+                            self.fail("The block device mapped launch did not "
+                                      " complete successfully: %s" % e)
+                        # TODO: Check instance attachments and make sure they
+                        # correspond to requested mappings
+
+    @helpers.skipIfNoService(['compute.instances', 'network',
+                              'security.security_groups'])
+    def test_instance_methods(self):
+        name = "CBInstProps-{0}-{1}".format(
+            self.provider.name,
+            uuid.uuid4())
 
-        net, _ = helpers.create_test_network(self.provider, name)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_inst = None
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                test_inst, net, sg)):
+            net, subnet = helpers.create_test_network(self.provider, name)
+            test_inst = helpers.get_test_instance(self.provider, name,
+                                                  subnet=subnet)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
+
+            # Check adding a security group to a running instance
+            test_inst.add_security_group(sg)
+            test_inst.refresh()
+            self.assertTrue(
+                sg in test_inst.security_groups, "Expected security group '%s'"
+                " to be among instance security_groups: [%s]" %
+                (sg, test_inst.security_groups))
 
-        inst = helpers.create_test_instance(
-            self.provider,
-            name,
-            network=net,
-            # We don't have a way to match the test net placement and this zone
-            # zone=helpers.get_provider_test_data(self.provider, 'placement'),
-            launch_config=lc)
-
-        def cleanup(instance, net):
-            instance.terminate()
-            instance.wait_for(
-                [InstanceState.TERMINATED, InstanceState.UNKNOWN],
-                terminal_states=[InstanceState.ERROR])
-            helpers.delete_test_network(net)
-
-        with helpers.cleanup_action(lambda: cleanup(inst, net)):
-            try:
-                inst.wait_till_ready()
-            except WaitStateException as e:
-                self.fail("The block device mapped launch did not "
-                          " complete successfully: %s" % e)
-            # TODO: Check instance attachments and make sure they
-            # correspond to requested mappings
+            # Check removing a security group from a running instance
+            test_inst.remove_security_group(sg)
+            test_inst.refresh()
+            self.assertTrue(
+                sg not in test_inst.security_groups, "Expected security group"
+                " '%s' to be removed from instance security_groups: [%s]" %
+                (sg, test_inst.security_groups))
+
+            # check floating ips
+            router = self.provider.network.create_router(name=name)
+
+            with helpers.cleanup_action(lambda: router.delete()):
+
+                # TODO: Cloud specific code, needs fixing
+                if self.provider.PROVIDER_ID == 'openstack':
+                    for n in self.provider.network.list():
+                        if n.external:
+                            external_net = n
+                            break
+                else:
+                    external_net = net
+                router.attach_network(external_net.id)
+                router.add_route(subnet.id)
+
+                def cleanup_router():
+                    router.remove_route(subnet.id)
+                    router.detach_network()
+
+                with helpers.cleanup_action(lambda: cleanup_router()):
+                    # check whether adding an elastic ip works
+                    fip = self.provider.network.create_floating_ip()
+                    with helpers.cleanup_action(lambda: fip.delete()):
+                        test_inst.add_floating_ip(fip.public_ip)
+                        test_inst.refresh()
+                        self.assertIn(fip.public_ip, test_inst.public_ips)
+
+                        if isinstance(self.provider, TestMockHelperMixin):
+                            # TODO: Moto bug does not refresh removed public ip
+                            return
+
+                        # check whether removing an elastic ip works
+                        test_inst.remove_floating_ip(fip.public_ip)
+                        test_inst.refresh()
+                        self.assertNotIn(fip.public_ip, test_inst.public_ips)

test/test_image_service.py

@@ -1,18 +1,18 @@
 import uuid
 
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces import MachineImageState
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
+import six
 
-class CloudImageServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudImageServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudImageServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['compute.images', 'network',
+                              'compute.instances'])
     def test_create_and_list_image(self):
         """
         Create a new image and check whether that image can be listed.
@@ -22,11 +22,18 @@ class CloudImageServiceTestCase(ProviderTestBase):
         instance_name = "CBImageTest-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, _ = helpers.create_test_network(self.provider, instance_name)
-        test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  network=net)
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_instance = None
+        net = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
+            net, subnet = helpers.create_test_network(
+                self.provider, instance_name)
+            test_instance = helpers.get_test_instance(
+                self.provider, instance_name, subnet=subnet)
+
             name = "CBUnitTestListImg-{0}".format(uuid.uuid4())
             test_image = test_instance.create_image(name)
 
@@ -48,26 +55,27 @@ class CloudImageServiceTestCase(ProviderTestBase):
                         test_image.description, six.string_types),
                     "Image description must be None or a string")
 
-                images = self.provider.compute.images.list()
-                list_images = [image for image in images
-                               if image.name == name]
-                self.assertTrue(
-                    len(list_images) == 1,
-                    "List images does not return the expected image %s" %
-                    name)
+                # This check won't work when >50 images are available
+                # images = self.provider.compute.images.list()
+                # list_images = [image for image in images
+                #                if image.name == name]
+                # self.assertTrue(
+                #     len(list_images) == 1,
+                #     "List images does not return the expected image %s" %
+                #     name)
 
                 # check iteration
                 iter_images = [image for image in self.provider.compute.images
                                if image.name == name]
                 self.assertTrue(
-                    len(iter_images) == 1,
-                    "Iter images does not return the expected image %s" %
-                    name)
+                    name in [ii.name for ii in iter_images],
+                    "Iter images (%s) does not contain the expected image %s" %
+                    (iter_images, name))
 
                 # find image
                 found_images = self.provider.compute.images.find(name=name)
                 self.assertTrue(
-                    len(found_images) == 1,
+                    name in [fi.name for fi in found_images],
                     "Find images error: expected image %s but found: %s" %
                     (name, found_images))
 
@@ -82,19 +90,22 @@ class CloudImageServiceTestCase(ProviderTestBase):
                 get_img = self.provider.compute.images.get(
                     test_image.id)
                 self.assertTrue(
-                    found_images[0] == iter_images[0] == get_img == test_image,
+                    found_images[0] == get_img == test_image,
                     "Objects returned by list: {0} and get: {1} are not as "
                     " expected: {2}" .format(found_images[0].id,
                                              get_img.id,
                                              test_image.id))
                 self.assertTrue(
-                    list_images[0].name == found_images[0].name ==
-                    get_img.name == test_image.name,
-                    "Names returned by list: {0}, find: {1} and get: {2} are"
-                    " not as expected: {3}" .format(list_images[0].name,
-                                                    found_images[0].name,
+                    found_images[0].name == get_img.name == test_image.name,
+                    "Names returned by find: {0} and get: {1} are"
+                    " not as expected: {2}" .format(found_images[0].name,
                                                     get_img.name,
                                                     test_image.name))
+                # TODO: Fix moto so that the BDM is populated correctly
+                if not isinstance(self.provider, TestMockHelperMixin):
+                    # check image size
+                    self.assertGreater(get_img.min_disk, 0, "Minimum disk size"
+                                       " required by image is invalid")
             # TODO: Images take a long time to deregister on EC2. Needs
             # investigation
             images = self.provider.compute.images.list()

test/test_instance_types_service.py

@@ -1,21 +1,19 @@
 from test import helpers
 
-import six
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces.resources import InstanceType
-from test.helpers import ProviderTestBase
 
+import six
 
-class CloudInstanceTypesServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudInstanceTypesServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudInstanceTypesServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types(self):
         instance_types = self.provider.compute.instance_types.list()
-        # check iteration
-        iter_instance_types = list(self.provider.compute.instance_types)
+        # Check iteration, keeping the first 50 entries (the .list() default)
+        iter_instance_types = list(self.provider.compute.instance_types)[:50]
         self.assertListEqual(iter_instance_types, instance_types)
 
         for inst_type in instance_types:
@@ -69,6 +67,7 @@ class CloudInstanceTypesServiceTestCase(ProviderTestBase):
                     inst_type.extra_data, dict),
                 "InstanceType extra_data must be None or a dict")
 
+    @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types_find(self):
         """
         Searching for an instance by name should return an
@@ -91,6 +90,7 @@ class CloudInstanceTypesServiceTestCase(ProviderTestBase):
             self.provider.compute.instance_types.find(
                 non_existent_param="random_value")
 
+    @helpers.skipIfNoService(['compute.instance_types'])
     def test_instance_types_get(self):
         """
         Searching for an instance by id should return an

test/test_interface.py

@@ -1,18 +1,17 @@
 import unittest
+
+from test.helpers import ProviderTestBase
+
 import cloudbridge
+
 from cloudbridge.cloud import interfaces
-from test.helpers import ProviderTestBase
-from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
-from cloudbridge.cloud.interfaces import TestMockHelperMixin
 from cloudbridge.cloud.factory import CloudProviderFactory
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
+from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 
 
 class CloudInterfaceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudInterfaceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
     def test_name_property(self):
         """
         Name should always return a value and should not raise an exception

test/test_network_service.py

@@ -1,17 +1,16 @@
 import test.helpers as helpers
 import uuid
 from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.interfaces.resources import RouterState
 
 
 class CloudNetworkServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudNetworkServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['network'])
     def test_crud_network_service(self):
         name = 'cbtestnetworkservice-{0}'.format(uuid.uuid4())
+        subnet_name = 'cbtestsubnetservice-{0}'.format(uuid.uuid4())
         net = self.provider.network.create(name=name)
         with helpers.cleanup_action(
             lambda:
@@ -32,21 +31,7 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 "Get network did not return the expected network {0}."
                 .format(name))
 
-        netl = self.provider.network.list()
-        found_net = [n for n in netl if n.name == name]
-        self.assertEqual(
-            len(found_net), 0,
-            "Network {0} should have been deleted but still exists."
-            .format(name))
-
-    def test_crud_subnet_service(self):
-        name = 'cbtestnetworkservice-{0}'.format(uuid.uuid4())
-        subnet_name = 'cbtestsubnetservice-{0}'.format(uuid.uuid4())
-        net = self.provider.network.create(name=name)
-        with helpers.cleanup_action(
-            lambda:
-                self.provider.network.delete(network_id=net.id)
-        ):
+            # check subnet
             subnet = self.provider.network.subnets.create(
                 network=net, cidr_block="10.0.0.1/24", name=subnet_name)
             with helpers.cleanup_action(
@@ -73,6 +58,38 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 "Subnet {0} should have been deleted but still exists."
                 .format(subnet_name))
 
+            # Check floating IP address
+            ip = self.provider.network.create_floating_ip()
+            ip_id = ip.id
+            with helpers.cleanup_action(lambda: ip.delete()):
+                ipl = self.provider.network.floating_ips()
+                self.assertTrue(
+                    ip in ipl,
+                    "Floating IP address {0} should exist in the list {1}"
+                    .format(ip.id, ipl))
+                # 2016-08: address filtering not implemented in moto
+                # empty_ipl = self.provider.network.floating_ips('dummy-net')
+                # self.assertFalse(
+                #     empty_ipl,
+                #     "Bogus network should not have any floating IPs: {0}"
+                #     .format(empty_ipl))
+                self.assertIn(
+                    ip.public_ip, repr(ip),
+                    "repr(obj) should contain the address public IP value.")
+                self.assertFalse(
+                    ip.private_ip,
+                    "Floating IP should not have a private IP value ({0})."
+                    .format(ip.private_ip))
+                self.assertFalse(
+                    ip.in_use(),
+                    "Newly created floating IP address should not be in use.")
+            ipl = self.provider.network.floating_ips()
+            found_ip = [a for a in ipl if a.id == ip_id]
+            self.assertTrue(
+                len(found_ip) == 0,
+                "Floating IP {0} should have been deleted but still exists."
+                .format(ip_id))
+
         netl = self.provider.network.list()
         found_net = [n for n in netl if n.name == name]
         self.assertEqual(
@@ -112,6 +129,9 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
             "Floating IP {0} should have been deleted but still exists."
             .format(ip_id))
 
+    @helpers.skipIfNoService(['network'])
+    @helpers.skipIfNoService(['network'])
+    @helpers.skipIfNoService(['network'])
     def test_crud_network(self):
         name = 'cbtestnetwork-{0}'.format(uuid.uuid4())
         subnet_name = 'cbtestsubnet-{0}'.format(uuid.uuid4())
@@ -121,8 +141,8 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
         ):
             net.wait_till_ready()
             self.assertEqual(
-                net.refresh(), 'available',
-                "Network in state %s , yet should be 'available'" % net.state)
+                net.state, 'available',
+                "Network in state '%s', yet should be 'available'" % net.state)
 
             self.assertIn(
                 net.id, repr(net),
@@ -135,7 +155,10 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 % net.cidr_block)
 
             cidr = '10.0.1.0/24'
-            sn = net.create_subnet(cidr_block=cidr, name=subnet_name)
+            sn = net.create_subnet(
+                cidr_block=cidr, name=subnet_name,
+                zone=helpers.get_provider_test_data(self.provider,
+                                                    'placement'))
             with helpers.cleanup_action(lambda: sn.delete()):
                 self.assertTrue(
                     sn.id in [s.id for s in net.subnets()],
@@ -152,21 +175,30 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                     "Subnet's CIDR %s should match the specified one %s." % (
                         sn.cidr_block, cidr))
 
+    @helpers.skipIfNoService(['network.routers'])
     def test_crud_router(self):
 
         def _cleanup(net, subnet, router):
-            router.remove_route(subnet.id)
-            router.detach_network()
-            router.delete()
-            subnet.delete()
-            net.delete()
+            with helpers.cleanup_action(lambda: net.delete()):
+                with helpers.cleanup_action(lambda: subnet.delete()):
+                    with helpers.cleanup_action(lambda: router.delete()):
+                        router.remove_route(subnet.id)
+                        router.detach_network()
 
         name = 'cbtestrouter-{0}'.format(uuid.uuid4())
-        router = self.provider.network.create_router(name=name)
-        net = self.provider.network.create(name=name)
-        cidr = '10.0.1.0/24'
-        sn = net.create_subnet(cidr_block=cidr, name=name)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sn = None
+        router = None
         with helpers.cleanup_action(lambda: _cleanup(net, sn, router)):
+            router = self.provider.network.create_router(name=name)
+            net = self.provider.network.create(name=name)
+            cidr = '10.0.1.0/24'
+            sn = net.create_subnet(cidr_block=cidr, name=name,
+                                   zone=helpers.get_provider_test_data(
+                                       self.provider, 'placement'))
+
             # Check basic router properties
             self.assertIn(
                 router, self.provider.network.routers(),
@@ -188,6 +220,7 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 "Router {0} should not be assoc. with a network {1}".format(
                     router.id, router.network_id))
 
+            # TODO: Cloud specific code, needs fixing
             # Check router connectivity
             # On OpenStack only one network is external and on AWS every
             # network is external, yet we need to use the one we've created?!

test/test_object_life_cycle.py

@@ -1,17 +1,15 @@
 import uuid
 
+from test import helpers
+from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.interfaces import VolumeState
 from cloudbridge.cloud.interfaces.exceptions import WaitStateException
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
 
 
 class CloudObjectLifeCycleTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudObjectLifeCycleTestCase, self).__init__(
-            methodName=methodName, provider=provider)
-
+    @helpers.skipIfNoService(['block_store.volumes'])
     def test_object_life_cycle(self):
         """
         Test object life cycle methods by using a volume.

test/test_object_store_service.py

@@ -1,18 +1,22 @@
+import filecmp
+import os
+import tempfile
+import uuid
+
 from datetime import datetime
 from io import BytesIO
-import uuid
+from test import helpers
+from test.helpers import ProviderTestBase
+from unittest import skip
 
 from cloudbridge.cloud.interfaces.resources import BucketObject
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
 
+import requests
 
-class CloudObjectStoreServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudObjectStoreServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudObjectStoreServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['object_store'])
     def test_crud_bucket(self):
         """
         Create a new bucket, check whether the expected values are set,
@@ -66,6 +70,7 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
             "Bucket %s should have been deleted but still exists." %
             name)
 
+    @helpers.skipIfNoService(['object_store'])
     def test_crud_bucket_objects(self):
         """
         Create a new bucket, upload some contents into the bucket, and
@@ -80,7 +85,8 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
         self.assertEqual([], objects)
 
         with helpers.cleanup_action(lambda: test_bucket.delete()):
-            obj_name = "hello_world.txt"
+            obj_name_prefix = "hello"
+            obj_name = obj_name_prefix + "_world.txt"
             obj = test_bucket.create_object(obj_name)
 
             self.assertTrue(
@@ -101,7 +107,7 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                     "Object size property needs to be a int, not {0}".format(
                         type(objs[0].size)))
                 self.assertTrue(
-                    datetime.strptime(objs[0].last_modified,
+                    datetime.strptime(objs[0].last_modified[:23],
                                       "%Y-%m-%dT%H:%M:%S.%f"),
                     "Object's last_modified field format {0} not matching."
                     .format(objs[0].last_modified))
@@ -130,6 +136,13 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                     isinstance(obj_too, BucketObject),
                     "Did not get object {0} of expected type.".format(obj_too))
 
+                prefix_filtered_list = test_bucket.list(prefix=obj_name_prefix)
+                self.assertTrue(
+                    len(objs) == len(prefix_filtered_list) == 1,
+                    'The number of objects returned by list function, '
+                    'with and without a prefix, are expected to be equal, '
+                    'but its detected otherwise.')
+
             objs = test_bucket.list()
             found_objs = [o for o in objs if o.name == obj_name]
             self.assertTrue(
@@ -137,8 +150,8 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                 "Object %s should have been deleted but still exists." %
                 obj_name)
 
+    @helpers.skipIfNoService(['object_store'])
     def test_upload_download_bucket_content(self):
-
         name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
         test_bucket = self.provider.object_store.create(name)
 
@@ -159,3 +172,70 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
                 for data in obj.iter_content():
                     target_stream2.write(data)
                 self.assertEqual(target_stream2.getvalue(), content)
+
+    @skip("Skip until OpenStack implementation is provided")
+    @helpers.skipIfNoService(['object_store'])
+    def test_generate_url(self):
+        name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
+        test_bucket = self.provider.object_store.create(name)
+
+        with helpers.cleanup_action(lambda: test_bucket.delete()):
+            obj_name = "hello_upload_download.txt"
+            obj = test_bucket.create_object(obj_name)
+
+            with helpers.cleanup_action(lambda: obj.delete()):
+                content = b"Hello World. Generate a url."
+                obj.upload(content)
+                target_stream = BytesIO()
+                obj.save_content(target_stream)
+
+                url = obj.generate_url(100)
+                self.assertEqual(requests.get(url).content, content)
+
+    @helpers.skipIfNoService(['object_store'])
+    def test_upload_download_bucket_content_from_file(self):
+        name = "cbtestbucketobjs-{0}".format(uuid.uuid4())
+        test_bucket = self.provider.object_store.create(name)
+
+        with helpers.cleanup_action(lambda: test_bucket.delete()):
+            obj_name = "hello_upload_download.txt"
+            obj = test_bucket.create_object(obj_name)
+
+            with helpers.cleanup_action(lambda: obj.delete()):
+                test_file = os.path.join(
+                    helpers.get_test_fixtures_folder(), 'logo.jpg')
+                obj.upload_from_file(test_file)
+                target_stream = BytesIO()
+                obj.save_content(target_stream)
+                with open(test_file, 'rb') as f:
+                    self.assertEqual(target_stream.getvalue(), f.read())
+
+    @skip("Skip unless you want to test swift objects bigger than 5 Gig")
+    @helpers.skipIfNoService(['object_store'])
+    def test_upload_download_bucket_content_with_large_file(self):
+        """
+        Creates a 6 Gig file in the temp directory, then uploads it to
+        Swift. Once uploaded, then downloads to a new file in the temp
+        directory and compares the two files to see if they match.
+        """
+        temp_dir = tempfile.gettempdir()
+        file_name = '6GigTest.tmp'
+        six_gig_file = os.path.join(temp_dir, file_name)
+        with open(six_gig_file, "wb") as out:
+            out.truncate(6 * 1024 * 1024 * 1024)  # 6 Gig...
+        with helpers.cleanup_action(lambda: os.remove(six_gig_file)):
+            download_file = "{0}/cbtestfile-{1}".format(temp_dir, file_name)
+            bucket_name = "cbtestbucketlargeobjs-{0}".format(uuid.uuid4())
+            test_bucket = self.provider.object_store.create(bucket_name)
+            with helpers.cleanup_action(lambda: test_bucket.delete()):
+                test_obj = test_bucket.create_object(file_name)
+                with helpers.cleanup_action(lambda: test_obj.delete()):
+                    file_uploaded = test_obj.upload_from_file(six_gig_file)
+                    self.assertTrue(file_uploaded, "Could not upload object?")
+                    with helpers.cleanup_action(
+                            lambda: os.remove(download_file)):
+                        with open(download_file, 'wb') as f:
+                            test_obj.save_content(f)
+                            self.assertTrue(
+                                filecmp.cmp(six_gig_file, download_file),
+                                "Uploaded file != downloaded")

test/test_region_service.py

@@ -1,16 +1,14 @@
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces import Region
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
 
+import six
 
-class CloudRegionServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudRegionServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudRegionServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_get_and_list_regions(self):
         """
         Test whether the region listing methods work,
@@ -47,6 +45,7 @@ class CloudRegionServiceTestCase(ProviderTestBase):
             "Region name {0} not in JSON representation {1}".format(
                 region.name, region.to_json()))
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_regions_unique(self):
         """
         Regions should not return duplicate items
@@ -55,6 +54,7 @@ class CloudRegionServiceTestCase(ProviderTestBase):
         unique_regions = set([region.id for region in regions])
         self.assertTrue(len(regions) == len(list(unique_regions)))
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_current_region(self):
         """
         RegionService.current should return a valid region
@@ -63,6 +63,7 @@ class CloudRegionServiceTestCase(ProviderTestBase):
         self.assertIsInstance(current_region, Region)
         self.assertTrue(current_region in self.provider.compute.regions.list())
 
+    @helpers.skipIfNoService(['compute.regions'])
     def test_zones(self):
         """
         Test whether regions return the correct zone information

test/test_security_service.py

@@ -1,20 +1,19 @@
 """Test cloudbridge.security modules."""
 import json
-from test.helpers import ProviderTestBase
-import time
+import unittest
 import uuid
 
-import test.helpers as helpers
+from test import helpers
+from test.helpers import ProviderTestBase
 
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
-class CloudSecurityServiceTestCase(ProviderTestBase):
 
-    def __init__(self, methodName, provider):
-        super(CloudSecurityServiceTestCase, self).__init__(
-            methodName=methodName, provider=provider)
+class CloudSecurityServiceTestCase(ProviderTestBase):
 
+    @helpers.skipIfNoService(['security.key_pairs'])
     def test_crud_key_pair_service(self):
-        name = 'cbtestkeypairA-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestkeypairA-{0}'.format(uuid.uuid4())
         kp = self.provider.security.key_pairs.create(name=name)
         with helpers.cleanup_action(
             lambda:
@@ -64,8 +63,9 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             no_kp,
             "Found a key pair {0} that should not exist?".format(no_kp))
 
+    @helpers.skipIfNoService(['security.key_pairs'])
     def test_key_pair(self):
-        name = 'cbtestkeypairB-{0}'.format(uuid.uuid4()).lower()
+        name = 'cbtestkeypairB-{0}'.format(uuid.uuid4())
         kp = self.provider.security.key_pairs.create(name=name)
         with helpers.cleanup_action(lambda: kp.delete()):
             kpl = self.provider.security.key_pairs.list()
@@ -100,14 +100,19 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 lambda: self.provider.network.delete(network_id=net.id)):
             self.provider.security.security_groups.delete(group_id=sg.id)
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_crud_security_group_service(self):
-        name = 'cbtestsecuritygroupA-{0}'.format(uuid.uuid4()).lower()
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        #Empty security groups don't exist in GCE. Let's add a dummy rule.
-        sg.add_rule(ip_protocol='tcp', cidr_ip='0.0.0.0/0')
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupA-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             self.assertEqual(name, sg.description)
 
             # test list method
@@ -155,13 +160,20 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             len(no_sg) == 0,
             "Found a bogus security group?!?".format(no_sg))
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group(self):
         """Test for proper creation of a security group."""
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4()).lower()
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupB-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             rule = sg.add_rule(ip_protocol='tcp', from_port=1111, to_port=1111,
                                cidr_ip='0.0.0.0/0')
             found_rule = sg.get_rule(ip_protocol='tcp', from_port=1111,
@@ -195,7 +207,7 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
 #                 sort_keys=True)
 #             self.assertTrue(
 #                 sg.to_json() == json_repr,
-#                 "JSON sec group representation {0} does not match expected {1}"
+#                 "JSON SG representation {0} does not match expected {1}"
 #                 .format(sg.to_json(), json_repr))
 
         sgl = self.provider.security.security_groups.list()
@@ -205,13 +217,26 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
             "Security group {0} should have been deleted but still exists."
             .format(name))
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group_rule_add_twice(self):
         """Test whether adding the same rule twice succeeds."""
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4()).lower()
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        if isinstance(self.provider, TestMockHelperMixin):
+            raise unittest.SkipTest(
+                "Mock provider returns InvalidParameterValue: "
+                "Value security_group is invalid for parameter.")
+
+        name = 'CBTestSecurityGroupC-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             rule = sg.add_rule(ip_protocol='tcp', from_port=1111, to_port=1111,
                                cidr_ip='0.0.0.0/0')
             # attempting to add the same rule twice should succeed
@@ -222,18 +247,25 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 "Expected rule {0} not found in security group: {1}".format(
                     same_rule, sg.rules))
 
+    @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group_group_rule(self):
         """Test for proper creation of a security group rule."""
-        name = 'cbtestsecuritygroupC-{0}'.format(uuid.uuid4()).lower()
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupD-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             self.assertTrue(
                 len(sg.rules) == 0,
                 "Expected no security group group rule. Got {0}."
                 .format(sg.rules))
-            rule = sg.add_rule(src_group=sg, ip_protocol='tcp', from_port=0,
+            rule = sg.add_rule(src_group=sg, ip_protocol='tcp', from_port=1,
                                to_port=65535)
             self.assertTrue(
                 rule.group.name == name,

tox.ini

@@ -1,14 +1,31 @@
 # Tox (http://tox.testrun.org/) is a tool for running tests
 # in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
+# test suite on all supported python versions and providers.
+# To use it, "pip install tox" and then run "tox" from this directory.
+# You will have to set all required environment variables (below) before
+# running the tests.
+#
+# Alternatively, to run mock tests only, run tox as follows:
+# CB_USE_MOCK_PROVIDERS=True tox -e py27-aws
+#
+# Simply running tox -e py27-aws also works, because the default is to use
+# mock providers.
 
 [tox]
-envlist = py27, py35, pypy
+envlist = {py27,py36,pypy}-{aws,openstack,gce}
 
 [testenv]
-commands = {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test
-passenv = AWS_ACCESS_KEY AWS_SECRET_KEY GCE_CLIENT_EMAIL GCE_PROJECT_NAME GCE_DEFAULT_ZONE GCE_SERVICE_CREDS_FILE OS_AUTH_URL OS_PASSWORD OS_TENANT_NAME OS_USERNAME OS_REGION_NAME NOVA_SERVICE_NAME CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS CB_TEST_PROVIDER CB_USE_MOCK_PROVIDERS
+commands = flake8 cloudbridge test setup.py 
+    {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test {posargs}
+setenv =
+    aws: CB_TEST_PROVIDER=aws
+    openstack: CB_TEST_PROVIDER=openstack
+    gce: CB_TEST_PROVIDER=gce
+passenv =
+    CB_USE_MOCK_PROVIDERS
+    aws: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY
+    openstack:  CB_IMAGE_OS CB_INSTANCE_TYPE_OS CB_PLACEMENT_OS OS_AUTH_URL OS_PASSWORD OS_PROJECT_NAME OS_TENANT_NAME OS_USERNAME OS_REGION_NAME OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME NOVA_SERVICE_NAME
+    gce: CB_IMAGE_GCE CB_INSTANCE_TYPE_GCE CB_PLACEMENT_GCE GCE_CLIENT_EMAIL GCE_PROJECT_NAME GCE_DEFAULT_ZONE GCE_SERVICE_CREDS_FILE
 deps =
     -rrequirements.txt
-    coverage
+    coverage