Merged in latest from master

CHANGELOG.rst

@@ -1,3 +1,29 @@
+0.3.2 - June 10, 2017. (sha f07f3cbd758a0872b847b5537d9073c90f87c24d)
+-------
+
+* Patch release to support files>5GB with OpenStack (thanks @MartinPaulo)
+* Misc bug fixes
+
+0.3.1 - April 18, 2017. (sha f36a462e886d8444cb2818f6573677ecf0565315)
+-------
+
+* Patch for binary file handling in openstack
+
+0.3.0 - April 11, 2017. (sha 13539ccda9e4809082796574d18b1b9bb3f2c624)
+-------
+
+* Reworked test framework to rely on tox's test generation features. This
+  allows for individual test cases to be run on a per provider basis.
+* Added more OpenStack swift config options (OS_AUTH_TOKEN and OS_STORAGE_URL)
+* Added supports for accessing EC2 containers with restricted permissions.
+* Removed exists() method from object store interface. Use get()==None check
+  instead.
+* New method (img.min_disk) for geting size of machine image.
+* Test improvements (flake8 during build, more tests)
+* Misc bug fixes and improvements
+* Changed library to beta state
+* General documentation updates (testing, release process)
+
 0.2.0 - March 23, 2017. (sha a442d96b829ea2c721728520b01981fa61774625)
 -------
 

cloudbridge/__init__.py

@@ -2,7 +2,7 @@
 import logging
 
 # Current version of the library
-__version__ = '0.2.0'
+__version__ = '0.3.2'
 
 
 def get_version():
@@ -32,6 +32,7 @@ class NullHandler(logging.Handler):
         """Don't emit a log."""
         pass
 
+
 TRACE = 5  # Lower than debug which is 10
 
 
@@ -46,13 +47,13 @@ class CBLogger(logging.Logger):
         """Add ``trace`` log level."""
         self.log(TRACE, msg, *args, **kwargs)
 
+
 # By default, do not force any logging by the library. If you want to see the
 # log messages in your scripts, add the following to the top of your script:
 #   import cloudbridge
 #   cloudbridge.set_stream_logger(__name__)
 #   OR
 #   cloudbridge.set_file_logger(__name__, '/tmp/cb.log')
-
 default_format_string = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
 logging.setLoggerClass(CBLogger)
 logging.addLevelName(TRACE, "TRACE")

cloudbridge/cloud/base/provider.py

@@ -1,15 +1,16 @@
 """Base implementation of a provider interface."""
+import functools
 import os
+from os.path import expanduser
 try:
     from configparser import SafeConfigParser
 except ImportError:  # Python 2
     from ConfigParser import SafeConfigParser
-from os.path import expanduser
-import functools
 
 from cloudbridge.cloud.interfaces import CloudProvider
-from cloudbridge.cloud.interfaces.resources import Configuration
 from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
+from cloudbridge.cloud.interfaces.resources import Configuration
+
 
 DEFAULT_RESULT_LIMIT = 50
 DEFAULT_WAIT_TIMEOUT = 600

cloudbridge/cloud/base/resources.py

@@ -9,14 +9,14 @@ import os
 import shutil
 import time
 
-import six
-
 from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
+from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 from cloudbridge.cloud.interfaces.resources import AttachmentInfo
 from cloudbridge.cloud.interfaces.resources import Bucket
 from cloudbridge.cloud.interfaces.resources import BucketObject
 from cloudbridge.cloud.interfaces.resources import CloudResource
+from cloudbridge.cloud.interfaces.resources import FloatingIP
 from cloudbridge.cloud.interfaces.resources import Instance
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import InstanceType
@@ -30,18 +30,17 @@ from cloudbridge.cloud.interfaces.resources import ObjectLifeCycleMixin
 from cloudbridge.cloud.interfaces.resources import PageableObjectMixin
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import Region
-from cloudbridge.cloud.interfaces.resources import Router
 from cloudbridge.cloud.interfaces.resources import ResultList
+from cloudbridge.cloud.interfaces.resources import Router
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SecurityGroupRule
 from cloudbridge.cloud.interfaces.resources import Snapshot
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import Subnet
-from cloudbridge.cloud.interfaces.resources import FloatingIP
 from cloudbridge.cloud.interfaces.resources import Volume
 from cloudbridge.cloud.interfaces.resources import VolumeState
-from cloudbridge.cloud.interfaces.exceptions import WaitStateException
 
+import six
 
 log = logging.getLogger(__name__)
 

cloudbridge/cloud/base/services.py

@@ -16,6 +16,7 @@ from cloudbridge.cloud.interfaces.services import SecurityService
 from cloudbridge.cloud.interfaces.services import SnapshotService
 from cloudbridge.cloud.interfaces.services import SubnetService
 from cloudbridge.cloud.interfaces.services import VolumeService
+
 from .resources import BasePageableObjectMixin
 
 

cloudbridge/cloud/factory.py

@@ -1,11 +1,12 @@
-from cloudbridge.cloud import providers
-from cloudbridge.cloud.interfaces import CloudProvider
-from cloudbridge.cloud.interfaces import TestMockHelperMixin
-from collections import defaultdict
 import importlib
 import inspect
 import logging
 import pkgutil
+from collections import defaultdict
+
+from cloudbridge.cloud import providers
+from cloudbridge.cloud.interfaces import CloudProvider
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
 
 log = logging.getLogger(__name__)

cloudbridge/cloud/interfaces/resources.py

@@ -580,6 +580,26 @@ class Instance(ObjectLifeCycleMixin, CloudResource):
         """
         pass
 
+    @abstractmethod
+    def add_security_group(self, sg):
+        """
+        Add a security group to this instance
+
+        :type sg: ``SecurityGroup``
+        :param sg: The SecurityGroup to associate with the instance.
+        """
+        pass
+
+    @abstractmethod
+    def remove_security_group(self, sg):
+        """
+        Remove a security group from this instance
+
+        :type sg: ``SecurityGroup``
+        :param sg: The SecurityGroup to associate with the instance.
+        """
+        pass
+
 
 class MachineImageState(object):
 
@@ -742,6 +762,17 @@ class MachineImage(ObjectLifeCycleMixin, CloudResource):
         """
         pass
 
+    @abstractproperty
+    def min_disk(self):
+        """
+        Returns the minimum size of the disk that's required to
+        boot this image (in GB)
+
+        :rtype: ``int``
+        :return: The minimum disk size needed by this image
+        """
+        pass
+
     @abstractmethod
     def delete(self):
         """
@@ -2167,4 +2198,3 @@ class Bucket(PageableObjectMixin, CloudResource):
         :return: The newly created bucket object
         """
         pass
-

cloudbridge/cloud/providers/aws/provider.py

@@ -1,6 +1,4 @@
-"""
-Provider implementation based on boto library for AWS-compatible clouds.
-"""
+"""Provider implementation based on boto library for AWS-compatible clouds."""
 
 import os
 
@@ -9,8 +7,8 @@ from boto.ec2.regioninfo import RegionInfo
 try:
     # These are installed only for the case of a dev instance
     from httpretty import HTTPretty
-    from moto.ec2 import mock_ec2
-    from moto.s3 import mock_s3
+    from moto import mock_ec2
+    from moto import mock_s3
 except ImportError:
     # TODO: Once library logging is configured, change this
     print("[aws provider] moto library not available!")
@@ -28,6 +26,8 @@ from .services import AWSSecurityService
 class AWSCloudProvider(BaseCloudProvider):
 
     PROVIDER_ID = 'aws'
+    AWS_INSTANCE_DATA_DEFAULT_URL = "https://d168wakzal7fp0.cloudfront.net/" \
+                                    "aws_instance_data.json"
 
     def __init__(self, config):
         super(AWSCloudProvider, self).__init__(config)
@@ -173,8 +173,8 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
         self.s3mock = mock_s3()
         self.s3mock.start()
         HTTPretty.register_uri(
-            method="GET",
-            uri="https://d168wakzal7fp0.cloudfront.net/aws_instance_data.json",
+            HTTPretty.GET,
+            self.AWS_INSTANCE_DATA_DEFAULT_URL,
             body=u"""
 [
   {

cloudbridge/cloud/providers/aws/resources.py

@@ -1,9 +1,19 @@
 """
 DataTypes used by this provider
 """
+import hashlib
+import inspect
+import json
+
+from datetime import datetime
+
+from boto.exception import EC2ResponseError
+from boto.s3.key import Key
+
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
 from cloudbridge.cloud.base.resources import BaseBucket
 from cloudbridge.cloud.base.resources import BaseBucketObject
+from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseInstance
 from cloudbridge.cloud.base.resources import BaseInstanceType
 from cloudbridge.cloud.base.resources import BaseKeyPair
@@ -17,25 +27,16 @@ from cloudbridge.cloud.base.resources import BaseSecurityGroup
 from cloudbridge.cloud.base.resources import BaseSecurityGroupRule
 from cloudbridge.cloud.base.resources import BaseSnapshot
 from cloudbridge.cloud.base.resources import BaseSubnet
-from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseVolume
 from cloudbridge.cloud.base.resources import ClientPagedResultList
-from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import MachineImageState
 from cloudbridge.cloud.interfaces.resources import NetworkState
 from cloudbridge.cloud.interfaces.resources import RouterState
+from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
 
-from datetime import datetime
-import hashlib
-import inspect
-import json
-
-from boto.exception import EC2ResponseError
-from boto.s3.key import Key
-
 from retrying import retry
 
 
@@ -85,6 +86,18 @@ class AWSMachineImage(BaseMachineImage):
         """
         return self._ec2_image.description
 
+    @property
+    def min_disk(self):
+        """
+        Returns the minimum size of the disk that's required to
+        boot this image (in GB)
+
+        :rtype: ``int``
+        :return: The minimum disk size needed by this image
+        """
+        bdm = self._ec2_image.block_device_mapping
+        return bdm[self._ec2_image.root_device_name].size
+
     def delete(self):
         """
         Delete this image
@@ -344,7 +357,7 @@ class AWSInstance(BaseInstance):
         """
         if self._ec2_instance.vpc_id:
             aid = self._provider._vpc_conn.get_all_addresses([ip_address])[0]
-            return self._provider._ec2_conn.associate_address(
+            return self._provider.ec2_conn.associate_address(
                 self._ec2_instance.id, allocation_id=aid.allocation_id)
         else:
             return self._ec2_instance.use_ip(ip_address)
@@ -353,8 +366,28 @@ class AWSInstance(BaseInstance):
         """
         Remove a elastic IP address from this instance.
         """
-        raise NotImplementedError(
-            'remove_floating_ip not implemented by this provider.')
+        ip_addr = self._provider._vpc_conn.get_all_addresses([ip_address])[0]
+        if self._ec2_instance.vpc_id:
+            return self._provider.ec2_conn.disassociate_address(
+                association_id=ip_addr.association_id)
+        else:
+            return self._provider.ec2_conn.disassociate_address(
+                public_ip=ip_addr.public_ip)
+
+    def add_security_group(self, sg):
+        """
+        Add a security group to this instance
+        """
+        self._ec2_instance.modify_attribute(
+            'groupSet', [g.id for g in self._ec2_instance.groups] + [sg.id])
+
+    def remove_security_group(self, sg):
+        """
+        Remove a security group from this instance
+        """
+        self._ec2_instance.modify_attribute(
+            'groupSet', [g.id for g in self._ec2_instance.groups
+                         if g.id != sg.id])
 
     @property
     def state(self):

cloudbridge/cloud/providers/aws/services.py

@@ -1,6 +1,6 @@
 """Services implemented by the AWS provider."""
-import time
 import string
+import time
 
 from boto.ec2.blockdevicemapping import BlockDeviceMapping
 from boto.ec2.blockdevicemapping import BlockDeviceType
@@ -22,9 +22,9 @@ from cloudbridge.cloud.base.services import BaseSecurityService
 from cloudbridge.cloud.base.services import BaseSnapshotService
 from cloudbridge.cloud.base.services import BaseSubnetService
 from cloudbridge.cloud.base.services import BaseVolumeService
-from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
+from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
 from cloudbridge.cloud.interfaces.resources import PlacementZone
@@ -644,9 +644,6 @@ class AWSInstanceService(BaseInstanceService):
                                      reservations.next_token,
                                      False, data=instances)
 
-AWS_INSTANCE_DATA_DEFAULT_URL = "https://d168wakzal7fp0.cloudfront.net/" \
-                                "aws_instance_data.json"
-
 
 class AWSInstanceTypesService(BaseInstanceTypesService):
 
@@ -659,8 +656,8 @@ class AWSInstanceTypesService(BaseInstanceTypesService):
         Fetch info about the available instances.
 
         To update this information, update the file pointed to by the
-        ``AWS_INSTANCE_DATA_DEFAULT_URL`` above. The content for this file
-        should be obtained from this repo
+        ``provider.AWS_INSTANCE_DATA_DEFAULT_URL`` above. The content for this
+        file should be obtained from this repo:
         https://github.com/powdahound/ec2instances.info (in particular, this
         file: https://raw.githubusercontent.com/powdahound/ec2instances.info/
         master/www/instances.json).
@@ -668,7 +665,8 @@ class AWSInstanceTypesService(BaseInstanceTypesService):
         TODO: Needs a caching function with timeout
         """
         r = requests.get(self.provider.config.get(
-            "aws_instance_info_url", AWS_INSTANCE_DATA_DEFAULT_URL))
+            "aws_instance_info_url",
+            self.provider.AWS_INSTANCE_DATA_DEFAULT_URL))
         return r.json()
 
     def list(self, limit=None, marker=None):

cloudbridge/cloud/providers/openstack/helpers.py

@@ -2,6 +2,7 @@
 Helper functions
 """
 import itertools
+
 from cloudbridge.cloud.base.resources import ServerPagedResultList
 
 

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,16 +1,23 @@
 """Provider implementation based on OpenStack Python clients for OpenStack."""
 
+import inspect
+
 import os
 
 from cinderclient import client as cinder_client
+
+from cloudbridge.cloud.base import BaseCloudProvider
+
 from keystoneauth1 import session
+
 from keystoneclient import client as keystone_client
+
 from neutronclient.v2_0 import client as neutron_client
+
 from novaclient import client as nova_client
 from novaclient import shell as nova_shell
-from swiftclient import client as swift_client
 
-from cloudbridge.cloud.base import BaseCloudProvider
+from swiftclient import client as swift_client
 
 from .services import OpenStackBlockStoreService
 from .services import OpenStackComputeService
@@ -231,19 +238,66 @@ class OpenStackCloudProvider(BaseCloudProvider):
 #         return glance_client.Client(version=api_version,
 #                                     session=self.keystone.session)
 
-    def _connect_swift(self):
+    @staticmethod
+    def _clean_options(options, method_to_match):
+        """
+        Returns a **copy** of the source options with all keys that are not in
+        the ``method_to_match`` parameter list removed.
+
+        .. note:: If ``options`` has the ``os_options`` key it will have
+            both the key and its value removed. This is because any entries
+            in this dictionary value will override our settings. This
+            situation is only going to happen when the `_connect_swift`
+            method is called by the SwiftService to manufacture new
+            connections.
+
+        .. seealso::
+            https://docs.openstack.org/developer/python-swiftclient/swiftclient.html#module-swiftclient.client
+
+        :param options: The source options.
+        :type options: ``dict``
+        :param method_to_match: The method whose signature is to be matched
+        :type method_to_match: A callable
+        :return: A copy of the source options with all keys that are not in the
+            ``method_to_match`` parameter list removed. If options is ``None``
+            then this will be an empty dictionary
+        :rtype: ``dict``
+        """
+        result = {}
+        if options:
+            try:
+                method_signature = inspect.signature(method_to_match)
+                parameters = set(method_signature.parameters.keys())
+            except AttributeError:
+                parameters = set(inspect.getargspec(method_to_match).args)
+            result = {key: val for key, val in options.items() if
+                      key in parameters}
+            # Don't allow the options to override our authentication
+            result.pop('os_options', None)
+        return result
+
+    def _connect_swift(self, options=None):
+        """
+        Get an OpenStack Swift (object store) client connection.
+
+        :param options: A dictionary of options from which values will be
+            passed to the connection.
+        :return: A Swift client connection using the auth credentials held by
+            the OpenStackCloudProvider instance
+        """
+        clean_options = self._clean_options(options,
+                                            swift_client.Connection.__init__)
         storage_url = self._get_config_value(
             'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
         auth_token = self._get_config_value(
             'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
-
-        """Get an OpenStack Swift (object store) client object cloud."""
         if storage_url and auth_token:
-            return swift_client.Connection(preauthurl=storage_url,
-                                           preauthtoken=auth_token)
+            clean_options['preauthurl'] = storage_url
+            clean_options['preauthtoken'] = auth_token
         else:
-            return swift_client.Connection(authurl=self.auth_url,
-                                           session=self._keystone_session)
+            clean_options['authurl'] = self.auth_url
+            clean_options['session'] = self._keystone_session
+        return swift_client.Connection(**clean_options)
 
     def _connect_neutron(self):
         """Get an OpenStack Neutron (networking) client object cloud."""

cloudbridge/cloud/providers/openstack/resources.py

@@ -1,9 +1,16 @@
 """
 DataTypes used by this provider
 """
+import inspect
+import ipaddress
+import json
+
+import os
+
 from cloudbridge.cloud.base.resources import BaseAttachmentInfo
 from cloudbridge.cloud.base.resources import BaseBucket
 from cloudbridge.cloud.base.resources import BaseBucketObject
+from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseInstance
 from cloudbridge.cloud.base.resources import BaseInstanceType
 from cloudbridge.cloud.base.resources import BaseKeyPair
@@ -16,27 +23,26 @@ from cloudbridge.cloud.base.resources import BaseSecurityGroup
 from cloudbridge.cloud.base.resources import BaseSecurityGroupRule
 from cloudbridge.cloud.base.resources import BaseSnapshot
 from cloudbridge.cloud.base.resources import BaseSubnet
-from cloudbridge.cloud.base.resources import BaseFloatingIP
 from cloudbridge.cloud.base.resources import BaseVolume
 from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import MachineImageState
 from cloudbridge.cloud.interfaces.resources import NetworkState
 from cloudbridge.cloud.interfaces.resources import RouterState
+from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
-from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.providers.openstack import helpers as oshelpers
 
-import inspect
-import json
-
-import ipaddress
-
 from keystoneclient.v3.regions import Region
 
 import novaclient.exceptions as novaex
 
-import swiftclient.exceptions as swiftex
+import swiftclient
+
+from swiftclient.service import SwiftService, SwiftUploadObject
+
+ONE_GIG = 1048576000  # in bytes
+FIVE_GIG = ONE_GIG * 5  # in bytes
 
 
 class OpenStackMachineImage(BaseMachineImage):
@@ -80,6 +86,17 @@ class OpenStackMachineImage(BaseMachineImage):
         """
         return None
 
+    @property
+    def min_disk(self):
+        """
+        Returns the minimum size of the disk that's required to
+        boot this image (in GB)
+
+        :rtype: ``int``
+        :return: The minimum disk size needed by this image
+        """
+        return self._os_image.minDisk
+
     def delete(self):
         """
         Delete this image
@@ -363,6 +380,18 @@ class OpenStackInstance(BaseInstance):
         """
         self._os_instance.remove_floating_ip(ip_address)
 
+    def add_security_group(self, sg):
+        """
+        Add a security group to this instance
+        """
+        self._os_instance.add_security_group(sg.id)
+
+    def remove_security_group(self, sg):
+        """
+        Remove a security group from this instance
+        """
+        self._os_instance.remove_security_group(sg.id)
+
     @property
     def state(self):
         return OpenStackInstance.INSTANCE_STATE_MAP.get(
@@ -672,6 +701,7 @@ class OpenStackNetwork(BaseNetwork):
 
     @property
     def state(self):
+        self.refresh()
         return OpenStackNetwork._NETWORK_STATE_MAP.get(
             self._network.get('status', None),
             NetworkState.UNKNOWN)
@@ -702,11 +732,9 @@ class OpenStackNetwork(BaseNetwork):
         return OpenStackSubnet(self._provider, subnet)
 
     def refresh(self):
-        """
-        Refreshes the state of this network by re-querying the cloud provider
-        for its latest state.
-        """
-        return self.state
+        """Refresh the state of this network by re-querying the provider."""
+        net = self._provider.neutron.list_networks(id=self.id).get('networks')
+        self._network = net[0] if net else {}
 
 
 class OpenStackSubnet(BaseSubnet):
@@ -767,7 +795,7 @@ class OpenStackFloatingIP(BaseFloatingIP):
         return self._ip.get('fixed_ip_address', None)
 
     def in_use(self):
-        return True if self._ip.get('status', None) == 'ACTIVE' else False
+        return bool(self._ip.get('port_id', None))
 
     def delete(self):
         self._provider.neutron.delete_floatingip(self.id)
@@ -1055,16 +1083,47 @@ class OpenStackBucketObject(BaseBucketObject):
         """
         Set the contents of this object to the data read from the source
         string.
+
+        .. warning:: Will fail if the data is larger than 5 Gig.
         """
         self._provider.swift.put_object(self.cbcontainer.name, self.name,
                                         data)
 
     def upload_from_file(self, path):
         """
-        Stores the contents of the file pointed by the "path" variable.
-        """
-        with open(path, 'r') as f:
-            self.upload(f.read())
+        Stores the contents of the file pointed by the ``path`` variable.
+        If the file is bigger than 5 Gig, it will be broken into segments.
+
+        :type path: ``str``
+        :param path: Absolute path to the file to be uploaded to Swift.
+        :rtype: ``bool``
+        :return: ``True`` if successful, ``False`` if not.
+
+        .. note::
+            * The size of the segments chosen (or any of the other upload
+              options) is not under user control.
+            * If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
+
+        .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
+        """
+        upload_options = {}
+        if 'segment_size' not in upload_options:
+            if os.path.getsize(path) >= FIVE_GIG:
+                upload_options['segment_size'] = FIVE_GIG
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            upload_object = SwiftUploadObject(path, object_name=self.name)
+            for up_res in swift.upload(self.cbcontainer.name,
+                                       [upload_object, ],
+                                       options=upload_options):
+                result = result and up_res['success']
+        return result
 
     def delete(self):
         """
@@ -1072,14 +1131,20 @@ class OpenStackBucketObject(BaseBucketObject):
 
         :rtype: ``bool``
         :return: True if successful
+
+        .. note:: If called this method will remap the
+              ``swiftclient.service.get_conn`` factory method to
+              ``self._provider._connect_swift``
         """
-        try:
-            self._provider.swift.delete_object(self.cbcontainer.name,
-                                               self.name)
-        except swiftex.ClientException as err:
-            if err.http_status == 404:
-                return True
-        return False
+
+        # remap the swift service's connection factory method
+        swiftclient.service.get_conn = self._provider._connect_swift
+
+        result = True
+        with SwiftService() as swift:
+            for del_res in swift.delete(self.cbcontainer.name, [self.name, ]):
+                result = result and del_res['success']
+        return result
 
     def generate_url(self, expires_in=0):
         """

cloudbridge/cloud/providers/openstack/services.py

@@ -33,10 +33,10 @@ from cloudbridge.cloud.interfaces.resources import Subnet
 from cloudbridge.cloud.interfaces.resources import Volume
 from cloudbridge.cloud.providers.openstack import helpers as oshelpers
 
-from novaclient.exceptions import NotFound as NovaNotFound
-
 from neutronclient.common.exceptions import NeutronClientException
 
+from novaclient.exceptions import NotFound as NovaNotFound
+
 from .resources import OpenStackBucket
 from .resources import OpenStackFloatingIP
 from .resources import OpenStackInstance

docs/topics/contributor_guide.rst

@@ -9,5 +9,6 @@ CloudBridge Provider.
 
     Design Goals <design_goals.rst>
     Testing <testing.rst>
-    Provider development walkthrough <provider_development.rst>
+    Provider Development Walkthrough <provider_development.rst>
+    Release Process <release_process.rst>
 

docs/topics/release_process.rst

@@ -0,0 +1,21 @@
+Release Process
+~~~~~~~~~~~~~~~
+
+1. Increment version number in cloudbridge/__init__.py as per semver rules.
+
+2. Freeze all library dependencies in setup.py. The version numbers can be a range
+   with the upper limit being the latest known working version, and the lowest being
+   the last known working version. 
+
+3. Run all tox tests.
+
+4. Add release notes to CHANGELOG.rst. Also add last commit hash to changelog.
+
+5. Release to PyPi
+
+.. code-block:: bash
+
+   python setup.py sdist upload
+   python setup.py bdist_wheel upload
+
+6. Tag release and make github release.

setup.py

@@ -1,7 +1,9 @@
+"""Library install script for setuptools."""
 import ast
 import os
 import re
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 # Cannot use "from cloudbridge import get_version" because that would try to
 # import the six package which may not be installed yet.
@@ -14,16 +16,20 @@ with open(os.path.join('cloudbridge', '__init__.py')) as f:
             break
 
 base_reqs = ['bunch>=1.0.1', 'six>=1.10.0', 'retrying>=1.3.3']
-openstack_reqs = ['python-novaclient>=7.0.0',
-                  'python-glanceclient>=2.5.0',
-                  'python-cinderclient>=1.9.0',
-                  'python-swiftclient>=3.2.0',
-                  'python-neutronclient>=6.0.0',
-                  'python-keystoneclient>=3.8.0']
-aws_reqs = ['boto>=2.38.0']
+openstack_reqs = ['requests<2.13.0',
+                  'Babel>=2.3.4,<2.4.0',
+                  'python-novaclient==7.0.0',
+                  'python-glanceclient>=2.5.0,<=2.6.0',
+                  'python-cinderclient>=1.9.0,<=2.0.1',
+                  'python-swiftclient>=3.2.0,<=3.3.0',
+                  'python-neutronclient>=6.0.0,<=6.1.0',
+                  'python-keystoneclient>=3.8.0,<=3.10.0']
+aws_reqs = ['boto>=2.38.0,<=2.46.1']
 gce_reqs = ['google-api-python-client>=1.4.2', "cryptography>=1.4"]
 full_reqs = base_reqs + aws_reqs + openstack_reqs + gce_reqs
-dev_reqs = (['tox>=2.1.1', 'moto>=0.4.18', 'sphinx>=1.3.1'] + full_reqs)
+# httpretty is required with/for moto 1.0.0 or AWS tests fail
+dev_reqs = (['tox>=2.1.1', 'moto<1.0.0', 'sphinx>=1.3.1', 'flake8>=3.3.0',
+             'flake8-import-order>=0.12', 'httpretty==0.8.10'] + full_reqs)
 
 setup(name='cloudbridge',
       version=version,

test/helpers.py

@@ -1,14 +1,16 @@
-from contextlib import contextmanager
+import functools
 import os
 import sys
 import unittest
-import functools
-from six import reraise
+
+from contextlib import contextmanager
 
 from cloudbridge.cloud.factory import CloudProviderFactory
 from cloudbridge.cloud.interfaces import InstanceState
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
+from six import reraise
+
 
 def parse_bool(val):
     if val:
@@ -122,14 +124,14 @@ def delete_test_network(network):
 
 
 def create_test_instance(
-        provider, instance_name, subnet, zone=None, launch_config=None,
+        provider, instance_name, subnet, launch_config=None,
         key_pair=None, security_groups=None):
     return provider.compute.instances.create(
         instance_name,
         get_provider_test_data(provider, 'image'),
         get_provider_test_data(provider, 'instance_type'),
         subnet=subnet,
-        zone=zone,
+        zone=get_provider_test_data(provider, 'placement'),
         key_pair=key_pair,
         security_groups=security_groups,
         launch_config=launch_config)
@@ -149,17 +151,26 @@ def get_test_instance(provider, name, key_pair=None, security_groups=None,
     return instance
 
 
+def get_test_fixtures_folder():
+    return os.path.join(os.path.dirname(__file__), 'fixtures/')
+
+
+def delete_test_instance(instance):
+    if instance:
+        instance.terminate()
+        instance.wait_for([InstanceState.TERMINATED, InstanceState.UNKNOWN],
+                          terminal_states=[InstanceState.ERROR])
+
+
 def cleanup_test_resources(instance=None, network=None, security_group=None,
                            key_pair=None):
+    """Clean up any combination of supplied resources."""
     with cleanup_action(lambda: delete_test_network(network)
                         if network else None):
         with cleanup_action(lambda: key_pair.delete() if key_pair else None):
             with cleanup_action(lambda: security_group.delete()
                                 if security_group else None):
-                instance.terminate()
-                instance.wait_for(
-                    [InstanceState.TERMINATED, InstanceState.UNKNOWN],
-                    terminal_states=[InstanceState.ERROR])
+                delete_test_instance(instance)
 
 
 class ProviderTestBase(unittest.TestCase):

test/test_block_store_service.py

@@ -1,14 +1,14 @@
 import time
 import uuid
 
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces import SnapshotState
 from cloudbridge.cloud.interfaces import VolumeState
 from cloudbridge.cloud.interfaces.resources import AttachmentInfo
 
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+import six
 
 
 class CloudBlockStoreServiceTestCase(ProviderTestBase):
@@ -98,11 +98,16 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
         instance_name = "CBVolOps-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, subnet = helpers.create_test_network(self.provider, instance_name)
-        test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  subnet=subnet)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        test_instance = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
+            net, subnet = helpers.create_test_network(
+                self.provider, instance_name)
+            test_instance = helpers.get_test_instance(
+                self.provider, instance_name, subnet=subnet)
             name = "CBUnitTestAttachVol-{0}".format(uuid.uuid4())
             test_vol = self.provider.block_store.volumes.create(
                 name, 1, test_instance.zone_id)
@@ -126,11 +131,17 @@ class CloudBlockStoreServiceTestCase(ProviderTestBase):
             self.provider.name,
             uuid.uuid4())
         vol_desc = 'newvoldesc1'
-        net, subnet = helpers.create_test_network(self.provider, instance_name)
-        test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  subnet=subnet)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_instance = None
+        net = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
+            net, subnet = helpers.create_test_network(
+                self.provider, instance_name)
+            test_instance = helpers.get_test_instance(
+                self.provider, instance_name, subnet=subnet)
+
             name = "CBUnitTestVolProps-{0}".format(uuid.uuid4())
             test_vol = self.provider.block_store.volumes.create(
                 name, 1, test_instance.zone_id, description=vol_desc)

test/test_cloud_factory.py

@@ -1,5 +1,7 @@
 import unittest
 
+from test import helpers
+
 from cloudbridge.cloud import factory
 from cloudbridge.cloud import interfaces
 from cloudbridge.cloud.factory import CloudProviderFactory
@@ -7,7 +9,6 @@ from cloudbridge.cloud.interfaces import TestMockHelperMixin
 from cloudbridge.cloud.interfaces.provider import CloudProvider
 from cloudbridge.cloud.providers.aws import AWSCloudProvider
 from cloudbridge.cloud.providers.aws.provider import MockAWSCloudProvider
-import test.helpers as helpers
 
 
 class CloudFactoryTestCase(unittest.TestCase):

test/test_cloud_helpers.py

@@ -1,8 +1,9 @@
 import itertools
 
+from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.base.resources import ClientPagedResultList
 from cloudbridge.cloud.base.resources import ServerPagedResultList
-from test.helpers import ProviderTestBase
 
 
 class DummyResult(object):
@@ -71,4 +72,4 @@ class CloudHelpersTestCase(ProviderTestBase):
         self.assertTrue(results.supports_server_paging, "Server paged result"
                         " lists should return True for server paging.")
         with self.assertRaises(NotImplementedError):
-            _ = results.data
+            results.data

test/test_compute_service.py

@@ -1,15 +1,17 @@
 import ipaddress
 import uuid
 
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
-from cloudbridge.cloud.interfaces import InvalidConfigurationException
 from cloudbridge.cloud.interfaces import InstanceState
-from cloudbridge.cloud.interfaces.resources import InstanceType
+from cloudbridge.cloud.interfaces import InvalidConfigurationException
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 from cloudbridge.cloud.interfaces.exceptions import WaitStateException
+from cloudbridge.cloud.interfaces.resources import InstanceType
+# from cloudbridge.cloud.interfaces.resources import SnapshotState
 
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+import six
 
 
 class CloudComputeServiceTestCase(ProviderTestBase):
@@ -19,11 +21,16 @@ class CloudComputeServiceTestCase(ProviderTestBase):
         name = "CBInstCrud-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, subnet = helpers.create_test_network(self.provider, name)
-        inst = helpers.get_test_instance(self.provider, name, subnet=subnet)
-
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        inst = None
+        net = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 inst, net)):
+            net, subnet = helpers.create_test_network(self.provider, name)
+            inst = helpers.get_test_instance(self.provider, name,
+                                             subnet=subnet)
+
             all_instances = self.provider.compute.instances.list()
 
             list_instances = [i for i in all_instances if i.name == name]
@@ -93,17 +100,24 @@ class CloudComputeServiceTestCase(ProviderTestBase):
         name = "CBInstProps-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, subnet = helpers.create_test_network(self.provider, name)
-        kp = self.provider.security.key_pairs.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        test_instance = helpers.get_test_instance(self.provider,
-                                                  name, key_pair=kp,
-                                                  security_groups=[sg],
-                                                  subnet=subnet)
 
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_instance = None
+        net = None
+        sg = None
+        kp = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net, sg, kp)):
+            net, subnet = helpers.create_test_network(self.provider, name)
+            kp = self.provider.security.key_pairs.create(name=name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
+            test_instance = helpers.get_test_instance(self.provider,
+                                                      name, key_pair=kp,
+                                                      security_groups=[sg],
+                                                      subnet=subnet)
+
             self.assertTrue(
                 test_instance.id in repr(test_instance),
                 "repr(obj) should contain the object id so that the object"
@@ -196,12 +210,12 @@ class CloudComputeServiceTestCase(ProviderTestBase):
         # Override root volume size
         image_id = helpers.get_provider_test_data(self.provider, "image")
         img = self.provider.compute.images.get(image_id)
+        # The size should be greater then the ami size
+        # and therefore, img.min_disk is used.
         lc.add_volume_device(
             is_root=True,
             source=img,
-            # TODO: This should be greater than the ami size or tests will fail
-            # on actual infrastructure. Needs an image.size method
-            size=2,
+            size=img.min_disk if img and img.min_disk else 2,
             delete_on_terminate=True)
 
         # Attempting to add more than one root volume should raise an
@@ -237,81 +251,156 @@ class CloudComputeServiceTestCase(ProviderTestBase):
             self.provider.name,
             uuid.uuid4())
 
-#         test_vol = self.provider.block_store.volumes.create(
-#             name,
-#             1,
-#             helpers.get_provider_test_data(self.provider, "placement"))
-#         with helpers.cleanup_action(lambda: test_vol.delete()):
-#             test_vol.wait_till_ready()
-#             test_snap = test_vol.create_snapshot(name=name,
-#                                                  description=name)
-#
-#             def cleanup_snap(snap):
-#                 snap.delete()
-#                 snap.wait_for(
-#                     [SnapshotState.UNKNOWN],
-#                     terminal_states=[SnapshotState.ERROR])
+        # Comment out BDM tests because OpenStack is not stable enough yet
+        if True:
+            if True:
+
+                # test_vol = self.provider.block_store.volumes.create(
+                #    name,
+                #    1,
+                #    helpers.get_provider_test_data(self.provider,
+                #                                   "placement"))
+                # with helpers.cleanup_action(lambda: test_vol.delete()):
+                #    test_vol.wait_till_ready()
+                #    test_snap = test_vol.create_snapshot(name=name,
+                #                                         description=name)
+                #
+                #    def cleanup_snap(snap):
+                #        snap.delete()
+                #        snap.wait_for(
+                #            [SnapshotState.UNKNOWN],
+                #            terminal_states=[SnapshotState.ERROR])
+                #
+                #    with helpers.cleanup_action(lambda:
+                #                                cleanup_snap(test_snap)):
+                #         test_snap.wait_till_ready()
+
+                lc = self.provider.compute.instances.create_launch_config()
+
+#                 # Add a new blank volume
+#                 lc.add_volume_device(size=1, delete_on_terminate=True)
 #
-#             with helpers.cleanup_action(lambda: cleanup_snap(test_snap)):
-#                 test_snap.wait_till_ready()
-
-        lc = self.provider.compute.instances.create_launch_config()
-
-        # Add a new blank volume
-#         lc.add_volume_device(size=1, delete_on_terminate=True)
-
-        # Attach an existing volume
+#                 # Attach an existing volume
 #                 lc.add_volume_device(size=1, source=test_vol,
 #                                      delete_on_terminate=True)
-
-        # Add a new volume based on a snapshot
+#
+#                 # Add a new volume based on a snapshot
 #                 lc.add_volume_device(size=1, source=test_snap,
 #                                      delete_on_terminate=True)
 
-        # Override root volume size
-        image_id = helpers.get_provider_test_data(
-            self.provider,
-            "image")
-        img = self.provider.compute.images.get(image_id)
-        lc.add_volume_device(
-            is_root=True,
-            source=img,
-            # TODO: This should be greater than the ami size or tests
-            # will fail on actual infrastructure. Needs an image.size
-            # method
-            size=8,
-            delete_on_terminate=True)
+                # Override root volume size
+                image_id = helpers.get_provider_test_data(
+                    self.provider,
+                    "image")
+                img = self.provider.compute.images.get(image_id)
+                # The size should be greater then the ami size
+                # and therefore, img.min_disk is used.
+                lc.add_volume_device(
+                    is_root=True,
+                    source=img,
+                    size=img.min_disk if img and img.min_disk else 2,
+                    delete_on_terminate=True)
+
+                # Add all available ephemeral devices
+                instance_type_name = helpers.get_provider_test_data(
+                    self.provider,
+                    "instance_type")
+                inst_type = self.provider.compute.instance_types.find(
+                    name=instance_type_name)[0]
+                for _ in range(inst_type.num_ephemeral_disks):
+                    lc.add_ephemeral_device()
+
+                net, subnet = helpers.create_test_network(self.provider, name)
+
+                with helpers.cleanup_action(lambda:
+                                            helpers.delete_test_network(net)):
+
+                    inst = helpers.create_test_instance(
+                        self.provider,
+                        name,
+                        subnet=subnet,
+                        launch_config=lc)
+
+                    with helpers.cleanup_action(lambda:
+                                                helpers.delete_test_instance(
+                                                    inst)):
+                        try:
+                            inst.wait_till_ready()
+                        except WaitStateException as e:
+                            self.fail("The block device mapped launch did not "
+                                      " complete successfully: %s" % e)
+                        # TODO: Check instance attachments and make sure they
+                        # correspond to requested mappings
 
-        # Add all available ephemeral devices
-        instance_type_name = helpers.get_provider_test_data(
-            self.provider,
-            "instance_type")
-        inst_type = self.provider.compute.instance_types.find(
-            name=instance_type_name)[0]
-        for _ in range(inst_type.num_ephemeral_disks):
-            lc.add_ephemeral_device()
+    @helpers.skipIfNoService(['compute.instances', 'network',
+                              'security.security_groups'])
+    def test_instance_methods(self):
+        name = "CBInstProps-{0}-{1}".format(
+            self.provider.name,
+            uuid.uuid4())
 
-        net, subnet = helpers.create_test_network(self.provider, name)
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_inst = None
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                test_inst, net, sg)):
+            net, subnet = helpers.create_test_network(self.provider, name)
+            test_inst = helpers.get_test_instance(self.provider, name,
+                                                  subnet=subnet)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
 
-        inst = helpers.create_test_instance(
-            self.provider,
-            name,
-            subnet=subnet,
-            zone=helpers.get_provider_test_data(self.provider, 'placement'),
-            launch_config=lc)
-
-        def cleanup(instance, net):
-            instance.terminate()
-            instance.wait_for(
-                [InstanceState.TERMINATED, InstanceState.UNKNOWN],
-                terminal_states=[InstanceState.ERROR])
-            helpers.delete_test_network(net)
-
-        with helpers.cleanup_action(lambda: cleanup(inst, net)):
-            try:
-                inst.wait_till_ready()
-            except WaitStateException as e:
-                self.fail("The block device mapped launch did not "
-                          " complete successfully: %s" % e)
-            # TODO: Check instance attachments and make sure they
-            # correspond to requested mappings
+            # Check adding a security group to a running instance
+            test_inst.add_security_group(sg)
+            test_inst.refresh()
+            self.assertTrue(
+                sg in test_inst.security_groups, "Expected security group '%s'"
+                " to be among instance security_groups: [%s]" %
+                (sg, test_inst.security_groups))
+
+            # Check removing a security group from a running instance
+            test_inst.remove_security_group(sg)
+            test_inst.refresh()
+            self.assertTrue(
+                sg not in test_inst.security_groups, "Expected security group"
+                " '%s' to be removed from instance security_groups: [%s]" %
+                (sg, test_inst.security_groups))
+
+            # check floating ips
+            router = self.provider.network.create_router(name=name)
+
+            with helpers.cleanup_action(lambda: router.delete()):
+
+                # TODO: Cloud specific code, needs fixing
+                if self.provider.PROVIDER_ID == 'openstack':
+                    for n in self.provider.network.list():
+                        if n.external:
+                            external_net = n
+                            break
+                else:
+                    external_net = net
+                router.attach_network(external_net.id)
+                router.add_route(subnet.id)
+
+                def cleanup_router():
+                    router.remove_route(subnet.id)
+                    router.detach_network()
+
+                with helpers.cleanup_action(lambda: cleanup_router()):
+                    # check whether adding an elastic ip works
+                    fip = self.provider.network.create_floating_ip()
+                    with helpers.cleanup_action(lambda: fip.delete()):
+                        test_inst.add_floating_ip(fip.public_ip)
+                        test_inst.refresh()
+                        self.assertIn(fip.public_ip, test_inst.public_ips)
+
+                        if isinstance(self.provider, TestMockHelperMixin):
+                            # TODO: Moto bug does not refresh removed public ip
+                            return
+
+                        # check whether removing an elastic ip works
+                        test_inst.remove_floating_ip(fip.public_ip)
+                        test_inst.refresh()
+                        self.assertNotIn(fip.public_ip, test_inst.public_ips)

test/test_image_service.py

@@ -1,11 +1,12 @@
 import uuid
 
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces import MachineImageState
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+import six
 
 
 class CloudImageServiceTestCase(ProviderTestBase):
@@ -21,11 +22,18 @@ class CloudImageServiceTestCase(ProviderTestBase):
         instance_name = "CBImageTest-{0}-{1}".format(
             self.provider.name,
             uuid.uuid4())
-        net, subnet = helpers.create_test_network(self.provider, instance_name)
-        test_instance = helpers.get_test_instance(self.provider, instance_name,
-                                                  subnet=subnet)
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        test_instance = None
+        net = None
         with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
                 test_instance, net)):
+            net, subnet = helpers.create_test_network(
+                self.provider, instance_name)
+            test_instance = helpers.get_test_instance(
+                self.provider, instance_name, subnet=subnet)
+
             name = "CBUnitTestListImg-{0}".format(uuid.uuid4())
             test_image = test_instance.create_image(name)
 
@@ -93,6 +101,11 @@ class CloudImageServiceTestCase(ProviderTestBase):
                     " not as expected: {2}" .format(found_images[0].name,
                                                     get_img.name,
                                                     test_image.name))
+                # TODO: Fix moto so that the BDM is populated correctly
+                if not isinstance(self.provider, TestMockHelperMixin):
+                    # check image size
+                    self.assertGreater(get_img.min_disk, 0, "Minimum disk size"
+                                       " required by image is invalid")
             # TODO: Images take a long time to deregister on EC2. Needs
             # investigation
             images = self.provider.compute.images.list()

test/test_instance_types_service.py

@@ -1,9 +1,10 @@
 from test import helpers
 
-import six
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces.resources import InstanceType
-from test.helpers import ProviderTestBase
+
+import six
 
 
 class CloudInstanceTypesServiceTestCase(ProviderTestBase):

test/test_interface.py

@@ -1,10 +1,13 @@
 import unittest
+
+from test.helpers import ProviderTestBase
+
 import cloudbridge
+
 from cloudbridge.cloud import interfaces
-from test.helpers import ProviderTestBase
-from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
-from cloudbridge.cloud.interfaces import TestMockHelperMixin
 from cloudbridge.cloud.factory import CloudProviderFactory
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
+from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 
 
 class CloudInterfaceTestCase(ProviderTestBase):

test/test_network_service.py

@@ -141,8 +141,8 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
         ):
             net.wait_till_ready()
             self.assertEqual(
-                net.refresh(), 'available',
-                "Network in state %s , yet should be 'available'" % net.state)
+                net.state, 'available',
+                "Network in state '%s', yet should be 'available'" % net.state)
 
             self.assertIn(
                 net.id, repr(net),
@@ -179,20 +179,26 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
     def test_crud_router(self):
 
         def _cleanup(net, subnet, router):
-            router.remove_route(subnet.id)
-            router.detach_network()
-            router.delete()
-            subnet.delete()
-            net.delete()
+            with helpers.cleanup_action(lambda: net.delete()):
+                with helpers.cleanup_action(lambda: subnet.delete()):
+                    with helpers.cleanup_action(lambda: router.delete()):
+                        router.remove_route(subnet.id)
+                        router.detach_network()
 
         name = 'cbtestrouter-{0}'.format(uuid.uuid4())
-        router = self.provider.network.create_router(name=name)
-        net = self.provider.network.create(name=name)
-        cidr = '10.0.1.0/24'
-        sn = net.create_subnet(
-            cidr_block=cidr, name=name,
-            zone=helpers.get_provider_test_data(self.provider, 'placement'))
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sn = None
+        router = None
         with helpers.cleanup_action(lambda: _cleanup(net, sn, router)):
+            router = self.provider.network.create_router(name=name)
+            net = self.provider.network.create(name=name)
+            cidr = '10.0.1.0/24'
+            sn = net.create_subnet(cidr_block=cidr, name=name,
+                                   zone=helpers.get_provider_test_data(
+                                       self.provider, 'placement'))
+
             # Check basic router properties
             self.assertIn(
                 router, self.provider.network.routers(),
@@ -214,6 +220,7 @@ class CloudNetworkServiceTestCase(ProviderTestBase):
                 "Router {0} should not be assoc. with a network {1}".format(
                     router.id, router.network_id))
 
+            # TODO: Cloud specific code, needs fixing
             # Check router connectivity
             # On OpenStack only one network is external and on AWS every
             # network is external, yet we need to use the one we've created?!

test/test_object_life_cycle.py

@@ -1,9 +1,10 @@
 import uuid
 
+from test import helpers
+from test.helpers import ProviderTestBase
+
 from cloudbridge.cloud.interfaces import VolumeState
 from cloudbridge.cloud.interfaces.exceptions import WaitStateException
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
 
 
 class CloudObjectLifeCycleTestCase(ProviderTestBase):

test/test_object_store_service.py

@@ -1,16 +1,17 @@
+import filecmp
+import os
+import tempfile
+import uuid
+
 from datetime import datetime
 from io import BytesIO
+from test import helpers
+from test.helpers import ProviderTestBase
 from unittest import skip
-import uuid
-
-import requests
-
-import tempfile
 
 from cloudbridge.cloud.interfaces.resources import BucketObject
 
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+import requests
 
 
 class CloudObjectStoreServiceTestCase(ProviderTestBase):
@@ -201,12 +202,40 @@ class CloudObjectStoreServiceTestCase(ProviderTestBase):
             obj = test_bucket.create_object(obj_name)
 
             with helpers.cleanup_action(lambda: obj.delete()):
-                content = b"Hello World. Upload from file."
-                with tempfile.NamedTemporaryFile() as tmpFile:
-                    tmpFile.write(content)
-                    tmpFile.flush()
-
-                    obj.upload_from_file(tmpFile.name)
-                    target_stream = BytesIO()
-                    obj.save_content(target_stream)
-                    self.assertEqual(target_stream.getvalue(), content)
+                test_file = os.path.join(
+                    helpers.get_test_fixtures_folder(), 'logo.jpg')
+                obj.upload_from_file(test_file)
+                target_stream = BytesIO()
+                obj.save_content(target_stream)
+                with open(test_file, 'rb') as f:
+                    self.assertEqual(target_stream.getvalue(), f.read())
+
+    @skip("Skip unless you want to test swift objects bigger than 5 Gig")
+    @helpers.skipIfNoService(['object_store'])
+    def test_upload_download_bucket_content_with_large_file(self):
+        """
+        Creates a 6 Gig file in the temp directory, then uploads it to
+        Swift. Once uploaded, then downloads to a new file in the temp
+        directory and compares the two files to see if they match.
+        """
+        temp_dir = tempfile.gettempdir()
+        file_name = '6GigTest.tmp'
+        six_gig_file = os.path.join(temp_dir, file_name)
+        with open(six_gig_file, "wb") as out:
+            out.truncate(6 * 1024 * 1024 * 1024)  # 6 Gig...
+        with helpers.cleanup_action(lambda: os.remove(six_gig_file)):
+            download_file = "{0}/cbtestfile-{1}".format(temp_dir, file_name)
+            bucket_name = "cbtestbucketlargeobjs-{0}".format(uuid.uuid4())
+            test_bucket = self.provider.object_store.create(bucket_name)
+            with helpers.cleanup_action(lambda: test_bucket.delete()):
+                test_obj = test_bucket.create_object(file_name)
+                with helpers.cleanup_action(lambda: test_obj.delete()):
+                    file_uploaded = test_obj.upload_from_file(six_gig_file)
+                    self.assertTrue(file_uploaded, "Could not upload object?")
+                    with helpers.cleanup_action(
+                            lambda: os.remove(download_file)):
+                        with open(download_file, 'wb') as f:
+                            test_obj.save_content(f)
+                            self.assertTrue(
+                                filecmp.cmp(six_gig_file, download_file),
+                                "Uploaded file != downloaded")

test/test_region_service.py

@@ -1,8 +1,9 @@
-import six
+from test import helpers
+from test.helpers import ProviderTestBase
 
 from cloudbridge.cloud.interfaces import Region
-from test.helpers import ProviderTestBase
-import test.helpers as helpers
+
+import six
 
 
 class CloudRegionServiceTestCase(ProviderTestBase):

test/test_security_service.py

@@ -3,10 +3,10 @@ import json
 import unittest
 import uuid
 
-from cloudbridge.cloud.interfaces import TestMockHelperMixin
-
+from test import helpers
 from test.helpers import ProviderTestBase
-import test.helpers as helpers
+
+from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
 
 class CloudSecurityServiceTestCase(ProviderTestBase):
@@ -102,13 +102,17 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
 
     @helpers.skipIfNoService(['security.security_groups'])
     def test_crud_security_group_service(self):
-        name = 'cbtestsecuritygroupA-{0}'.format(uuid.uuid4())
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        #Empty security groups don't exist in GCE. Let's add a dummy rule.
-        sg.add_rule(ip_protocol='tcp', cidr_ip='0.0.0.0/0')
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupA-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             self.assertEqual(name, sg.description)
 
             # test list method
@@ -159,11 +163,17 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
     @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group(self):
         """Test for proper creation of a security group."""
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4())
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupB-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             rule = sg.add_rule(ip_protocol='tcp', from_port=1111, to_port=1111,
                                cidr_ip='0.0.0.0/0')
             found_rule = sg.get_rule(ip_protocol='tcp', from_port=1111,
@@ -215,11 +225,18 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
                 "Mock provider returns InvalidParameterValue: "
                 "Value security_group is invalid for parameter.")
 
-        name = 'cbtestsecuritygroupB-{0}'.format(uuid.uuid4())
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupC-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             rule = sg.add_rule(ip_protocol='tcp', from_port=1111, to_port=1111,
                                cidr_ip='0.0.0.0/0')
             # attempting to add the same rule twice should succeed
@@ -233,11 +250,17 @@ class CloudSecurityServiceTestCase(ProviderTestBase):
     @helpers.skipIfNoService(['security.security_groups'])
     def test_security_group_group_rule(self):
         """Test for proper creation of a security group rule."""
-        name = 'cbtestsecuritygroupC-{0}'.format(uuid.uuid4())
-        net = self.provider.network.create(name=name)
-        sg = self.provider.security.security_groups.create(
-            name=name, description=name, network_id=net.id)
-        with helpers.cleanup_action(lambda: self.cleanup_sg(sg, net)):
+        name = 'CBTestSecurityGroupD-{0}'.format(uuid.uuid4())
+
+        # Declare these variables and late binding will allow
+        # the cleanup method access to the most current values
+        net = None
+        sg = None
+        with helpers.cleanup_action(lambda: helpers.cleanup_test_resources(
+                network=net, security_group=sg)):
+            net, _ = helpers.create_test_network(self.provider, name)
+            sg = self.provider.security.security_groups.create(
+                name=name, description=name, network_id=net.id)
             self.assertTrue(
                 len(sg.rules) == 0,
                 "Expected no security group group rule. Got {0}."

tox.ini

@@ -15,7 +15,8 @@
 envlist = {py27,py36,pypy}-{aws,openstack,gce}
 
 [testenv]
-commands = {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test {posargs}
+commands = flake8 cloudbridge test setup.py 
+    {envpython} -m coverage run --branch --source=cloudbridge --omit=cloudbridge/cloud/interfaces/* setup.py test {posargs}
 setenv =
     aws: CB_TEST_PROVIDER=aws
     openstack: CB_TEST_PROVIDER=openstack