Initial merge of branch 'boto3-upgrade' of https://github.com/01000101/cloudbridge into boto3

cloudbridge/cloud/providers/aws/provider.py

@@ -1,17 +1,16 @@
 """Provider implementation based on boto library for AWS-compatible clouds."""
-
+import logging as log
 import os
 
-import boto
-from boto.ec2.regioninfo import RegionInfo
+import boto3
 try:
     # These are installed only for the case of a dev instance
-    from httpretty import HTTPretty
+    from moto.packages.responses import responses
     from moto import mock_ec2
     from moto import mock_s3
 except ImportError:
     # TODO: Once library logging is configured, change this
-    print("[aws provider] moto library not available!")
+    log.debug('[aws provider] moto library not available!')
 
 from cloudbridge.cloud.base import BaseCloudProvider
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
@@ -24,7 +23,7 @@ from .services import AWSSecurityService
 
 
 class AWSCloudProvider(BaseCloudProvider):
-
+    '''AWS cloud provider interface'''
     PROVIDER_ID = 'aws'
     AWS_INSTANCE_DATA_DEFAULT_URL = "https://d168wakzal7fp0.cloudfront.net/" \
                                     "aws_instance_data.json"
@@ -33,30 +32,32 @@ class AWSCloudProvider(BaseCloudProvider):
         super(AWSCloudProvider, self).__init__(config)
 
         # Initialize cloud connection fields
-        self.a_key = self._get_config_value(
-            'aws_access_key', os.environ.get('AWS_ACCESS_KEY', None))
-        self.s_key = self._get_config_value(
-            'aws_secret_key', os.environ.get('AWS_SECRET_KEY', None))
-        self.session_token = self._get_config_value('aws_session_token', None)
-        # EC2 connection fields
-        self.ec2_is_secure = self._get_config_value('ec2_is_secure', True)
-        self.region_name = self._get_config_value(
-            'ec2_region_name', 'us-east-1')
-        self.region_endpoint = self._get_config_value(
-            'ec2_region_endpoint', 'ec2.us-east-1.amazonaws.com')
-        self.ec2_port = self._get_config_value('ec2_port', None)
-        self.ec2_conn_path = self._get_config_value('ec2_conn_path', '/')
-        self.ec2_validate_certs = self._get_config_value(
-            'ec2_validate_certs', False)
-        # S3 connection fields
-        self.s3_is_secure = self._get_config_value('s3_is_secure', True)
-        self.s3_host = self._get_config_value('s3_host', 's3.amazonaws.com')
-        self.s3_port = self._get_config_value('s3_port', None)
-        self.s3_conn_path = self._get_config_value('s3_conn_path', '/')
-        self.s3_validate_certs = self._get_config_value(
-            's3_validate_certs', False)
+        # These are passed as-is to Boto
+        self.region_name = self._get_config_value('aws_region_name',
+                                                  'us-east-1')
+        self.session_cfg = {
+            'aws_access_key_id': self._get_config_value(
+                'aws_access_key', os.environ.get('AWS_ACCESS_KEY', None)),
+            'aws_secret_access_key': self._get_config_value(
+                'aws_secret_key', os.environ.get('AWS_SECRET_KEY', None)),
+            'aws_session_token': self._get_config_value(
+                'aws_session_token', None)
+        }
+        self.ec2_cfg = {
+            'service_name': 'ec2',
+            'use_ssl': self._get_config_value('ec2_is_secure', True),
+            'verify': self._get_config_value('ec2_validate_certs', False),
+            'endpoint_url': self._get_config_value('ec2_endpoint_url', None)
+        }
+        self.s3_cfg = {
+            'service_name': 's3',
+            'use_ssl': self._get_config_value('s3_is_secure', True),
+            'verify': self._get_config_value('s3_validate_certs', False),
+            'endpoint_url': self._get_config_value('s3_endpoint_url', None)
+        }
 
         # service connections, lazily initialized
+        self._session = None
         self._ec2_conn = None
         self._vpc_conn = None
         self._s3_conn = None
@@ -68,18 +69,21 @@ class AWSCloudProvider(BaseCloudProvider):
         self._block_store = AWSBlockStoreService(self)
         self._object_store = AWSObjectStoreService(self)
 
+    @property
+    def session(self):
+        '''Get a low-level session object or create one if needed'''
+        if not self._session:
+            if self.config.debug_mode:
+                boto3.set_stream_logger(level=log.DEBUG)
+            self._session = boto3.session.Session(**self.session_cfg)
+        return self._session
+
     @property
     def ec2_conn(self):
         if not self._ec2_conn:
             self._ec2_conn = self._connect_ec2()
         return self._ec2_conn
 
-    @property
-    def vpc_conn(self):
-        if not self._vpc_conn:
-            self._vpc_conn = self._connect_vpc()
-        return self._vpc_conn
-
     @property
     def s3_conn(self):
         if not self._s3_conn:
@@ -110,52 +114,16 @@ class AWSCloudProvider(BaseCloudProvider):
         """
         Get a boto ec2 connection object.
         """
-        r = RegionInfo(name=self.region_name, endpoint=self.region_endpoint)
-        return self._conect_ec2_region(r)
-
-    def _conect_ec2_region(self, region):
-        ec2_conn = boto.connect_ec2(
-            aws_access_key_id=self.a_key,
-            aws_secret_access_key=self.s_key,
-            is_secure=self.ec2_is_secure,
-            region=region,
-            port=self.ec2_port,
-            path=self.ec2_conn_path,
-            validate_certs=self.ec2_validate_certs,
-            debug=2 if self.config.debug_mode else 0)
-        return ec2_conn
-
-    def _connect_vpc(self):
-        """
-        Get a boto VPC connection object.
-        """
-        r = RegionInfo(name=self.region_name, endpoint=self.region_endpoint)
-        vpc_conn = boto.connect_vpc(
-            aws_access_key_id=self.a_key,
-            aws_secret_access_key=self.s_key,
-            security_token=self.session_token,
-            is_secure=self.ec2_is_secure,
-            region=r,
-            port=self.ec2_port,
-            path=self.ec2_conn_path,
-            validate_certs=self.ec2_validate_certs,
-            debug=2 if self.config.debug_mode else 0)
-        return vpc_conn
+        return self._conect_ec2_region()
+
+    def _conect_ec2_region(self, region=None):
+        '''Get an EC2 resource object'''
+        return self.session.resource(region_name=region or self.region_name,
+                                     **self.ec2_cfg)
 
     def _connect_s3(self):
-        """
-        Get a boto S3 connection object.
-        """
-        s3_conn = boto.connect_s3(aws_access_key_id=self.a_key,
-                                  aws_secret_access_key=self.s_key,
-                                  security_token=self.session_token,
-                                  is_secure=self.s3_is_secure,
-                                  port=self.s3_port,
-                                  host=self.s3_host,
-                                  path=self.s3_conn_path,
-                                  validate_certs=self.s3_validate_certs,
-                                  debug=2 if self.config.debug_mode else 0)
-        return s3_conn
+        '''Get an S3 resource object'''
+        return self.session.resource(**self.s3_cfg)
 
 
 class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
@@ -171,8 +139,8 @@ class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin):
         self.ec2mock.start()
         self.s3mock = mock_s3()
         self.s3mock.start()
-        HTTPretty.register_uri(
-            HTTPretty.GET,
+        responses.add(
+            responses.GET,
             self.AWS_INSTANCE_DATA_DEFAULT_URL,
             body=u"""
 [

cloudbridge/cloud/providers/aws/resources.py

@@ -39,15 +39,17 @@ from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import SubnetState
 from cloudbridge.cloud.interfaces.resources import VolumeState
 
-from retrying import retry
-
 
 class AWSMachineImage(BaseMachineImage):
 
     IMAGE_STATE_MAP = {
         'pending': MachineImageState.PENDING,
+        'transient': MachineImageState.PENDING,
         'available': MachineImageState.AVAILABLE,
-        'failed': MachineImageState.ERROR
+        'deregistered': MachineImageState.ERROR,
+        'failed': MachineImageState.ERROR,
+        'error': MachineImageState.ERROR,
+        'invalid': MachineImageState.ERROR
     }
 
     def __init__(self, provider, image):
@@ -75,8 +77,12 @@ class AWSMachineImage(BaseMachineImage):
 
         :rtype: ``str``
         :return: Name for this image as returned by the cloud middleware.
+                 Returns `None` if the image is deregistered.
         """
-        return self._ec2_image.name
+        try:
+            return self._ec2_image.name
+        except AttributeError:
+            return None
 
     @property
     def description(self):
@@ -84,9 +90,13 @@ class AWSMachineImage(BaseMachineImage):
         Get the image description.
 
         :rtype: ``str``
-        :return: Description for this image as returned by the cloud middleware
+        :return: Description for this image as returned by the cloud
+                 middleware. Returns `None` if the image is deregistered.
         """
-        return self._ec2_image.description
+        try:
+            return self._ec2_image.description
+        except AttributeError:
+            return None
 
     @property
     def min_disk(self):
@@ -103,27 +113,27 @@ class AWSMachineImage(BaseMachineImage):
 
     def delete(self):
         """
-        Delete this image
+        Delete this image.
+        TODO: The Boto implementation used to delete snapshots. Should we
+        delete the snapshots too?
+        http://ranman.com/cleaning-up-aws-with-boto3/
         """
-        self._ec2_image.deregister(delete_snapshot=True)
+        self._ec2_image.deregister()
 
     @property
     def state(self):
-        return AWSMachineImage.IMAGE_STATE_MAP.get(
-            self._ec2_image.state, MachineImageState.UNKNOWN)
+        try:
+            return AWSMachineImage.IMAGE_STATE_MAP.get(
+                self._ec2_image.state, MachineImageState.UNKNOWN)
+        except AttributeError:
+            return MachineImageState.UNKNOWN
 
     def refresh(self):
         """
         Refreshes the state of this instance by re-querying the cloud provider
         for its latest state.
         """
-        image = self._provider.compute.images.get(self.id)
-        if image:
-            # pylint:disable=protected-access
-            self._ec2_image = image._ec2_image
-        else:
-            # image no longer exists
-            self._ec2_image.state = "unknown"
+        self._ec2_image.reload()
 
 
 class AWSPlacementZone(BasePlacementZone):
@@ -252,7 +262,10 @@ class AWSInstance(BaseInstance):
 
         .. note:: an instance must have a (case sensitive) tag ``Name``
         """
-        return self._ec2_instance.tags.get('Name')
+        for tag in self._ec2_instance.tags or []:
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -261,14 +274,14 @@ class AWSInstance(BaseInstance):
         Set the instance name.
         """
         self.assert_valid_resource_name(value)
-        self._ec2_instance.add_tag('Name', value)
+        self._ec2_instance.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     @property
     def public_ips(self):
         """
         Get all the public IP addresses for this instance.
         """
-        return [self._ec2_instance.ip_address]
+        return [self._ec2_instance.public_ip_address]
 
     @property
     def private_ips(self):
@@ -316,7 +329,7 @@ class AWSInstance(BaseInstance):
         """
         Get the placement zone id where this instance is running.
         """
-        return self._ec2_instance.placement
+        return self._ec2_instance.placement.get('AvailabilityZone')
 
     @property
     def security_groups(self):
@@ -326,15 +339,20 @@ class AWSInstance(BaseInstance):
         # boto instance.groups field returns a ``Group`` object so need to
         # convert that into a ``SecurityGroup`` object before creating a
         # cloudbridge SecurityGroup object
-        return [self._provider.security.security_groups.get(group.id)
-                for group in self._ec2_instance.groups]
+        return [
+            self._provider.security.security_groups.get(group_id)
+            for group_id in self.security_group_ids
+        ]
 
     @property
     def security_group_ids(self):
         """
         Get the security groups IDs associated with this instance.
         """
-        return [group.id for group in self._ec2_instance.groups]
+        return list(set([
+            group.get('GroupId') for group in
+            self._ec2_instance.security_groups
+        ]))
 
     @property
     def key_pair_name(self):
@@ -349,24 +367,33 @@ class AWSInstance(BaseInstance):
         """
         self.assert_valid_resource_name(name)
 
-        image_id = self._ec2_instance.create_image(name)
-        # Sometimes, the image takes a while to register, so retry a few times
-        # if the image cannot be found
-        retry_decorator = retry(retry_on_result=lambda result: result is None,
-                                stop_max_attempt_number=3, wait_fixed=1000)
-        image = retry_decorator(self._provider.compute.images.get)(image_id)
+        image = AWSMachineImage(self._provider,
+                                self._ec2_instance.create_image(Name=name))
+        # Wait for the image to exist
+        self._provider.ec2_conn.meta.client.get_waiter('image_exists').wait(
+            ImageIds=[image.id])
+        # Return the image
+        image.refresh()
         return image
 
     def add_floating_ip(self, ip_address):
         """
         Add an elastic IP address to this instance.
         """
-        if self._ec2_instance.vpc_id:
-            aid = self._provider._vpc_conn.get_all_addresses([ip_address])[0]
-            return self._provider.ec2_conn.associate_address(
-                self._ec2_instance.id, allocation_id=aid.allocation_id)
-        else:
-            return self._ec2_instance.use_ip(ip_address)
+        return self._provider.ec2_conn.meta.client.associate_address(**{
+            k: v for k, v in {
+                'InstanceId': self.id,
+                'PublicIp': None if self._ec2_instance.vpc_id else ip_address,
+                'AllocationId':
+                    None if not self._ec2_instance.vpc_id else
+                    ip_address.id if isinstance(ip_address, 'FloatingIP') else
+                    [
+                        x for x in
+                        self._provider.network.floating_ips()
+                        if x.public_ip == ip_address
+                    ][0].id
+            }.items() if v is not None
+        })
 
     def remove_floating_ip(self, ip_address):
         """
@@ -398,7 +425,7 @@ class AWSInstance(BaseInstance):
     @property
     def state(self):
         return AWSInstance.INSTANCE_STATE_MAP.get(
-            self._ec2_instance.state, InstanceState.UNKNOWN)
+            self._ec2_instance.state['Name'], InstanceState.UNKNOWN)
 
     def refresh(self):
         """
@@ -406,11 +433,17 @@ class AWSInstance(BaseInstance):
         for its latest state.
         """
         try:
-            self._ec2_instance.update(validate=True)
+            self._ec2_instance.reload()
         except (EC2ResponseError, ValueError):
             # The volume no longer exists and cannot be refreshed.
             # set the status to unknown
-            self._ec2_instance.status = 'unknown'
+            self._ec2_instance.state = {'Name': InstanceState.UNKNOWN}
+
+    def wait_till_ready(self, timeout=None, interval=None):
+        self._ec2_instance.wait_until_running()
+
+    def wait_till_exists(self, timeout=None, interval=None):
+        self._ec2_instance.wait_until_exists()
 
 
 class AWSVolume(BaseVolume):
@@ -442,7 +475,10 @@ class AWSVolume(BaseVolume):
 
         .. note:: an instance must have a (case sensitive) tag ``Name``
         """
-        return self._volume.tags.get('Name')
+        for tag in self._volume.tags or list():
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -451,15 +487,18 @@ class AWSVolume(BaseVolume):
         Set the volume name.
         """
         self.assert_valid_resource_name(value)
-        self._volume.add_tag('Name', value)
+        self._volume.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     @property
     def description(self):
-        return self._volume.tags.get('Description')
+        for tag in self._volume.tags or list():
+            if tag.get('Key') == 'Description':
+                return tag.get('Value')
+        return None
 
     @description.setter
     def description(self, value):
-        self._volume.add_tag('Description', value)
+        self._volume.create_tags(Tags=[{'Key': 'Description', 'Value': value}])
 
     @property
     def size(self):
@@ -471,7 +510,7 @@ class AWSVolume(BaseVolume):
 
     @property
     def zone_id(self):
-        return self._volume.zone
+        return self._volume.availability_zone
 
     @property
     def source(self):
@@ -482,12 +521,12 @@ class AWSVolume(BaseVolume):
 
     @property
     def attachments(self):
-        if self._volume.attach_data and self._volume.attach_data.id:
-            return BaseAttachmentInfo(self,
-                                      self._volume.attach_data.instance_id,
-                                      self._volume.attach_data.device)
-        else:
-            return None
+        return [
+            BaseAttachmentInfo(self,
+                               a.InstanceId,
+                               a.Device)
+            for a in self._volume.attachments
+        ] if self._volume.attachments else None
 
     def attach(self, instance, device):
         """
@@ -496,13 +535,18 @@ class AWSVolume(BaseVolume):
         instance_id = instance.id if isinstance(
             instance,
             AWSInstance) else instance
-        self._volume.attach(instance_id, device)
+        self._volume.attach_to_instance(InstanceId=instance_id,
+                                        Device=device)
 
     def detach(self, force=False):
         """
         Detach this volume from an instance.
         """
-        self._volume.detach()
+        for a in self.attachments:
+            self._volume.detach_from_instance(
+                InstanceId=a.instance_id,
+                Device=a.device,
+                Force=force)
 
     def create_snapshot(self, name, description=None):
         """
@@ -511,7 +555,7 @@ class AWSVolume(BaseVolume):
         snap = AWSSnapshot(
             self._provider,
             self._volume.create_snapshot(
-                description=description))
+                Description=description))
         snap.name = name
         return snap
 
@@ -524,7 +568,7 @@ class AWSVolume(BaseVolume):
     @property
     def state(self):
         return AWSVolume.VOLUME_STATE_MAP.get(
-            self._volume.status, VolumeState.UNKNOWN)
+            self._volume.state, VolumeState.UNKNOWN)
 
     def refresh(self):
         """
@@ -532,11 +576,11 @@ class AWSVolume(BaseVolume):
         for its latest state.
         """
         try:
-            self._volume.update(validate=True)
+            self._volume.reload()
         except (EC2ResponseError, ValueError):
             # The volume no longer exists and cannot be refreshed.
             # set the status to unknown
-            self._volume.status = 'unknown'
+            self._volume.state = VolumeState.UNKNOWN
 
 
 class AWSSnapshot(BaseSnapshot):
@@ -545,6 +589,7 @@ class AWSSnapshot(BaseSnapshot):
     # ApiReference-cmd-DescribeSnapshots.html
     SNAPSHOT_STATE_MAP = {
         'pending': SnapshotState.PENDING,
+        'deleting': SnapshotState.PENDING,
         'completed': SnapshotState.AVAILABLE,
         'error': SnapshotState.ERROR
     }
@@ -564,7 +609,10 @@ class AWSSnapshot(BaseSnapshot):
 
         .. note:: an instance must have a (case sensitive) tag ``Name``
         """
-        return self._snapshot.tags.get('Name')
+        for tag in self._snapshot.tags or list():
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -573,15 +621,19 @@ class AWSSnapshot(BaseSnapshot):
         Set the snapshot name.
         """
         self.assert_valid_resource_name(value)
-        self._snapshot.add_tag('Name', value)
+        self._snapshot.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     @property
     def description(self):
-        return self._snapshot.tags.get('Description')
+        for tag in self._snapshot.tags or list():
+            if tag.get('Key') == 'Description':
+                return tag.get('Value')
+        return None
 
     @description.setter
     def description(self, value):
-        self._snapshot.add_tag('Description', value)
+        self._snapshot.create_tags(Tags=[{
+            'Key': 'Description', 'Value': value}])
 
     @property
     def size(self):
@@ -598,7 +650,7 @@ class AWSSnapshot(BaseSnapshot):
     @property
     def state(self):
         return AWSSnapshot.SNAPSHOT_STATE_MAP.get(
-            self._snapshot.status, SnapshotState.UNKNOWN)
+            self._snapshot.state, SnapshotState.UNKNOWN)
 
     def refresh(self):
         """
@@ -606,11 +658,11 @@ class AWSSnapshot(BaseSnapshot):
         for its latest state.
         """
         try:
-            self._snapshot.update(validate=True)
+            self._snapshot.reload()
         except (EC2ResponseError, ValueError):
             # The snapshot no longer exists and cannot be refreshed.
             # set the status to unknown
-            self._snapshot.status = 'unknown'
+            self._snapshot.state = SnapshotState.UNKNOWN
 
     def delete(self):
         """
@@ -622,10 +674,13 @@ class AWSSnapshot(BaseSnapshot):
         """
         Create a new Volume from this Snapshot.
         """
-        ec2_vol = self._snapshot.create_volume(placement, size, volume_type,
-                                               iops)
-        cb_vol = AWSVolume(self._provider, ec2_vol)
-        cb_vol.name = "from_snap_{0}".format(self.id or self.name)
+        cb_vol = self._provider.block_store.volumes.create(
+            name=self.name,
+            size=size,
+            zone=placement,
+            snapshot=self._snapshot)
+        cb_vol.wait_till_ready()
+        cb_vol.name = "from_snap_{0}".format(self.name or self.id)
         return cb_vol
 
 
@@ -643,7 +698,7 @@ class AWSKeyPair(BaseKeyPair):
         :return: Unencrypted private key or ``None`` if not available.
 
         """
-        return self._key_pair.material
+        return self._key_pair.key_material
 
 
 class AWSSecurityGroup(BaseSecurityGroup):
@@ -651,6 +706,13 @@ class AWSSecurityGroup(BaseSecurityGroup):
     def __init__(self, provider, security_group):
         super(AWSSecurityGroup, self).__init__(provider, security_group)
 
+    @property
+    def name(self):
+        """
+        Return the name of this security group.
+        """
+        return self._security_group.group_name
+
     @property
     def network_id(self):
         return self._security_group.vpc_id
@@ -693,38 +755,47 @@ class AWSSecurityGroup(BaseSecurityGroup):
                 src_group = self._provider.security.security_groups.get(
                     src_group)
 
-            if self._security_group.authorize(
-                    ip_protocol=ip_protocol,
-                    from_port=from_port,
-                    to_port=to_port,
-                    cidr_ip=cidr_ip,
-                    # pylint:disable=protected-access
-                    src_group=src_group._security_group if src_group
-                    else None):
-                return self.get_rule(ip_protocol, from_port, to_port, cidr_ip,
-                                     src_group)
+            self._security_group.authorize_ingress(
+                IpProtocol=ip_protocol,
+                FromPort=from_port,
+                ToPort=to_port,
+                CidrIp=cidr_ip,
+                SourceSecurityGroupName=(src_group.name if src_group
+                                         else None)
+            )
+            return self.get_rule(ip_protocol, from_port, to_port, cidr_ip,
+                                 src_group)
         except EC2ResponseError as ec2e:
-            if ec2e.code == "InvalidPermission.Duplicate":
+            if ec2e.response['Error']['Code'] == "InvalidPermission.Duplicate":
                 return self.get_rule(ip_protocol, from_port, to_port, cidr_ip,
                                      src_group)
             else:
                 raise ec2e
-        return None
 
     def get_rule(self, ip_protocol=None, from_port=None, to_port=None,
                  cidr_ip=None, src_group=None):
-        for rule in self._security_group.rules:
-            if (rule.ip_protocol == ip_protocol and
-                rule.from_port == from_port and
-                rule.to_port == to_port and
-                rule.grants[0].cidr_ip == cidr_ip) or \
-                    (rule.grants[0].group_id == src_group.id if src_group and
-                        hasattr(rule.grants[0], 'group_id') else False):
-                return AWSSecurityGroupRule(self._provider, rule, self)
+        src_group_id = (src_group.id if isinstance(src_group, SecurityGroup)
+                        else src_group)
+        for rule in self._security_group.ip_permissions:
+            if ip_protocol and rule['IpProtocol'] != ip_protocol:
+                continue
+            elif from_port and rule['FromPort'] != from_port:
+                continue
+            elif to_port and rule['ToPort'] != to_port:
+                continue
+            elif cidr_ip:
+                if cidr_ip not in [x['CidrIp'] for x in rule['IpRanges']]:
+                    continue
+            elif src_group_id:
+                if src_group_id not in [
+                    group_pair.get('GroupId') for group_pair in
+                        rule.get('UserIdGroupPairs', [])]:
+                    continue
+            return AWSSecurityGroupRule(self._provider, rule, self)
         return None
 
     def to_json(self):
-        attr = inspect.getmembers(self, lambda a: not (inspect.isroutine(a)))
+        attr = inspect.getmembers(self, lambda a: not inspect.isroutine(a))
         js = {k: v for (k, v) in attr if not k.startswith('_')}
         json_rules = [r.to_json() for r in self.rules]
         js['rules'] = json_rules
@@ -751,33 +822,30 @@ class AWSSecurityGroupRule(BaseSecurityGroupRule):
 
     @property
     def ip_protocol(self):
-        return self._rule.ip_protocol
+        return self._rule.get('IpProtocol')
 
     @property
     def from_port(self):
-        if str(self._rule.from_port).isdigit():
-            return int(self._rule.from_port)
-        return 0
+        return self._rule.get('FromPort', 0)
 
     @property
     def to_port(self):
-        if str(self._rule.to_port).isdigit():
-            return int(self._rule.to_port)
-        return 0
+        return self._rule.get('ToPort', 0)
 
     @property
     def cidr_ip(self):
-        if len(self._rule.grants) > 0:
-            return self._rule.grants[0].cidr_ip
+        if len(self._rule.get('IpRanges', [])) > 0:
+            return self._rule['IpRanges'][0].get('CidrIp')
         return None
 
     @property
     def group(self):
-        if len(self._rule.grants) > 0:
-            if self._rule.grants[0].group_id:
-                cg = self._provider.ec2_conn.get_all_security_groups(
-                    group_ids=[self._rule.grants[0].group_id])[0]
-                return AWSSecurityGroup(self._provider, cg)
+        if len(self._rule['UserIdGroupPairs']) > 0:
+            if self._rule['UserIdGroupPairs'][0]['GroupId']:
+                return AWSSecurityGroup(
+                    self._provider,
+                    self._provider.ec2_conn.SecurityGroup(
+                        self._rule['UserIdGroupPairs'][0]['GroupId']))
         return None
 
     def to_json(self):
@@ -790,17 +858,15 @@ class AWSSecurityGroupRule(BaseSecurityGroupRule):
     def delete(self):
         if self.group:
             # pylint:disable=protected-access
-            self.parent._security_group.revoke(
-                ip_protocol=self.ip_protocol,
-                from_port=self.from_port,
-                to_port=self.to_port,
-                src_group=self.group._security_group)
+            self.parent._security_group.revoke_ingress(
+                SourceSecurityGroupName=self.group.name)
         else:
             # pylint:disable=protected-access
-            self.parent._security_group.revoke(self.ip_protocol,
-                                               self.from_port,
-                                               self.to_port,
-                                               self.cidr_ip)
+            self._security_group.revoke_ingress(
+                IpProtocol=self.ip_protocol,
+                FromPort=self.from_port,
+                ToPort=self.to_port,
+                CidrIp=self.cidr_ip)
 
 
 class AWSBucketObject(BaseBucketObject):
@@ -939,30 +1005,27 @@ class AWSRegion(BaseRegion):
 
     @property
     def id(self):
-        return self._aws_region.name
+        return self._aws_region
 
     @property
     def name(self):
-        return self._aws_region.name
+        return self.id
 
     @property
     def zones(self):
         """
         Accesss information about placement zones within this region.
         """
-        if self.name == self._provider.region_name:  # optimisation
-            zones = self._provider.ec2_conn.get_all_zones()
-            return [AWSPlacementZone(self._provider, zone.name,
-                                     self._provider.region_name)
-                    for zone in zones]
+        if self.id == self._provider.session.region_name:  # optimisation
+            conn = self._provider.ec2_conn
         else:
-            region = [region for region in
-                      self._provider.ec2_conn.get_all_regions()
-                      if self.name == region.name][0]
-            conn = self._provider._conect_ec2_region(region)
-            zones = conn.get_all_zones()
-            return [AWSPlacementZone(self._provider, zone.name, region.name)
-                    for zone in zones]
+            conn = self._provider._conect_ec2_region(region_name=self.id)
+
+        zones = (conn.meta.client.describe_availability_zones()
+                 .get('AvailabilityZones', []))
+        return [AWSPlacementZone(self._provider, zone,
+                                 self._aws_region)
+                for zone in zones]
 
 
 class AWSNetwork(BaseNetwork):
@@ -989,7 +1052,10 @@ class AWSNetwork(BaseNetwork):
 
         .. note:: the network must have a (case sensitive) tag ``Name``
         """
-        return self._vpc.tags.get('Name')
+        for tag in self._vpc.tags or []:
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -998,7 +1064,7 @@ class AWSNetwork(BaseNetwork):
         Set the network name.
         """
         self.assert_valid_resource_name(value)
-        self._vpc.add_tag('Name', value)
+        self._vpc.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     @property
     def external(self):
@@ -1022,9 +1088,7 @@ class AWSNetwork(BaseNetwork):
 
     @property
     def subnets(self):
-        flter = {'vpc-id': self.id}
-        subnets = self._provider.vpc_conn.get_all_subnets(filters=flter)
-        return [AWSSubnet(self._provider, subnet) for subnet in subnets]
+        return [AWSSubnet(self._provider, s) for s in self._vpc.subnets.all()]
 
     def refresh(self):
         """
@@ -1032,11 +1096,16 @@ class AWSNetwork(BaseNetwork):
         for its latest state.
         """
         try:
-            self._vpc.update(validate=True)
+            self._vpc.reload()
         except (EC2ResponseError, ValueError):
             # The network no longer exists and cannot be refreshed.
             # set the status to unknown
-            self._vpc.state = 'unknown'
+            self._vpc.state = NetworkState.UNKNOWN
+
+    def wait_till_ready(self, timeout=None, interval=None):
+        self._provider.ec2_conn.meta.client.get_waiter('vpc_available').wait(
+            VpcIds=[self.id])
+        self.refresh()
 
 
 class AWSSubnet(BaseSubnet):
@@ -1062,7 +1131,10 @@ class AWSSubnet(BaseSubnet):
 
         .. note:: the subnet must have a (case sensitive) tag ``Name``
         """
-        return self._subnet.tags.get('Name')
+        for tag in self._subnet.tags or []:
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -1071,7 +1143,7 @@ class AWSSubnet(BaseSubnet):
         Set the subnet name.
         """
         self.assert_valid_resource_name(value)
-        self._subnet.add_tag('Name', value)
+        self._subnet.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     @property
     def cidr_block(self):
@@ -1087,12 +1159,12 @@ class AWSSubnet(BaseSubnet):
                                 self._provider.region_name)
 
     def delete(self):
-        return self._provider.vpc_conn.delete_subnet(subnet_id=self.id)
+        self._subnet.delete()
 
     @property
     def state(self):
         return self._SUBNET_STATE_MAP.get(
-            self._subnet.state, NetworkState.UNKNOWN)
+            self._subnet.state, SubnetState.UNKNOWN)
 
     def refresh(self):
         subnet = self._provider.networking.subnets.get(self.id)
@@ -1101,7 +1173,7 @@ class AWSSubnet(BaseSubnet):
             self._subnet = subnet._subnet
         else:
             # subnet no longer exists
-            self._subnet.state = "unknown"
+            self._subnet.state = SubnetState.UNKNOWN
 
 
 class AWSFloatingIP(BaseFloatingIP):
@@ -1146,7 +1218,10 @@ class AWSRouter(BaseRouter):
 
         .. note:: the router must have a (case sensitive) tag ``Name``
         """
-        return self._route_table.tags.get('Name')
+        for tag in self._router.tags or []:
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -1155,11 +1230,13 @@ class AWSRouter(BaseRouter):
         Set the router name.
         """
         self.assert_valid_resource_name(value)
-        self._route_table.add_tag('Name', value)
+        self._router.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     def refresh(self):
-        self._route_table = self._provider.vpc_conn.get_all_route_tables(
-            [self.id])[0]
+        try:
+            self._route_table.reload()
+        except (EC2ResponseError, ValueError):
+            self._route_table.associations = None
 
     @property
     def state(self):
@@ -1172,27 +1249,32 @@ class AWSRouter(BaseRouter):
         return self._route_table.vpc_id
 
     def delete(self):
-        self._provider.vpc_conn.delete_route_table(self.id)
+        self._route_table.delete()
 
     def attach_subnet(self, subnet):
         subnet_id = subnet.id if isinstance(subnet, AWSSubnet) else subnet
-        self._provider.vpc_conn.associate_route_table(self.id, subnet_id)
+        self._route_table.associate_with_subnet(SubnetId=subnet_id)
         self.refresh()
 
     def detach_subnet(self, subnet):
         subnet_id = subnet.id if isinstance(subnet, AWSSubnet) else subnet
-        association_ids = [a.id for a in self._route_table.associations
-                           if a.subnet_id == subnet_id]
-        for a_id in association_ids:
-            self._provider.vpc_conn.disassociate_route_table(a_id)
+        associations = [a for a in self._route_table.associations
+                        if a.subnet_id == subnet_id]
+        for a in associations:
+            a.delete()
+        self.refresh()
 
     def attach_gateway(self, gateway):
-        return self._provider.vpc_conn.attach_internet_gateway(
-            gateway.id, self.network_id)
+        gw_id = (gateway.id if isinstance(gateway, AWSInternetGateway)
+                 else gateway)
+        return self._provider.ec2_conn.meta.client.attach_internet_gateway(
+            InternetGatewayId=gw_id, VpcId=self.vpc_id)
 
     def detach_gateway(self, gateway):
-        return self._provider.vpc_conn.detach_internet_gateway(
-            gateway.id, self.network_id)
+        gw_id = (gateway.id if isinstance(gateway, AWSInternetGateway)
+                 else gateway)
+        return self._provider.ec2_conn.meta.client.detach_internet_gateway(
+            InternetGatewayId=gw_id, VpcId=self.vpc_id)
 
 
 class AWSInternetGateway(BaseInternetGateway):
@@ -1208,12 +1290,10 @@ class AWSInternetGateway(BaseInternetGateway):
 
     @property
     def name(self):
-        """
-        Get the gateway name.
-
-        .. note:: the gateway must have a (case sensitive) tag ``Name``
-        """
-        return self._gateway.tags.get('Name')
+        for tag in self._gateway.tags or []:
+            if tag.get('Key') == 'Name':
+                return tag.get('Value')
+        return None
 
     @name.setter
     # pylint:disable=arguments-differ
@@ -1222,13 +1302,12 @@ class AWSInternetGateway(BaseInternetGateway):
         Set the router name.
         """
         self.assert_valid_resource_name(value)
-        self._gateway.add_tag('Name', value)
+        self._gateway.create_tags(Tags=[{'Key': 'Name', 'Value': value}])
 
     def refresh(self):
-        gateways = self._provider.vpc_conn.get_all_internet_gateways([self.id])
-        if gateways:
-            self._gateway = gateways[0]
-        else:
+        try:
+            self._gateway.reload()
+        except (EC2ResponseError, ValueError):
             self._gateway.state = GatewayState.UNKNOWN
 
     @property
@@ -1241,11 +1320,11 @@ class AWSInternetGateway(BaseInternetGateway):
     @property
     def network_id(self):
         if self._gateway.attachments:
-            return self._gateway.attachments[0].vpc_id
+            return self._gateway.attachments[0].get('VpcId')
         return None
 
     def delete(self):
-        return self._provider._vpc_conn.delete_internet_gateway(self.id)
+        self._gateway.delete()
 
 
 class AWSLaunchConfig(BaseLaunchConfig):

cloudbridge/cloud/providers/aws/services.py

@@ -2,12 +2,10 @@
 import string
 import time
 
-from boto.ec2.blockdevicemapping import BlockDeviceMapping
-from boto.ec2.blockdevicemapping import BlockDeviceType
-from boto.exception import EC2ResponseError, S3ResponseError
+from botocore.exceptions import ClientError
 
 from cloudbridge.cloud.base.resources import ClientPagedResultList
-from cloudbridge.cloud.base.resources import ServerPagedResultList
+# from cloudbridge.cloud.base.resources import ServerPagedResultList
 from cloudbridge.cloud.base.services import BaseBlockStoreService
 from cloudbridge.cloud.base.services import BaseComputeService
 from cloudbridge.cloud.base.services import BaseGatewayService
@@ -27,15 +25,12 @@ from cloudbridge.cloud.base.services import BaseSubnetService
 from cloudbridge.cloud.base.services import BaseVolumeService
 from cloudbridge.cloud.interfaces.exceptions \
     import InvalidConfigurationException
-from cloudbridge.cloud.interfaces.resources import InstanceState
 from cloudbridge.cloud.interfaces.resources import InstanceType
 from cloudbridge.cloud.interfaces.resources import KeyPair
 from cloudbridge.cloud.interfaces.resources import MachineImage
-from cloudbridge.cloud.interfaces.resources import NetworkState
 from cloudbridge.cloud.interfaces.resources import PlacementZone
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import Snapshot
-from cloudbridge.cloud.interfaces.resources import SubnetState
 from cloudbridge.cloud.interfaces.resources import Volume
 
 import requests
@@ -61,6 +56,102 @@ from .resources import AWSVolume
 # cb.set_stream_logger(__name__)
 
 
+class EC2ServiceFilter(object):
+    '''
+        Generic AWS EC2 service filter interface
+
+    :param AWSCloudProvider provider: AWS EC2 provider interface
+    :param str service: Name of the EC2 service to use
+    :param BaseCloudResource cb_iface: CloudBridge class to use
+    '''
+    def __init__(self, provider, service, cb_iface):
+        self.provider = provider
+        self.service = getattr(self.provider.ec2_conn, service)
+        self.iface = cb_iface
+
+    def get(self, val, filter_name, wrapper=True):
+        '''
+            Returns a single resource by filter
+
+        :param str val: Value to filter with
+        :param str filter_name: Name of the filter to use
+        :param bool wrapper: If True, wraps the resulting Boto
+            object in a CloudBridge object
+        :returns: Boto resource object or CloudBridge object or None
+        '''
+        try:
+            objs = list(self.service.filter(Filters=[{
+                'Name': filter_name,
+                'Values': [val]
+            }]).limit(1))
+            obj = objs[0] if objs else None
+            if wrapper:
+                return self.iface(self.provider, obj) if obj else None
+            return obj
+        except ClientError:
+            return None
+
+    def list(self, limit=None, marker=None):
+        '''Returns a list of resources'''
+        try:
+            objs = [self.iface(self.provider, obj)
+                    for obj in self.service.limit(limit)]
+        except ClientError:
+            objs = list()
+        return ClientPagedResultList(self.provider, objs,
+                                     limit=limit, marker=marker)
+
+    def find(self, val, filter_name, limit=None, marker=None):
+        '''
+            Returns a list of resources by filter
+
+        :param str val: Value to filter with
+        :param str filter_name: Name of the filter to use
+        '''
+        try:
+            objs = [
+                self.iface(self.provider, obj)
+                for obj in self.service.filter(Filters=[{
+                    'Name': filter_name,
+                    'Values': [val]
+                }])
+            ]
+        except ClientError:
+            objs = list()
+        return ClientPagedResultList(self.provider, objs,
+                                     limit=limit, marker=marker)
+
+    def create(self, method, **kwargs):
+        '''
+            Creates a resource
+
+        :param str method: Service method to invoke
+        :param object kwargs: Arguments to be passed as-is to
+            the service method
+        '''
+        res = getattr(self.provider.ec2_conn, method)(**kwargs)
+        if isinstance(res, list):
+            return [self.iface(self.provider, x) if x else None for x in res]
+        return self.iface(self.provider, res) if res else None
+
+    def delete(self, val, filter_name):
+        '''
+            Deletes a resource by filter
+
+        :param str val: Value to filter with
+        :param str filter_name: Name of the filter to use
+        :returns: False on error, True if the resource
+            does not exist or was deleted successfully
+        '''
+        res = self.get(val, filter_name, wrapper=False)
+        if res:
+            try:
+                res.delete()
+            except ClientError:
+                return False
+        return True
+
+
 class AWSSecurityService(BaseSecurityService):
 
     def __init__(self, provider):
@@ -95,159 +186,53 @@ class AWSKeyPairService(BaseKeyPairService):
 
     def __init__(self, provider):
         super(AWSKeyPairService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'key_pairs', AWSKeyPair)
 
     def get(self, key_pair_id):
-        """
-        Returns a KeyPair given its ID.
-        """
-        try:
-            kps = self.provider.ec2_conn.get_all_key_pairs(
-                keynames=[key_pair_id])
-            return AWSKeyPair(self.provider, kps[0])
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidKeyPair.NotFound':
-                return None
-            elif ec2e.code == 'InvalidParameterValue':
-                return None
-            else:
-                raise ec2e
+        return self.iface.get(key_pair_id, 'key-name')
 
     def list(self, limit=None, marker=None):
-        """
-        List all key pairs associated with this account.
-
-        :rtype: ``list`` of :class:`.KeyPair`
-        :return:  list of KeyPair objects
-        """
-        key_pairs = [AWSKeyPair(self.provider, kp)
-                     for kp in self.provider.ec2_conn.get_all_key_pairs()]
-        return ClientPagedResultList(self.provider, key_pairs,
-                                     limit=limit, marker=marker)
+        return self.iface.list(limit=limit, marker=marker)
 
     def find(self, name, limit=None, marker=None):
-        """
-        Searches for a key pair by a given list of attributes.
-        """
-        try:
-            key_pairs = [
-                AWSKeyPair(self.provider, kp) for kp in
-                self.provider.ec2_conn.get_all_key_pairs(keynames=[name])]
-            return ClientPagedResultList(self.provider, key_pairs,
-                                         limit=limit, marker=marker)
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidKeyPair.NotFound':
-                return []
-            elif ec2e.code == 'InvalidParameterValue':
-                return []
-            else:
-                raise ec2e
+        return self.iface.find(name, 'key-name', limit=limit, marker=marker)
 
     def create(self, name):
-        """
-        Create a new key pair or raise an exception if one already exists.
-
-        :type name: str
-        :param name: The name of the key pair to be created.
-
-        :rtype: ``object`` of :class:`.KeyPair`
-        :return:  A key pair instance or ``None`` if one was not be created.
-        """
-        AWSKeyPair.assert_valid_resource_name(name)
-
-        kp = self.provider.ec2_conn.create_key_pair(name)
-        if kp:
-            return AWSKeyPair(self.provider, kp)
-        return None
+        return self.iface.create('create_key_pair', KeyName=name)
 
 
 class AWSSecurityGroupService(BaseSecurityGroupService):
 
     def __init__(self, provider):
         super(AWSSecurityGroupService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider,
+                                      'security_groups', AWSSecurityGroup)
 
     def get(self, sg_id):
-        """
-        Returns a SecurityGroup given its id.
-        """
-        try:
-            sgs = self.provider.ec2_conn.get_all_security_groups(
-                group_ids=[sg_id])
-            return AWSSecurityGroup(self.provider, sgs[0]) if sgs else None
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidGroup.NotFound':
-                return None
-            elif ec2e.code == 'InvalidGroupId.Malformed':
-                return None
-            else:
-                raise ec2e
+        return self.iface.get(sg_id, 'group-id')
 
     def list(self, limit=None, marker=None):
-        """
-        List all security groups associated with this account.
-
-        :rtype: ``list`` of :class:`.SecurityGroup`
-        :return:  list of SecurityGroup objects
-        """
-        sgs = [AWSSecurityGroup(self.provider, sg)
-               for sg in self.provider.ec2_conn.get_all_security_groups()]
-
-        return ClientPagedResultList(self.provider, sgs,
-                                     limit=limit, marker=marker)
+        return self.iface.list(limit=limit, marker=marker)
 
     def create(self, name, description, network_id):
-        """
-        Create a new SecurityGroup.
-
-        :type name: str
-        :param name: The name of the new security group.
-
-        :type description: str
-        :param description: The description of the new security group.
-
-        :type  network_id: ``str``
-        :param network_id: The ID of the VPC under which to create the security
-                           group.
-
-        :rtype: ``object`` of :class:`.SecurityGroup`
-        :return:  A SecurityGroup instance or ``None`` if one was not created.
-        """
         AWSSecurityGroup.assert_valid_resource_name(name)
-
-        sg = self.provider.ec2_conn.create_security_group(name, description,
-                                                          network_id)
-        if sg:
-            return AWSSecurityGroup(self.provider, sg)
-        return None
+        res = self.iface.create('create_security_group', **{
+            k: v for k, v in {
+                'GroupName': name,
+                'Description': description,
+                'VpcId': network_id,
+            }.items() if v is not None})
+        if not self.iface.wait_for_create(res.id, 'group-id'):
+            return None
+        return res
 
     def find(self, name, limit=None, marker=None):
-        """
-        Get all security groups associated with your account.
-        """
-        flters = {'group-name': name}
-        ec2_sgs = self.provider.ec2_conn.get_all_security_groups(
-            filters=flters)
-        sgs = [AWSSecurityGroup(self.provider, sg) for sg in ec2_sgs]
-        return ClientPagedResultList(self.provider, sgs,
-                                     limit=limit, marker=marker)
+        return self.iface.find(name, 'group-name', limit=limit, marker=marker)
 
     def delete(self, group_id):
-        """
-        Delete an existing SecurityGroup.
-
-        :type group_id: str
-        :param group_id: The security group ID to be deleted.
-
-        :rtype: ``bool``
-        :return:  ``True`` if the security group does not exist, ``False``
-                  otherwise. Note that this implies that the group may not have
-                  been deleted by this method but instead has not existed in
-                  the first place.
-        """
-        sg = self.get(group_id)
+        sg = self.iface.get(group_id, 'group-id')
         if sg:
             sg.delete()
-            return True
-        return False
 
 
 class AWSBlockStoreService(BaseBlockStoreService):
@@ -272,41 +257,16 @@ class AWSVolumeService(BaseVolumeService):
 
     def __init__(self, provider):
         super(AWSVolumeService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'volumes', AWSVolume)
 
     def get(self, volume_id):
-        """
-        Returns a volume given its id.
-        """
-        try:
-            vols = self.provider.ec2_conn.get_all_volumes(
-                volume_ids=[volume_id])
-            return AWSVolume(self.provider, vols[0]) if vols else None
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidVolume.NotFound':
-                return None
-            elif ec2e.code == 'InvalidParameterValue':
-                # Occurs if volume_id does not start with 'vol-...'
-                return None
-            raise ec2e
+        return self.iface.get(volume_id, 'volume-id')
 
     def find(self, name, limit=None, marker=None):
-        """
-        Searches for a volume by a given list of attributes.
-        """
-        filtr = {'tag:Name': name}
-        aws_vols = self.provider.ec2_conn.get_all_volumes(filters=filtr)
-        cb_vols = [AWSVolume(self.provider, vol) for vol in aws_vols]
-        return ClientPagedResultList(self.provider, cb_vols,
-                                     limit=limit, marker=marker)
+        return self.iface.find(name, 'tag:Name', limit=limit, marker=marker)
 
     def list(self, limit=None, marker=None):
-        """
-        List all volumes.
-        """
-        aws_vols = self.provider.ec2_conn.get_all_volumes()
-        cb_vols = [AWSVolume(self.provider, vol) for vol in aws_vols]
-        return ClientPagedResultList(self.provider, cb_vols,
-                                     limit=limit, marker=marker)
+        return self.iface.list(limit=limit, marker=marker)
 
     def create(self, name, size, zone, snapshot=None, description=None):
         """
@@ -317,14 +277,17 @@ class AWSVolumeService(BaseVolumeService):
         zone_id = zone.id if isinstance(zone, PlacementZone) else zone
         snapshot_id = snapshot.id if isinstance(
             snapshot, AWSSnapshot) and snapshot else snapshot
-
-        ec2_vol = self.provider.ec2_conn.create_volume(
-            size,
-            zone_id,
-            snapshot=snapshot_id)
-        cb_vol = AWSVolume(self.provider, ec2_vol)
+        params = {
+            'Size': size,
+            'AvailabilityZone': zone_id
+        }
+        if snapshot_id:
+            params['SnapshotId'] = snapshot_id
+        cb_vol = self.iface.create('create_volume', **params)
+        # Wait until ready to tag instance
+        cb_vol.wait_till_ready()
         cb_vol.name = name
-        if description:
+        if cb_vol.description:
             cb_vol.description = description
         return cb_vol
 
@@ -333,42 +296,16 @@ class AWSSnapshotService(BaseSnapshotService):
 
     def __init__(self, provider):
         super(AWSSnapshotService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'snapshots', AWSSnapshot)
 
     def get(self, snapshot_id):
-        """
-        Returns a snapshot given its id.
-        """
-        try:
-            snaps = self.provider.ec2_conn.get_all_snapshots(
-                snapshot_ids=[snapshot_id])
-            return AWSSnapshot(self.provider, snaps[0]) if snaps else None
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidSnapshot.NotFound':
-                return None
-            elif ec2e.code == 'InvalidParameterValue':
-                # Occurs if snapshot_id does not start with 'snap-...'
-                return None
-            raise ec2e
+        return self.iface.get(snapshot_id, 'snapshot-id')
 
     def find(self, name, limit=None, marker=None):
-        """
-        Searches for a snapshot by a given list of attributes.
-        """
-        filtr = {'tag-value': name}
-        snaps = [AWSSnapshot(self.provider, snap) for snap in
-                 self.provider.ec2_conn.get_all_snapshots(filters=filtr)]
-        return ClientPagedResultList(self.provider, snaps,
-                                     limit=limit, marker=marker)
+        return self.iface.find(name, 'tag:Name', limit=limit, marker=marker)
 
     def list(self, limit=None, marker=None):
-        """
-        List all snapshots.
-        """
-        snaps = [AWSSnapshot(self.provider, snap)
-                 for snap in self.provider.ec2_conn.get_all_snapshots(
-                 owner='self')]
-        return ClientPagedResultList(self.provider, snaps,
-                                     limit=limit, marker=marker)
+        return self.iface.list(limit=limit, marker=marker)
 
     def create(self, name, volume, description=None):
         """
@@ -378,12 +315,11 @@ class AWSSnapshotService(BaseSnapshotService):
 
         volume_id = volume.id if isinstance(volume, AWSVolume) else volume
 
-        ec2_snap = self.provider.ec2_conn.create_snapshot(
-            volume_id,
-            description=description)
-        cb_snap = AWSSnapshot(self.provider, ec2_snap)
+        cb_snap = self.iface.create('create_snapshot', VolumeId=volume_id)
+        # Wait until ready to tag instance
+        cb_snap.wait_till_ready()
         cb_snap.name = name
-        if description:
+        if cb_snap.description:
             cb_snap.description = description
         return cb_snap
 
@@ -405,7 +341,7 @@ class AWSObjectStoreService(BaseObjectStoreService):
             # user simply does not have permissions to access it. See below.
             bucket = self.provider.s3_conn.get_bucket(bucket_id)
             return AWSBucket(self.provider, bucket)
-        except S3ResponseError as e:
+        except ClientError as e:
             # If 403, it means the bucket exists, but the user does not have
             # permissions to access the bucket. However, limited operations
             # may be permitted (with a session token for example), so return a
@@ -454,40 +390,16 @@ class AWSImageService(BaseImageService):
 
     def __init__(self, provider):
         super(AWSImageService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'images', AWSMachineImage)
 
     def get(self, image_id):
-        """
-        Returns an Image given its id
-        """
-        try:
-            image = self.provider.ec2_conn.get_image(image_id)
-            return AWSMachineImage(self.provider, image) if image else None
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidAMIID.NotFound':
-                return None
-            elif ec2e.code == 'InvalidAMIID.Malformed':
-                # Occurs if image_id does not start with 'ami-...'
-                return None
-            raise ec2e
+        return self.iface.get(image_id, 'image-id')
 
     def find(self, name, limit=None, marker=None):
-        """
-        Searches for an image by a given list of attributes
-        """
-        filters = {'name': name}
-        images = [AWSMachineImage(self.provider, image) for image in
-                  self.provider.ec2_conn.get_all_images(filters=filters)]
-        return ClientPagedResultList(self.provider, images,
-                                     limit=limit, marker=marker)
+        return self.iface.find(name, 'name', limit=limit, marker=marker)
 
     def list(self, limit=None, marker=None):
-        """
-        List all images.
-        """
-        images = [AWSMachineImage(self.provider, image)
-                  for image in self.provider.ec2_conn.get_all_images()]
-        return ClientPagedResultList(self.provider, images,
-                                     limit=limit, marker=marker)
+        return self.iface.list(limit=limit, marker=marker)
 
 
 class AWSComputeService(BaseComputeService):
@@ -520,6 +432,7 @@ class AWSInstanceService(BaseInstanceService):
 
     def __init__(self, provider):
         super(AWSInstanceService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'instances', AWSInstance)
 
     def create(self, name, image, instance_type, subnet, zone=None,
                key_pair=None, security_groups=None, user_data=None,
@@ -543,20 +456,30 @@ class AWSInstanceService(BaseInstanceService):
         subnet_id, zone_id, security_group_ids = \
             self._resolve_launch_options(subnet, zone_id, security_groups)
 
-        reservation = self.provider.ec2_conn.run_instances(
-            image_id=image_id, instance_type=instance_size,
-            min_count=1, max_count=1, placement=zone_id,
-            key_name=key_pair_name, security_group_ids=security_group_ids,
-            user_data=user_data, block_device_map=bdm, subnet_id=subnet_id)
-        instance = None
-        if reservation:
-            instance = AWSInstance(self.provider, reservation.instances[0])
-            instance.wait_for(
-                [InstanceState.PENDING, InstanceState.RUNNING],
-                terminal_states=[InstanceState.TERMINATED,
-                                 InstanceState.ERROR])
-            instance.name = name
-        return instance
+        ress = self.iface.create('create_instances', **{
+            k: v for k, v in {
+                'ImageId': image_id,
+                'MinCount': 1,
+                'MaxCount': 1,
+                'KeyName': key_pair_name,
+                'SecurityGroupIds': security_group_ids or None,
+                'UserData': user_data,
+                'InstanceType': instance_size,
+                'Placement': {
+                    'AvailabilityZone': zone_id
+                },
+                'BlockDeviceMappings': bdm,
+                'SubnetId': subnet_id
+            }.items() if v is not None
+        })
+        if ress and len(ress) == 1:
+            # Wait until the resource exists
+            ress[0].wait_till_exists()
+            # Tag the instance w/ the name
+            ress[0].name = name
+            return ress[0]
+        raise ValueError(
+            'Expected a single object response, got a list: %s' % ress)
 
     def _resolve_launch_options(self, subnet=None, zone_id=None,
                                 security_groups=None):
@@ -597,103 +520,56 @@ class AWSInstanceService(BaseInstanceService):
         are requested (source is None and destination is VOLUME), they will be
         created and the relevant volume ids included in the mapping.
         """
-        bdm = BlockDeviceMapping()
+        bdml = []
         # Assign letters from f onwards
         # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
         next_letter = iter(list(string.ascii_lowercase[6:]))
         # assign ephemeral devices from 0 onwards
         ephemeral_counter = 0
         for device in launch_config.block_devices:
-            bd_type = BlockDeviceType()
-
+            bdm = {}
             if device.is_volume:
-                if device.is_root:
-                    bdm['/dev/sda1'] = bd_type
-                else:
-                    bdm['sd' + next(next_letter)] = bd_type
-
+                # Generate the device path
+                bdm['DeviceName'] = \
+                    '/dev/sd' + 'a1' if device.is_root else next(next_letter)
+                ebs_def = {}
                 if isinstance(device.source, Snapshot):
-                    bd_type.snapshot_id = device.source.id
+                    ebs_def['SnapshotId'] = device.source.id
                 elif isinstance(device.source, Volume):
-                    bd_type.volume_id = device.source.id
+                    ebs_def['VolumeId'] = device.source.id
                 elif isinstance(device.source, MachineImage):
                     # Not supported
                     pass
                 else:
                     # source is None, but destination is volume, therefore
-                    # create a blank volume. If the Zone is None, this
-                    # could fail since the volume and instance may be created
-                    # in two different zones.
-                    if not zone:
+                    # create a blank volume. This requires a size though.
+                    if not device.size:
                         raise InvalidConfigurationException(
-                            "A zone must be specified when launching with a"
-                            " new blank volume block device mapping.")
-                    new_vol = self.provider.block_store.volumes.create(
-                        '',
-                        device.size,
-                        zone)
-                    bd_type.volume_id = new_vol.id
-                bd_type.delete_on_terminate = device.delete_on_terminate
+                            "The source is none and the destination is a"
+                            " volume. Therefore, you must specify a size.")
+                ebs_def['DeleteOnTermination'] = device.delete_on_terminate
                 if device.size:
-                    bd_type.size = device.size
+                    ebs_def['VolumeSize'] = device.size
+                if ebs_def:
+                    bdm['Ebs'] = ebs_def
             else:  # device is ephemeral
-                bd_type.ephemeral_name = 'ephemeral%s' % ephemeral_counter
+                bdm['VirtualName'] = 'ephemeral%s' % ephemeral_counter
+            # Append the config
+            bdml.append(bdm)
 
-        return bdm
+        return bdml
 
     def create_launch_config(self):
         return AWSLaunchConfig(self.provider)
 
     def get(self, instance_id):
-        """
-        Returns an instance given its id. Returns None
-        if the object does not exist.
-        """
-        try:
-            reservation = self.provider.ec2_conn.get_all_reservations(
-                instance_ids=[instance_id])
-            return (AWSInstance(self.provider, reservation[0].instances[0])
-                    if reservation else None)
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidInstanceID.NotFound':
-                return None
-            elif ec2e.code == 'InvalidParameterValue':
-                # Occurs if id does not start with 'inst-...'
-                return None
-            raise ec2e
+        return self.iface.get(instance_id, 'instance-id')
 
     def find(self, name, limit=None, marker=None):
-        """
-        Searches for an instance by a given list of attributes.
-
-        :rtype: ``object`` of :class:`.Instance`
-        :return: an Instance object
-        """
-        filtr = {'tag:Name': name}
-        reservations = self.provider.ec2_conn.get_all_reservations(
-            filters=filtr,
-            max_results=limit,
-            next_token=marker)
-        instances = [AWSInstance(self.provider, inst)
-                     for res in reservations
-                     for inst in res.instances]
-        return ServerPagedResultList(reservations.is_truncated,
-                                     reservations.next_token,
-                                     False, data=instances)
+        return self.iface.find(name, 'tag:Name', limit=limit, marker=marker)
 
     def list(self, limit=None, marker=None):
-        """
-        List all instances.
-        """
-        reservations = self.provider.ec2_conn.get_all_reservations(
-            max_results=limit,
-            next_token=marker)
-        instances = [AWSInstance(self.provider, inst)
-                     for res in reservations
-                     for inst in res.instances]
-        return ServerPagedResultList(reservations.is_truncated,
-                                     reservations.next_token,
-                                     False, data=instances)
+        return self.iface.list(limit=limit, marker=marker)
 
 
 class AWSInstanceTypesService(BaseInstanceTypesService):
@@ -740,8 +616,10 @@ class AWSRegionService(BaseRegionService):
             return None
 
     def list(self, limit=None, marker=None):
-        regions = [AWSRegion(self.provider, region)
-                   for region in self.provider.ec2_conn.get_all_regions()]
+        regions = [
+            AWSRegion(self.provider, region) for region in
+            self.provider.ec2_conn.meta.client.describe_regions()
+            .get('Regions', [])]
         return ClientPagedResultList(self.provider, regions,
                                      limit=limit, marker=marker)
 
@@ -780,55 +658,36 @@ class AWSNetworkService(BaseNetworkService):
 
     def __init__(self, provider):
         super(AWSNetworkService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'vpcs', AWSNetwork)
 
     def get(self, network_id):
-        try:
-            network = self.provider.vpc_conn.get_all_vpcs(vpc_ids=[network_id])
-            return AWSNetwork(self.provider, network[0]) if network else None
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidVpcID.NotFound':
-                return None
-            elif ec2e.code == 'InvalidParameterValue':
-                # Occurs if id does not start with 'vpc-...'
-                return None
-            raise ec2e
+        return self.iface.get(network_id, 'vpc-id')
 
     def list(self, limit=None, marker=None):
-        networks = [AWSNetwork(self.provider, network)
-                    for network in self.provider.vpc_conn.get_all_vpcs()]
-        return ClientPagedResultList(self.provider, networks,
-                                     limit=limit, marker=marker)
+        return self.iface.list(limit=limit, marker=marker)
 
     def find(self, name, limit=None, marker=None):
-        filtr = {'tag:Name': name}
-        networks = [AWSNetwork(self.provider, network)
-                    for network in self.provider.vpc_conn.get_all_vpcs(
-                        filters=filtr)]
-        return ClientPagedResultList(self.provider, networks,
-                                     limit=limit, marker=marker)
+        return self.iface.find(name, 'tag:Name', limit=limit, marker=marker)
 
     def create(self, name, cidr_block):
         AWSNetwork.assert_valid_resource_name(name)
 
-        network = self.provider.vpc_conn.create_vpc(cidr_block=cidr_block)
-        cb_network = AWSNetwork(self.provider, network)
+        cb_net = self.iface.create('create_vpc', CidrBlock=cidr_block)
+        # Wait until ready to tag instance
+        cb_net.wait_till_ready()
         if name:
-            cb_network.wait_for(
-                [NetworkState.PENDING, NetworkState.AVAILABLE],
-                terminal_states=[NetworkState.ERROR])
-            cb_network.name = name
-        return cb_network
+            cb_net.name = name
+        return cb_net
 
     @property
     def floating_ips(self):
-        # fltrs = None
-        # if network_id:
-        #     fltrs = {'network-interface-id': network_id}
-        al = self.provider.vpc_conn.get_all_addresses()
-        return [AWSFloatingIP(self.provider, a) for a in al]
+        self.iface_vips = EC2ServiceFilter(self.provider,
+                                           'vpc_addresses', AWSFloatingIP)
+        return self.iface_vips.list()
 
     def create_floating_ip(self):
-        ip = self.provider.ec2_conn.allocate_address(domain='vpc')
+        ip = self.provider.ec2_conn.meta.client.allocate_address(
+            Domain='vpc')['AllocationId']
         return AWSFloatingIP(self.provider, ip)
 
 
@@ -836,43 +695,35 @@ class AWSSubnetService(BaseSubnetService):
 
     def __init__(self, provider):
         super(AWSSubnetService, self).__init__(provider)
+        self.iface = EC2ServiceFilter(self.provider, 'subnets', AWSSubnet)
 
     def get(self, subnet_id):
-        try:
-            subnets = self.provider.vpc_conn.get_all_subnets([subnet_id])
-            return AWSSubnet(self.provider, subnets[0]) if subnets else None
-        except EC2ResponseError as ec2e:
-            if ec2e.code == 'InvalidSubnetID.NotFound':
-                return None
-            elif ec2e.code == 'InvalidParameterValue':
-                # Occurs if id does not start with 'subnet-...'
-                return None
-            raise ec2e
+        return self.iface.get(subnet_id, 'subnet-id')
 
     def list(self, network=None, limit=None, marker=None):
-        fltr = None
-        if network:
-            network_id = (network.id if isinstance(network, AWSNetwork) else
-                          network)
-            fltr = {'vpc-id': network_id}
-        subnets = [AWSSubnet(self.provider, subnet) for subnet in
-                   self.provider.vpc_conn.get_all_subnets(filters=fltr)]
-        return ClientPagedResultList(self.provider, subnets,
-                                     limit=limit, marker=marker)
+        network_id = network.id if isinstance(network, AWSNetwork) else network
+        if network_id:
+            return self.iface.find(network_id, 'VpcId',
+                                   limit=limit, marker=marker)
+        else:
+            return self.iface.list(limit=limit, marker=marker)
+
+    def find(self, name, limit=None, marker=None):
+        return self.iface.find(name, 'tag:Name', limit=limit, marker=marker)
 
     def create(self, name, network, cidr_block, zone=None):
         AWSSubnet.assert_valid_resource_name(name)
 
         network_id = network.id if isinstance(network, AWSNetwork) else network
-        subnet = self.provider.vpc_conn.create_subnet(network_id, cidr_block,
-                                                      availability_zone=zone)
-        cb_subnet = AWSSubnet(self.provider, subnet)
+        res = self.iface.create('create_subnet', **{
+            k: v for k, v in {
+                'VpcId': network_id,
+                'CidrBlock': cidr_block,
+                'AvailabilityZone': zone,
+            }.items() if v is not None})
         if name:
-            cb_subnet.wait_for(
-                [SubnetState.PENDING, SubnetState.AVAILABLE],
-                terminal_states=[SubnetState.ERROR])
-            cb_subnet.name = name
-        return cb_subnet
+            res.name = name
+        return res
 
     def get_or_create_default(self, zone=None):
         filtr = {'availabilityZone': zone} if zone else None
@@ -903,7 +754,7 @@ class AWSSubnetService(BaseSubnetService):
 
     def delete(self, subnet):
         subnet_id = subnet.id if isinstance(subnet, AWSSubnet) else subnet
-        return self.provider.vpc_conn.delete_subnet(subnet_id=subnet_id)
+        return self.iface.delete(subnet_id, 'subnet-id')
 
 
 class AWSRouterService(BaseRouterService):
@@ -916,7 +767,7 @@ class AWSRouterService(BaseRouterService):
         try:
             routers = self.provider.vpc_conn.get_all_route_tables([router_id])
             return AWSRouter(self.provider, routers[0]) if routers else None
-        except EC2ResponseError as ec2e:
+        except ClientError as ec2e:
             if ec2e.code == 'InvalidRouteTableID.NotFound':
                 return None
             elif ec2e.code == 'InvalidParameterValue':
@@ -953,13 +804,14 @@ class AWSGatewayService(BaseGatewayService):
 
     def __init__(self, provider):
         super(AWSGatewayService, self).__init__(provider)
+        self.iface_igws = EC2ServiceFilter(self.provider,
+                                           'internet_gateways', AWSRouter)
 
     def get_or_create_inet_gateway(self, name):
         AWSInternetGateway.assert_valid_resource_name(name)
 
-        gateway = self.provider.vpc_conn.create_internet_gateway()
-        cb_gateway = AWSInternetGateway(self.provider, gateway)
-        cb_gateway.wait_till_ready()
+        cb_gateway = self.iface_igws.create('create_internet_gateway')
+        self.iface_igws.wait_for_create(cb_gateway.id, 'internet-gateway-id')
         cb_gateway.name = name
         return cb_gateway
 

setup.py

@@ -1,4 +1,7 @@
-"""Library install script for setuptools."""
+"""
+Package install information
+"""
+
 import ast
 import os
 import re
@@ -15,50 +18,61 @@ with open(os.path.join('cloudbridge', '__init__.py')) as f:
             version = ast.literal_eval(m.group(1))
             break
 
-base_reqs = ['bunch>=1.0.1', 'six>=1.10.0', 'retrying>=1.3.3']
-openstack_reqs = ['python-novaclient==7.0.0',
-                  'python-glanceclient>=2.5.0,<=2.6.0',
-                  'python-cinderclient>=1.9.0,<=2.0.1',
-                  'python-swiftclient>=3.2.0,<=3.3.0',
-                  'python-neutronclient>=6.0.0,<=6.1.0',
-                  'python-keystoneclient>=3.8.0,<=3.10.0']
-aws_reqs = ['boto>=2.38.0,<=2.46.1']
-full_reqs = base_reqs + aws_reqs + openstack_reqs
+REQS_BASE = [
+    'bunch>=1.0.1',
+    'six>=1.10.0',
+    'retrying>=1.3.3'
+]
+REQS_AWS = ['boto3']
+REQS_OPENSTACK = [
+    'python-novaclient==7.0.0',
+    'python-glanceclient>=2.5.0,<=2.6.0',
+    'python-cinderclient>=1.9.0,<=2.0.1',
+    'python-swiftclient>=3.2.0,<=3.3.0',
+    'python-neutronclient>=6.0.0,<=6.1.0',
+    'python-keystoneclient>=3.8.0,<=3.10.0'
+]
+REQS_FULL = REQS_BASE + REQS_AWS + REQS_OPENSTACK
 # httpretty is required with/for moto 1.0.0 or AWS tests fail
-dev_reqs = (['tox>=2.1.1', 'moto<1.0.0', 'sphinx>=1.3.1', 'flake8>=3.3.0',
-             'flake8-import-order>=0.12', 'httpretty==0.8.10'] + full_reqs)
+REQS_DEV = ([
+    'tox>=2.1.1',
+    'moto>=1.0.0',
+    'sphinx>=1.3.1',
+    'flake8>=3.3.0',
+    'flake8-import-order>=0.12'] + REQS_FULL
+)
 
-setup(name='cloudbridge',
-      version=version,
-      description='A simple layer of abstraction over multiple cloud'
-      'providers.',
-      author='Galaxy and GVL Projects',
-      author_email='help@genome.edu.au',
-      url='http://cloudbridge.readthedocs.org/',
-      install_requires=full_reqs,
-      extras_require={
-          ':python_version=="2.7"': ['py2-ipaddress'],
-          ':python_version=="3"': ['py2-ipaddress'],
-          'full': full_reqs,
-          'dev': dev_reqs
-      },
-      packages=find_packages(),
-      license='MIT',
-      classifiers=[
-          'Development Status :: 4 - Beta',
-          'Environment :: Console',
-          'Intended Audience :: Developers',
-          'Intended Audience :: System Administrators',
-          'License :: OSI Approved :: MIT License',
-          'Operating System :: OS Independent',
-          'Programming Language :: Python',
-          'Topic :: Software Development :: Libraries :: Python Modules',
-          'Programming Language :: Python :: 2.7',
-          'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.4',
-          'Programming Language :: Python :: 3.5',
-          'Programming Language :: Python :: 3.6',
-          'Programming Language :: Python :: Implementation :: CPython',
-          'Programming Language :: Python :: Implementation :: PyPy'],
-      test_suite="test"
-      )
+setup(
+    name='cloudbridge',
+    version=version,
+    description='A simple layer of abstraction over multiple cloud providers.',
+    author='Galaxy and GVL Projects',
+    author_email='help@genome.edu.au',
+    url='http://cloudbridge.readthedocs.org/',
+    install_requires=REQS_FULL,
+    extras_require={
+        ':python_version=="2.7"': ['py2-ipaddress'],
+        ':python_version=="3"': ['py2-ipaddress'],
+        'full': REQS_FULL,
+        'dev': REQS_DEV
+    },
+    packages=find_packages(),
+    license='MIT',
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy'],
+    test_suite="test"
+)