Initial implementation of iterable objects. Also updated moto version

cloudbridge/cloud/base.py

@@ -21,6 +21,7 @@ from cloudbridge.cloud.interfaces.resources import MachineImage
 from cloudbridge.cloud.interfaces.resources import MachineImageState
 from cloudbridge.cloud.interfaces.resources import ObjectLifeCycleMixin
 from cloudbridge.cloud.interfaces.resources import Region
+from cloudbridge.cloud.interfaces.resources import ResultList
 from cloudbridge.cloud.interfaces.resources import SecurityGroup
 from cloudbridge.cloud.interfaces.resources import SecurityGroupRule
 from cloudbridge.cloud.interfaces.resources import Snapshot
@@ -160,6 +161,50 @@ class BaseObjectLifeCycleMixin(ObjectLifeCycleMixin):
             interval)
 
 
+class BaseResultList(ResultList):
+
+    def __init__(self, is_truncated, marker, supports_total, total=None):
+        list.__init__(self)
+        self._marker = marker
+        self._is_truncated = is_truncated
+        self._supports_total = True if supports_total else False
+        self._total = total
+
+    @property
+    def marker(self):
+        return self._marker
+
+    @property
+    def is_truncated(self):
+        return self._is_truncated
+
+    @property
+    def supports_total(self):
+        return self._supports_total
+
+    @property
+    def total_results(self):
+        return self._total
+
+
+class BaseIterableObjectMixin():
+    """
+    A mixin to provide iteration capability for a class
+    that support a list(limit, marker) method.
+    """
+
+    def __iter__(self):
+        more_results = True
+        marker = None
+
+        while more_results:
+            result_list = self.list(marker=marker)
+            for result in result_list:
+                yield result
+            marker = result_list.marker
+            more_results = result_list.is_truncated
+
+
 class BaseInstanceType(InstanceType):
 
     @property
@@ -562,7 +607,8 @@ class BaseInstanceTypesService(InstanceTypesService, BaseProviderService):
             return None
 
 
-class BaseInstanceService(InstanceService, BaseProviderService):
+class BaseInstanceService(
+        BaseIterableObjectMixin, InstanceService, BaseProviderService):
 
     def __init__(self, provider):
         super(BaseInstanceService, self).__init__(provider)

cloudbridge/cloud/interfaces/resources.py

@@ -112,6 +112,69 @@ class ObjectLifeCycleMixin(object):
         pass
 
 
+class ResultList(list):
+    """
+    This is a wrapper class around a standard Python :py:class:`list` class
+    and provides some extra properties to aid with paging through a large
+    number of results.
+
+    example:
+    ```
+    # get first page of results
+    rl = provider.compute.instances.list(limit=50)
+    for result in rl:
+        print("Instance Data: {0}", result)
+    if rl.supports_total:
+        print("Total results: {0}".format(rl.total_results))
+    else:
+        print("Total records unknown,"
+              "but has more data?: {0}."format(rl.is_truncated))
+
+    # Page to next set of results
+    if (rl.is_truncated)
+        rl = provider.compute.instances.list(limit=100,
+                                             marker=rl.marker)
+    ```
+    """
+    __metaclass__ = ABCMeta
+
+    @abstractproperty
+    def marker(self):
+        """
+        This is an opaque identifier used to assist in paging through very long
+        lists of objects. This marker can be provided to the list method to get
+        the next set of results.
+        """
+        pass
+
+    @abstractproperty
+    def is_truncated(self):
+        """
+        Indicates whether this result list has more results
+        that can be paged in.
+        """
+        pass
+
+    @abstractproperty
+    def supports_total(self):
+        """
+        Indicates whether the provider supports returning the total number of
+        available results. The supports_total property should be checked
+        before accessing the total_results property.
+        """
+        pass
+
+    @abstractproperty
+    def total_results(self):
+        """
+        Indicates the total number of results for a particular query. The
+        supports_total property should be used to check whether the provider
+        supports returning the total number of results, before accessing this
+        property, or the behaviour is indeterminate.
+        """
+        pass
+
+
 class InstanceState(object):
 
     """
@@ -471,7 +534,8 @@ class MachineImage(ObjectLifeCycleMixin):
         Get the image description.
 
         :rtype: ``str``
-        :return: Description for this image as returned by the cloud middleware.
+        :return: Description for this image as returned by the cloud
+                 middleware.
         """
         pass
 

cloudbridge/cloud/interfaces/services.py

@@ -78,6 +78,22 @@ class InstanceService(ProviderService):
     """
     __metaclass__ = ABCMeta
 
+    @abstractmethod
+    def __iter__(self):
+        """
+        Iterate through the  list of instances.
+
+        Example:
+        ```
+        for instance in provider.compute.instances:
+            print(instance.name)
+        ```
+
+        :rtype: ``object`` of :class:`.Instance`
+        :return:  an Instance object
+        """
+        pass
+
     @abstractmethod
     def get(self, instance_id):
         """
@@ -100,12 +116,60 @@ class InstanceService(ProviderService):
         pass
 
     @abstractmethod
-    def list(self):
-        """
-        List all instances.
-
-        :rtype: ``list`` of :class:`.Instance`
-        :return: list of Instance objects
+    def list(self, limit=None, marker=None):
+        """
+        List all instances. If a limit and marker are specified,
+        the records will be fetched up to the limit starting from the
+        marker onwards. The returned list is a list of class
+        ResultList, which has extra properties like is_truncated,
+        supports_total and total_records to provide extra information
+        about record availability.
+
+        If limit is not specified, the limit will default to the underlying
+        provider's default limit. Therefore, you need to check the is_truncated
+        property to determine whether more records are available.
+
+        The total number of results can be determined through the total_results
+        property. Not all provides will support returning the total_results
+        property, so the supports_total property can be used to determine
+        whether a total is supported.
+
+        To iterate through all the records, it's recommended to iterate
+        directly through the instances using __iter__ instead of calling
+        the list method.
+
+        example:
+        ```
+        # get first page of results
+        instlist = provider.compute.instances.list(limit=50)
+        for instance in instlist:
+            print("Instance Data: {0}", instance)
+        if instlist.supports_total:
+            print("Total results: {0}".format(instlist.total_results))
+        else:
+            print("Total records unknown,"
+                  "but has more data?: {0}".format(instlist.is_truncated))
+
+        # Page to next set of results
+        if (instlist.is_truncated)
+            instlist = provider.compute.instances.list(limit=100,
+                                                       marker=instlist.marker)
+
+        # Alternative: iterate through total available records
+        for instance in provider.compute.instances:
+            print(instance)
+        ```
+
+        :type  limit: ``int``
+        :param limit: The maximum number of objects to return
+
+        :type  marker: ``str``
+        :param marker: The marker is an opaque identifier used to assist
+        in paging through very long lists of objects. It is returned on each
+        invocation of the list method.
+
+        :rtype: ``ResultList`` of :class:`.Instance`
+        :return: A ResultList object containing a list of Instances
         """
         pass
 

cloudbridge/cloud/providers/aws/services.py

@@ -17,6 +17,7 @@ from cloudbridge.cloud.base import BaseKeyPairService
 from cloudbridge.cloud.base import BaseLaunchConfig
 from cloudbridge.cloud.base import BaseObjectStoreService
 from cloudbridge.cloud.base import BaseRegionService
+from cloudbridge.cloud.base import BaseResultList
 from cloudbridge.cloud.base import BaseSecurityGroupService
 from cloudbridge.cloud.base import BaseSecurityService
 from cloudbridge.cloud.base import BaseSnapshotService
@@ -528,14 +529,18 @@ class AWSInstanceService(BaseInstanceService):
         raise NotImplementedError(
             'find_instance not implemented by this provider')
 
-    def list(self):
+    def list(self, limit=None, marker=None):
         """
         List all instances.
         """
         reservations = self.provider.ec2_conn.get_all_reservations()
-        return [AWSInstance(self.provider, inst)
-                for res in reservations
-                for inst in res.instances]
+        results = BaseResultList(reservations.is_truncated,
+                                 reservations.next_token,
+                                 False)
+        for res in reservations:
+            for inst in res.instances:
+                results.append(AWSInstance(self.provider, inst))
+        return results
 
 AWS_INSTANCE_DATA_DEFAULT_URL = "https://swift.rc.nectar.org.au:8888/v1/" \
                                 "AUTH_377/cloud-bridge/aws/instance_data.json"

cloudbridge/cloud/providers/openstack/services.py

@@ -1,6 +1,8 @@
 """
 Services implemented by the OpenStack provider.
 """
+import itertools
+
 from cinderclient.exceptions import NotFound as CinderNotFound
 from novaclient.exceptions import NotFound as NovaNotFound
 
@@ -13,6 +15,7 @@ from cloudbridge.cloud.base import BaseKeyPairService
 from cloudbridge.cloud.base import BaseLaunchConfig
 from cloudbridge.cloud.base import BaseObjectStoreService
 from cloudbridge.cloud.base import BaseRegionService
+from cloudbridge.cloud.base import BaseResultList
 from cloudbridge.cloud.base import BaseSecurityGroupService
 from cloudbridge.cloud.base import BaseSecurityService
 from cloudbridge.cloud.base import BaseSnapshotService
@@ -536,13 +539,25 @@ class OpenStackInstanceService(BaseInstanceService):
         raise NotImplementedError(
             'find_instance not implemented by this provider')
 
-    def list(self):
+    def list(self, limit=None, marker=None):
         """
         List all instances.
         """
-        instances = self._provider.nova.servers.list()
-        return [OpenStackInstance(self._provider, instance)
-                for instance in instances]
+        # TODO: Move hardcoded number to config setting
+        limit = limit or 50
+        instances = self._provider.nova.servers.list(
+            limit=limit + 1,
+            marker=marker)
+        # since we fetched one more than the limit, we can
+        # detect whether extra records are present
+        is_truncated = len(instances) > limit
+        next_token = instances[-2].id if is_truncated else instances[-1].id
+        results = BaseResultList(is_truncated,
+                                 next_token,
+                                 False)
+        for instance in itertools.islice(instances, limit):
+            results.append(OpenStackInstance(self._provider, instance))
+        return results
 
     def get(self, instance_id):
         """

requirements.txt

@@ -1,3 +1,3 @@
 -e ".[full]"
-httpretty==0.8.6
-git+git://github.com/spulec/moto.git@1d58aae42a7ea10d1617428970f6f5d55a76fc80
+httpretty==0.8.5
+git+git://github.com/spulec/moto.git@ab3682a55c1c4f2983819e73a8cc59c0159a33f3

setup.py

@@ -15,7 +15,7 @@ setup(name='cloudbridge',
       author='Galaxy and GVL Projects',
       author_email='support@genome.edu.au',
       url='http://cloudbridge.readthedocs.org/',
-      install_requires=base_reqs,
+      install_requires=full_reqs,
       extras_require={
           ':python_version=="2.7"': ['py2-ipaddress'],
           ':python_version=="3"': ['py2-ipaddress'],