Cleaner listing of resources

cloudbridge/cloud/providers/gce/helpers.py

@@ -1,4 +1,5 @@
 # based on http://stackoverflow.com/a/39126754
+import cloudbridge as cb
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization as crypt_serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -17,3 +18,14 @@ def generate_key_pair():
         crypt_serialization.Encoding.OpenSSH,
         crypt_serialization.PublicFormat.OpenSSH)
     return private_key, public_key
+
+
+def iter_all(resource, **kwargs):
+    token = None
+    while True:
+        response = resource.list(pageToken=token, **kwargs).execute()
+        for item in response.get('items', []):
+            yield item
+        if 'nextPageToken' not in response:
+            return
+        token = response['nextPageToken']

cloudbridge/cloud/providers/gce/resources.py

@@ -32,6 +32,7 @@ from cloudbridge.cloud.interfaces.resources import MachineImageState
 from cloudbridge.cloud.interfaces.resources import RouterState
 from cloudbridge.cloud.interfaces.resources import SnapshotState
 from cloudbridge.cloud.interfaces.resources import VolumeState
+from cloudbridge.cloud.providers.gce import helpers
 
 import googleapiclient
 
@@ -411,24 +412,9 @@ class GCEFirewallsDelegate(object):
         """
         Sync the local cache of all firewalls with the server.
         """
-        self._list_response = None
-        token = None
-        while True:
-            list_response = (self._provider
-                                 .gce_compute
-                                 .firewalls()
-                                 .list(project=self._provider.project_name,
-                                       pageToken=token)
-                                 .execute())
-            if self._list_response is None:
-                self._list_response = list_response
-                if 'items' not in self._list_response:
-                    self._list_response['items'] = []
-            else:
-                self._list_response['items'] += list_response.get('items', [])
-            if 'nextPageToken' not in list_response:
-                return
-            token = list_response['nextPageToken']
+        self._list_response = list(
+                helpers.iter_all(self._provider.gce_compute.firewalls(),
+                                 project=self._provider.project_name))
 
     def _check_list_in_dict(self, dictionary, field_name, value):
         """
@@ -961,24 +947,13 @@ class GCEInstance(BaseInstance):
         """
         self_url = self._provider.parse_url(self._gce_instance['selfLink'])
         try:
-            token = None
-            while True:
-                response = (self._provider
-                                .gce_compute
-                                .targetInstances()
-                                .list(project=self_url.parameters['project'],
-                                      zone=self_url.parameters['zone'],
-                                      pageToken=token)
-                                .execute())
-                if 'items' not in response:
-                    break
-                for target_instance in response['items']:
-                    url = self._provider.parse_url(target_instance['instance'])
-                    if url.parameters['instance'] == self.name:
-                        return target_instance
-                if 'nextPageToken' not in response:
-                    break
-                token = response['nextPageToken']
+            for target_instance in helpers.iter_all(
+                    self._provider.gce_compute.targetInstances(),
+                    project=self_url.parameters['project'],
+                    zone=self_url.parameters['zone']):
+                url = self._provider.parse_url(target_instance['instance'])
+                if url.parameters['instance'] == self.name:
+                    return target_instance
         except Exception as e:
             cb.log.warning('Exception while listing target instances: %s', e)
         return None
@@ -1024,42 +999,29 @@ class GCEInstance(BaseInstance):
                                   .parameters['zone'])
         new_name = target_instance['name']
         new_url = target_instance['selfLink']
-        token = None
         try:
-            while True:
+            for rule in helpers.iter_all(
+                    self._provider.gce_compute.forwardingRules(),
+                    project=self._provider.project_name,
+                    region=ip.region):
+                if rule['IPAddress'] != ip.public_ip:
+                    continue
+                parsed_target_url = self._provider.parse_url(rule['target'])
+                old_zone = parsed_target_url.parameters['zone']
+                old_name = parsed_target_url.parameters['targetInstance']
+                if old_zone == new_zone and old_name == new_name:
+                    return True
                 response = (self._provider
                                 .gce_compute
                                 .forwardingRules()
-                                .list(project=self._provider.project_name,
-                                      region=ip.region,
-                                      pageToken=token)
+                                .setTarget(
+                                    project=self._provider.project_name,
+                                    region=ip.region,
+                                    forwardingRule=rule['name'],
+                                    body={'target': new_url})
                                 .execute())
-                if 'items' not in response:
-                    break
-                for rule in response['items']:
-                    if rule['IPAddress'] != ip.public_ip:
-                        continue
-                    parsed_target_url = self._provider.parse_url(
-                            rule['target'])
-                    old_zone = parsed_target_url.parameters['zone']
-                    old_name = parsed_target_url.parameters['targetInstance']
-                    if old_zone == new_zone and old_name == new_name:
-                        return True
-                    response = (self._provider
-                                    .gce_compute
-                                    .forwardingRules()
-                                    .setTarget(
-                                        project=self._provider.project_name,
-                                        region=ip.region,
-                                        forwardingRule=rule['name'],
-                                        body={'target': new_url})
-                                    .execute())
-                    self._provider.wait_for_operation(response,
-                                                      region=ip.region)
-                    return True
-                if 'nextPageToken' not in response:
-                    break
-                token = response['nextPageToken']
+                self._provider.wait_for_operation(response, region=ip.region)
+                return True
         except Exception as e:
             cb.log.warning(
                 'Exception while listing/changing forwarding rules: %s', e)
@@ -1099,44 +1061,31 @@ class GCEInstance(BaseInstance):
         zone = (self._provider.parse_url(target_instance['zone'])
                               .parameters['zone'])
         name = target_instance['name']
-        token = None
         try:
-            while True:
+            for rule in helpers.iter_all(
+                    self._provider.gce_compute.forwardingRules(),
+                    project=self._provider.project_name,
+                    region=ip.region):
+                if rule['IPAddress'] != ip.public_ip:
+                    continue
+                parsed_target_url = self._provider.parse_url(rule['target'])
+                temp_zone = parsed_target_url.parameters['zone']
+                temp_name = parsed_target_url.parameters['targetInstance']
+                if temp_zone != zone or temp_name != name:
+                    cb.log.warning(
+                            '"%s" is forwarded to "%s" in zone "%s"',
+                            ip.public_ip, temp_name, temp_zone)
+                    return False
                 response = (self._provider
                                 .gce_compute
                                 .forwardingRules()
-                                .list(project=self._provider.project_name,
-                                      region=ip.region,
-                                      pageToken=token)
+                                .delete(
+                                    project=self._provider.project_name,
+                                    region=ip.region,
+                                    forwardingRule=rule['name'])
                                 .execute())
-                if 'items' not in response:
-                    return False
-                for rule in response['items']:
-                    if rule['IPAddress'] != ip.public_ip:
-                        continue
-                    parsed_target_url = self._provider.parse_url(
-                            rule['target'])
-                    temp_zone = parsed_target_url.parameters['zone']
-                    temp_name = parsed_target_url.parameters['targetInstance']
-                    if temp_zone != zone or temp_name != name:
-                        cb.log.warning(
-                                '"%s" is forwarded to "%s" in zone "%s"',
-                                ip.public_ip, temp_name, temp_zone)
-                        return False
-                    response = (self._provider
-                                    .gce_compute
-                                    .forwardingRules()
-                                    .delete(
-                                        project=self._provider.project_name,
-                                        region=ip.region,
-                                        forwardingRule=rule['name'])
-                                    .execute())
-                    self._provider.wait_for_operation(response,
-                                                      region=ip.region)
-                    return True
-                if 'nextPageToken' not in response:
-                    break
-                token = response['nextPageToken']
+                self._provider.wait_for_operation(response, region=ip.region)
+                return True
         except Exception as e:
             cb.log.warning(
                 'Exception while listing/deleting forwarding rules: %s', e)

cloudbridge/cloud/providers/gce/services.py

@@ -373,23 +373,13 @@ class GCEImageService(BaseImageService):
         if self._public_images is not None:
             return
         self._public_images = []
-        for project in GCEImageService._PUBLIC_IMAGE_PROJECTS:
-            try:
-                token = None
-                while True:
-                    response = (self.provider
-                                    .gce_compute
-                                    .images()
-                                    .list(project=project, pageToken=token)
-                                    .execute())
-                    if 'items' in response:
-                        self._public_images.extend(
-                            [GCEMachineImage(self.provider, image) for image
-                             in response['items']])
-                    if 'nextPageToken' not in response:
-                        break
-                    token = response['nextPageToken']
-            except googleapiclient.errors.HttpError as http_error:
+        try:
+            for project in GCEImageService._PUBLIC_IMAGE_PROJECTS:
+                for image in helpers.iter_all(
+                        self.provider.gce_compute.images(), project=project):
+                    self._public_iamges.append(
+                        GCEMachineImage(self.provider, image))
+        except googleapiclient.errors.HttpError as http_error:
                 cb.log.warning("googleapiclient.errors.HttpError: {0}".format(
                     http_error))
 
@@ -440,19 +430,10 @@ class GCEImageService(BaseImageService):
         if (self.provider.project_name not in
                 GCEImageService._PUBLIC_IMAGE_PROJECTS):
             try:
-                token = None
-                while True:
-                    response = (self.provider
-                                    .gce_compute
-                                    .images()
-                                    .list(project=self.provider.project_name,
-                                          pageToken=token)
-                                    .execute())
-                    for image in response.get('items', []):
-                        images.append(GCEMachineImage(self.provider, image))
-                    if 'nextPageToken' not in response:
-                        break
-                    token = response['nextPageToken']
+                for image in helpers.iter_all(
+                        self.provider.gce_compute.images(),
+                        project=self.provider.project_name):
+                    images.append(GCEMachineImage(self.provider, image))
             except googleapiclient.errors.HttpError as http_error:
                 cb.log.warning(
                     "googleapiclient.errors.HttpError: {0}".format(http_error))
@@ -683,20 +664,10 @@ class GCENetworkService(BaseNetworkService):
             region = self.provider.region_name
         try:
             ips = []
-            token = None
-            while True:
-                response = (self.provider
-                                .gce_compute
-                                .addresses()
-                                .list(project=self.provider.project_name,
-                                      region=region,
-                                      pageToken=token)
-                                .execute())
-                for ip in response.get('items', []):
-                    ips.append(GCEFloatingIP(self.provider, ip))
-                if 'nextPageToken' not in response:
-                    break
-                token = response['nextPageToken']
+            for ip in helpers.iter_all(self.provider.gce_compute.addresses(),
+                                       project=self.provider.project_name,
+                                       region=region):
+                ips.append(GCEFloatingIP(self.provider, ip))
             # TODO: if network_id is given, filter out IPs that are assigned to
             # resources in a different network.
             return ips
@@ -730,20 +701,10 @@ class GCENetworkService(BaseNetworkService):
             region = self.provider.region_name
         try:
             routers = []
-            token = None
-            while True:
-                response = (self.provider
-                                .gce_compute
-                                .routers()
-                                .list(project=self.provider.project_name,
-                                      region=region,
-                                      pageToken=token)
-                                .execute())
-                for router in response.get('items', []):
-                    routers.append(GCERouter(self.provider, router))
-                if 'nextPageToken' not in response:
-                    break
-                token = response['nextPageToken']
+            for router in helpers.iter_all(self.provider.gce_compute.routers(),
+                                           project=self.provider.project_name,
+                                           region=region):
+                routers.append(GCERouter(self.provider, router))
             return routers
         except:
             return []
@@ -785,28 +746,29 @@ class GCESubnetService(BaseSubnetService):
                 return subnet
         return None
 
-    def list(self, network=None, region=None):
+    def list(self, network=None, region=None, limit=None, marker=None):
         if not region:
             region = self.provider.region_name
-        try:
-            subnets = []
-            token = None
-            while True:
-                response = (self.provider
-                                .gce_compute
-                                .subnetworks()
-                                .list(project=self.provider.project_name,
-                                      region=region,
-                                      pageToken=token)
-                                .execute())
-                for subnet in response.get('items', []):
-                    subnets.append(GCESubnet(self.provider, subnet))
-                if 'nextPageToken' not in response:
-                    break
-                token = response['nextPageToken']
-            return subnets
-        except:
-            return []
+        filter = None
+        if network is not None:
+            filter = 'network eq %s' % network.resource_url
+        max_result = limit if limit is not None and limit < 500 else 500
+        response = (self.provider
+                        .gce_compute
+                        .subnetworks()
+                        .list(project=self.provider.project_name,
+                              region=region,
+                              filter=filter,
+                              maxResults=max_result,
+                              pageToken=marker)
+                        .execute())
+        subnets = [GCESubnet(self.provider, item) for item in response['items']]
+        if len(subnets) > max_result:
+            cb.log.warning('Expected at most %d results; got %d',
+                           max_result, len(subnets))
+        return ServerPagedResultList('nextPageToken' in response,
+                                     response.get('nextPageToken'),
+                                     False, data=subnets)
 
     def create(self, network, cidr_block, name=None, zone=None):
         if not name:
@@ -1106,16 +1068,7 @@ class GCSObjectStoreService(BaseObjectStoreService):
         """
         Searches in bucket names for a substring.
         """
-        buckets = []
-        token = None
-        while True:
-            list_result = self.list(marker=token)
-            for bucket in list_result:
-                if name in bucket.name:
-                    buckets.append(bucket)
-            if not list_result.is_truncated:
-                break
-            token = list_result.marker
+        buckets = [bucket for bucket in self if name in bucket.name]
         return ClientPagedResultList(self.provider, buckets, limit=limit,
                                      marker=marker)