cloudbridge/tests/test_aws_pagination.py

"""
Unit tests for ``BotoEC2Service`` server-side pagination.

``_get_paginated_results`` hands boto3 a ``PaginationConfig`` built from two
separate numbers: ``MaxItems``, how many results the caller asked for, and
``PageSize``, how many the service returns per request. They used to be the
same value, which made a small result limit walk a large scan in tiny
increments - a filtered ``describe_images`` measured 977.6s at a page size of
5 against 10.0s at 1000, for the same single result.

Separating them moves where page boundaries fall, and the resume token
handed back to callers is defined in terms of those boundaries. These tests
pin the round trip: paging a collection with a small limit must still visit
every object exactly once and terminate, whatever the transport page size.

Exercised against moto so they run in CI without cloud credentials.
"""
import unittest

from moto import mock_aws

from cloudbridge.factory import CloudProviderFactory
from cloudbridge.factory import ProviderList
from cloudbridge.providers.aws.helpers import DEFAULT_PAGE_SIZE

# More objects than one page at the result limit below, so paging is forced.
OBJECT_COUNT = 7
RESULT_LIMIT = 2


class AWSPaginationTestCase(unittest.TestCase):

    def setUp(self):
        self.mock = mock_aws()
        self.mock.start()
        self.provider = CloudProviderFactory().create_provider(
            ProviderList.AWS,
            {'aws_access_key': 'a', 'aws_secret_key': 'b',
             'aws_region_name': 'us-east-1',
             'default_result_limit': RESULT_LIMIT})

    def tearDown(self):
        self.mock.stop()

    def _create_volumes(self):
        created = []
        for i in range(OBJECT_COUNT):
            vol = self.provider.storage.volumes.create(
                'cb-page-%d' % i, 1)
            created.append(vol.id)
        return created

    def test_paging_visits_every_object_exactly_once(self):
        expected = self._create_volumes()

        seen = []
        page = self.provider.storage.volumes.list()
        seen.extend(o.id for o in page)
        pages = 1
        while page.is_truncated:
            page = self.provider.storage.volumes.list(marker=page.marker)
            seen.extend(o.id for o in page)
            pages += 1
            self.assertLess(pages, OBJECT_COUNT + 5,
                            "Paging failed to terminate; the resume token is "
                            "probably not advancing.")

        self.assertEqual(sorted(seen), sorted(expected),
                         "Paging must visit every object exactly once.")
        self.assertEqual(len(seen), len(set(seen)),
                         "Paging returned duplicates across pages.")

    def test_a_page_holds_no_more_than_the_result_limit(self):
        # The caller's limit bounds what comes back, independently of how
        # much was fetched per request to produce it.
        self._create_volumes()
        page = self.provider.storage.volumes.list()
        self.assertLessEqual(len(page), RESULT_LIMIT)

    def _page_size(self, list_op):
        # pylint:disable=protected-access
        return self.provider.storage.volumes.svc._page_size(
            self.provider.ec2_conn.meta.client, list_op)

    def test_transport_page_size_is_independent_of_the_result_limit(self):
        # The regression this guards: PageSize tracking the result limit is
        # what made a sparse scan pathological.
        page_size = self._page_size('describe_volumes')
        self.assertEqual(page_size, DEFAULT_PAGE_SIZE)
        self.assertNotEqual(page_size, RESULT_LIMIT)

    def test_page_size_is_clamped_to_the_operations_ceiling(self):
        # DescribeRouteTables allows 100 where most EC2 describes allow more,
        # and exceeding a ceiling is a hard InvalidParameterValue.
        self.assertEqual(self._page_size('describe_route_tables'), 100)

    def test_page_size_survives_an_unknown_operation(self):
        self.assertEqual(self._page_size('not_an_operation'),
                         DEFAULT_PAGE_SIZE)

    def test_configured_page_size_is_used(self):
        self.provider.config['aws_page_size'] = 250
        self.assertEqual(self._page_size('describe_volumes'), 250)

    def test_configured_page_size_is_clamped_to_the_ceiling(self):
        # A configured value is no safer than the default: DescribeRouteTables
        # would reject anything above 100 outright.
        self.provider.config['aws_page_size'] = 900
        self.assertEqual(self._page_size('describe_route_tables'), 100)

    def test_configured_page_size_is_raised_to_the_floor(self):
        # Several EC2 describes require at least 5, so a smaller configured
        # value would be rejected rather than honoured.
        self.provider.config['aws_page_size'] = 1
        self.assertEqual(self._page_size('describe_vpcs'), 5)


if __name__ == "__main__":
    unittest.main()