Added instance type service for AWS + tests.

cloudbridge/providers/aws/impl.py

@@ -6,6 +6,7 @@ import os
 
 import boto
 from boto.ec2.regioninfo import RegionInfo
+from httpretty import HTTPretty
 from moto.ec2 import mock_ec2
 from moto.s3 import mock_s3
 
@@ -108,6 +109,43 @@ class MockAWSCloudProvider(AWSCloudProviderV1, TestMockHelperMixin):
         self.ec2mock.start()
         self.s3mock = mock_s3()
         self.s3mock.start()
+        HTTPretty.register_uri(
+            method="GET",
+            uri="https://raw.githubusercontent.com/powdahound/ec2instances."
+            "info/master/www/instances.json",
+            body="""
+[
+  {
+    "family": "General Purpose",
+    "enhanced_networking": false,
+    "vCPU": 1,
+    "generation": "previous",
+    "ebs_iops": 0,
+    "network_performance": "Low",
+    "ebs_throughput": 0,
+    "vpc": {
+      "ips_per_eni": 4,
+      "max_enis": 2
+    },
+    "arch": [
+      "i386",
+      "x86_64"
+    ],
+    "linux_virtualization_types": [],
+    "ebs_optimized": false,
+    "storage": {
+      "ssd": false,
+      "devices": 1,
+      "size": 160
+    },
+    "max_bandwidth": 0,
+    "instance_type": "m1.small",
+    "ECU": 1.0,
+    "memory": 1.7
+  }
+]
+"""
+        )
 
     def tearDownMock(self):
         """

cloudbridge/providers/aws/resources.py

@@ -2,11 +2,13 @@
 DataTypes used by this provider
 """
 import shutil
+
 from boto.exception import EC2ResponseError
 from boto.s3.key import Key
 from retrying import retry
 
 from cloudbridge.providers.base import BaseInstance
+from cloudbridge.providers.base import BaseInstanceType
 from cloudbridge.providers.base import BaseKeyPair
 from cloudbridge.providers.base import BaseMachineImage
 from cloudbridge.providers.base import BaseSecurityGroup
@@ -15,7 +17,6 @@ from cloudbridge.providers.base import BaseVolume
 from cloudbridge.providers.interfaces import Container
 from cloudbridge.providers.interfaces import ContainerObject
 from cloudbridge.providers.interfaces import InstanceState
-from cloudbridge.providers.interfaces import InstanceType
 from cloudbridge.providers.interfaces import MachineImageState
 from cloudbridge.providers.interfaces import SnapshotState
 from cloudbridge.providers.interfaces import VolumeState
@@ -121,21 +122,48 @@ class AWSPlacementZone(PlacementZone):
         return self._aws_zone.region_name
 
 
-class AWSInstanceType(InstanceType):
+class AWSInstanceType(BaseInstanceType):
 
-    def __init__(self, instance_type):
-        self.instance_type = instance_type
+    def __init__(self, provider, instance_dict):
+        self._provider = provider
+        self._inst_dict = instance_dict
 
     @property
     def id(self):
-        return self.instance_type
+        return self._inst_dict['instance_type']
 
     @property
     def name(self):
-        return self.instance_type
+        return self._inst_dict['instance_type']
 
-    def __repr__(self):
-        return "<CB-AWSInstanceType: {0}>".format(self.id)
+    @property
+    def family(self):
+        return self._inst_dict.get('family')
+
+    @property
+    def vcpus(self):
+        return self._inst_dict.get('vCPU')
+
+    @property
+    def ram(self):
+        return self._inst_dict.get('memory')
+
+    @property
+    def root_disk(self):
+        return 0
+
+    @property
+    def ephemeral_disk(self):
+        storage = self._inst_dict.get('storage')
+        if storage:
+            return storage.get('size') * storage.get("devices")
+        else:
+            return 0
+
+    @property
+    def extra_data(self):
+        return {key: val for key, val in enumerate(self._inst_dict)
+                if key not in ["instance_type", "family", "vCPU", "memory"]}
 
 
 class AWSInstance(BaseInstance):

cloudbridge/providers/aws/services.py

@@ -2,11 +2,13 @@
 Services implemented by the AWS provider.
 """
 from boto.exception import EC2ResponseError
+import requests
 
 from cloudbridge.providers.interfaces import BlockStoreService
 from cloudbridge.providers.interfaces import ComputeService
 from cloudbridge.providers.interfaces import ImageService
 from cloudbridge.providers.interfaces import InstanceType
+from cloudbridge.providers.interfaces import InstanceTypesService
 from cloudbridge.providers.interfaces import KeyPair
 from cloudbridge.providers.interfaces import KeyPairService
 from cloudbridge.providers.interfaces import MachineImage
@@ -20,6 +22,7 @@ from cloudbridge.providers.interfaces import VolumeService
 
 from .resources import AWSContainer
 from .resources import AWSInstance
+from .resources import AWSInstanceType
 from .resources import AWSKeyPair
 from .resources import AWSMachineImage
 from .resources import AWSSecurityGroup
@@ -375,6 +378,15 @@ class AWSComputeService(ComputeService):
 
     def __init__(self, provider):
         self._provider = provider
+        self._instance_types = AWSInstanceTypesService(self.provider)
+
+    @property
+    def provider(self):
+        return self._provider
+
+    @property
+    def instance_types(self):
+        return self._instance_types
 
     def create_instance(self, name, image, instance_type, zone=None,
                         keypair=None, security_groups=None, user_data=None,
@@ -440,3 +452,29 @@ class AWSComputeService(ComputeService):
         return [AWSInstance(self._provider, inst)
                 for res in reservations
                 for inst in res.instances]
+
+
+class AWSInstanceTypesService(InstanceTypesService):
+
+    def __init__(self, provider):
+        self._provider = provider
+
+    @property
+    def instance_data(self):
+        """
+        TODO: Neeeds a caching function with timeout
+        """
+        print "###########################", self._provider
+        r = requests.get(self._provider.config.get(
+            "aws_instance_info_url",
+            "https://raw.githubusercontent.com/powdahound/ec2instances.info"
+            "/master/www/instances.json"))
+        return r.json()
+
+    def list(self):
+        return [AWSInstanceType(self._provider, inst_data)
+                for inst_data in self.instance_data]
+
+    def find_by_name(self, name):
+        return next(
+            (itype for itype in self.list() if itype.name == name), None)

cloudbridge/providers/base.py

@@ -8,6 +8,7 @@ import time
 from cloudbridge.providers.interfaces import CloudProvider
 from cloudbridge.providers.interfaces import Instance
 from cloudbridge.providers.interfaces import InstanceState
+from cloudbridge.providers.interfaces import InstanceType
 from cloudbridge.providers.interfaces import KeyPair
 from cloudbridge.providers.interfaces import MachineImage
 from cloudbridge.providers.interfaces import MachineImageState
@@ -26,7 +27,7 @@ log = logging.getLogger(__name__)
 class BaseCloudProvider(CloudProvider):
 
     def __init__(self, config):
-        self.config = config
+        self._config = config
 
     @property
     def config(self):
@@ -137,6 +138,16 @@ class BaseObjectLifeCycleMixin(ObjectLifeCycleMixin):
             interval)
 
 
+class BaseInstanceType(InstanceType):
+
+    @property
+    def total_disk(self):
+        return self.root_disk + self.ephemeral_disk
+
+    def __repr__(self):
+        return "<CB-{0}: {1}>".format(self.__class__.__name__, self.name)
+
+
 class BaseInstance(BaseObjectLifeCycleMixin, Instance):
 
     @property

cloudbridge/providers/interfaces/resources.py

@@ -624,6 +624,87 @@ class InstanceType(object):
         raise NotImplementedError(
             'InstanceType.name not implemented by this provider')
 
+    @property
+    def family(self):
+        """
+        The family/group that this instance type belongs to. For example,
+        General Purpose Instances or High-Memory Instances. If the provider
+        does not support such a grouping, it may return None.
+
+        :rtype: str
+        :return: Name of the instance family.
+        """
+        raise NotImplementedError(
+            'InstanceType.family not implemented by this provider')
+
+    @property
+    def vcpus(self):
+        """
+        The number of VCPUs supported by this instance type.
+
+        :rtype: int
+        :return: Number of VCPUs
+        """
+        raise NotImplementedError(
+            'InstanceType.vcpus not implemented by this provider')
+
+    @property
+    def ram(self):
+        """
+        The amount of RAM (in mb) supported by this instance type.
+
+        :rtype: int
+        :return: Total RAM (in MB).
+        """
+        raise NotImplementedError(
+            'InstanceType.ram not implemented by this provider')
+
+    @property
+    def root_disk(self):
+        """
+        The size of this instance types's root disk (in GB).
+
+        :rtype: int
+        :return: Size of root disk (in GB).
+        """
+        raise NotImplementedError(
+            'InstanceType.root_disk not implemented by this provider')
+
+    @property
+    def ephemeral_disk(self):
+        """
+        The size of this instance types's total ephemeral storage (in GB).
+
+        :rtype: int
+        :return: Size of ephemeral disks (in GB).
+        """
+        raise NotImplementedError(
+            'InstanceType.ephemeral_disk not implemented by this provider')
+
+    @property
+    def total_disk(self):
+        """
+        The total disk space available on this instance type.
+        (root_disk + ephemeral)
+
+        :rtype: int
+        :return: Size of total disk space (in GB).
+        """
+        raise NotImplementedError(
+            'InstanceType.total_disk not implemented by this provider')
+
+    @property
+    def extra_data(self):
+        """
+        A dictionary of extra data about this instance. May contain
+        nested dictionaries, but all key value pairs are strings or integers.
+
+        :rtype: dict
+        :return: Extra attributes for this instance type
+        """
+        raise NotImplementedError(
+            'InstanceType.extra_data not implemented by this provider')
+
 
 class SecurityGroup(object):
 

cloudbridge/providers/interfaces/services.py

@@ -57,6 +57,7 @@ class ComputeService(ProviderService):
         raise NotImplementedError(
             'list_instances not implemented by this provider')
 
+    @property
     def instance_types(self):
         """
         Provides access to all Instance type related services in this provider.

cloudbridge/providers/openstack/resources.py

@@ -7,6 +7,7 @@ import ipaddress
 from swiftclient.exceptions import ClientException
 
 from cloudbridge.providers.base import BaseInstance
+from cloudbridge.providers.base import BaseInstanceType
 from cloudbridge.providers.base import BaseKeyPair
 from cloudbridge.providers.base import BaseMachineImage
 from cloudbridge.providers.base import BaseSecurityGroup
@@ -15,7 +16,6 @@ from cloudbridge.providers.base import BaseVolume
 from cloudbridge.providers.interfaces import Container
 from cloudbridge.providers.interfaces import ContainerObject
 from cloudbridge.providers.interfaces import InstanceState
-from cloudbridge.providers.interfaces import InstanceType
 from cloudbridge.providers.interfaces import MachineImageState
 from cloudbridge.providers.interfaces import PlacementZone
 from cloudbridge.providers.interfaces import Region
@@ -128,21 +128,44 @@ class OpenStackPlacementZone(PlacementZone):
         return self._os_zone.region_name
 
 
-class OpenStackInstanceType(InstanceType):
+class OpenStackInstanceType(BaseInstanceType):
 
     def __init__(self, os_flavor):
-        self.os_flavor = os_flavor
+        self._os_flavor = os_flavor
 
     @property
     def id(self):
-        return self.os_flavor.id
+        return self._os_flavor.id
 
     @property
     def name(self):
-        return self.os_flavor.name
+        return self._os_flavor.name
 
-    def __repr__(self):
-        return "<CB-OSInstanceType: {0}={1}>".format(self.id, self.name)
+    @property
+    def family(self):
+        # TODO: This may not be standardised accross openstack
+        # but NeCTAR is using it this way
+        return self._os_flavor.extras.get('group')
+
+    @property
+    def vcpus(self):
+        return self._os_flavor.vcpus
+
+    @property
+    def ram(self):
+        return self._os_flavor.ram
+
+    @property
+    def root_disk(self):
+        return self._os_flavor.disk
+
+    @property
+    def ephemeral_disk(self):
+        return self._os_flavor.get('OS-FLV-EXT-DATA:ephemeral', 0)
+
+    @property
+    def extra_data(self):
+        return self._os_flavor.extras
 
 
 class OpenStackInstance(BaseInstance):

cloudbridge/providers/openstack/services.py

@@ -2,8 +2,6 @@
 Services implemented by the OpenStack provider.
 """
 from cinderclient.exceptions import NotFound as CinderNotFound
-from novaclient.exceptions import NotFound as NovaNotFound
-
 from cloudbridge.providers.interfaces import BlockStoreService
 from cloudbridge.providers.interfaces import ComputeService
 from cloudbridge.providers.interfaces import ImageService
@@ -19,6 +17,7 @@ from cloudbridge.providers.interfaces import SecurityGroupService
 from cloudbridge.providers.interfaces import SecurityService
 from cloudbridge.providers.interfaces import SnapshotService
 from cloudbridge.providers.interfaces import VolumeService
+from novaclient.exceptions import NotFound as NovaNotFound
 
 from .resources import OpenStackContainer
 from .resources import OpenStackInstance
@@ -391,7 +390,15 @@ class OpenStackComputeService(ComputeService):
 
     def __init__(self, provider):
         self._provider = provider
-        self.instance_types = OpenStackInstanceTypesService(self._provider)
+        self._instance_types = OpenStackInstanceTypesService(self._provider)
+
+    @property
+    def provider(self):
+        return self._provider
+
+    @property
+    def instance_types(self):
+        return self._instance_types
 
     def create_instance(self, name, image, instance_type, zone=None,
                         keypair=None, security_groups=None, user_data=None,

test/test_provider_compute_service.py

@@ -91,3 +91,16 @@ class ProviderComputeServiceTestCase(ProviderTestBase):
                 self._is_valid_ip(ip_address),
                 "Instance must have a valid IP address")
             test_instance.terminate()
+
+    def test_instance_types(self):
+        instance_types = self.provider.compute.instance_types.list()
+        for inst_type in instance_types:
+            self.assertIsNotNone(
+                inst_type.id,
+                "Instance type id must have a value")
+            self.assertIsNotNone(
+                inst_type.name,
+                "Instance type name must have a value")
+            self.assertIsNotNone(
+                inst_type.name,
+                "Instance type name must have a value")