Merge pull request #248 from CloudVE/new_vm_types

AWS VMTypes from new APIs

cloudbridge/providers/aws/resources.py

@@ -180,7 +180,7 @@ class AWSVMType(BaseVMType):
 
     @property
     def id(self):
-        return str(self._inst_dict['instance_type'])
+        return str(self._inst_dict.get('InstanceType'))
 
     @property
     def name(self):
@@ -188,18 +188,26 @@ class AWSVMType(BaseVMType):
 
     @property
     def family(self):
-        return self._inst_dict.get('family')
+        # Limited to whether CurrentGeneration or not
+        curr = self._inst_dict.get('CurrentGeneration')
+        if curr:
+            return 'CurrentGeneration'
+        return None
 
     @property
     def vcpus(self):
-        vcpus = self._inst_dict.get('vCPU')
-        if vcpus == 'N/A':
-            return None
-        return vcpus
+        vcpus = self._inst_dict.get('VCpuInfo')
+        if vcpus:
+            return vcpus.get('DefaultVCpus', 0)
+        return 0
 
     @property
     def ram(self):
-        return self._inst_dict.get('memory')
+        ram = self._inst_dict.get('MemoryInfo')
+        if ram:
+            mib = ram.get('SizeInMiB', 0)
+            return mib / 1024
+        return 0
 
     @property
     def size_root_disk(self):
@@ -207,24 +215,26 @@ class AWSVMType(BaseVMType):
 
     @property
     def size_ephemeral_disks(self):
-        storage = self._inst_dict.get('storage')
+        storage = self._inst_dict.get('InstanceStorageInfo')
         if storage:
-            return storage.get('size') * storage.get("devices")
-        else:
-            return 0
+            return storage.get('TotalSizeInGB', 0)
+        return 0
 
     @property
     def num_ephemeral_disks(self):
-        storage = self._inst_dict.get('storage')
+        storage = self._inst_dict.get('InstanceStorageInfo')
         if storage:
-            return storage.get("devices")
-        else:
-            return 0
+            disks = storage.get("Disks", [])
+            count = 0
+            for disk in disks:
+                count += disk.get("Count", 0)
+            return count
+        return 0
 
     @property
     def extra_data(self):
         return {key: val for key, val in self._inst_dict.items()
-                if key not in ["instance_type", "family", "vCPU", "memory"]}
+                if key not in ["InstanceType", "VCpuInfo", "MemoryInfo"]}
 
 
 class AWSInstance(BaseInstance):

cloudbridge/providers/aws/services.py

@@ -6,10 +6,6 @@ import uuid
 
 from botocore.exceptions import ClientError
 
-import cachetools
-
-import requests
-
 import tenacity
 
 import cloudbridge.base.helpers as cb_helpers
@@ -840,35 +836,54 @@ class AWSVMTypeService(BaseVMTypeService):
     def __init__(self, provider):
         super(AWSVMTypeService, self).__init__(provider)
 
-    @property
-    @cachetools.cached(cachetools.TTLCache(maxsize=1, ttl=24*3600))
-    def instance_data(self):
-        """
-        Fetch info about the available instances.
-
-        To update this information, update the file pointed to by the
-        ``provider.AWS_INSTANCE_DATA_DEFAULT_URL`` above. The content for this
-        file should be obtained from this repo:
-        https://github.com/powdahound/ec2instances.info (in particular, this
-        file: https://raw.githubusercontent.com/powdahound/ec2instances.info/
-        master/www/instances.json).
-        """
-        r = requests.get(self.provider.config.get(
-            "aws_instance_info_url",
-            self.provider.AWS_INSTANCE_DATA_DEFAULT_URL))
-        # Some instances are only available in certain regions. Use pricing
-        # info to determine and filter out instance types that are not
-        # available in the current region
-        vm_types_list = r.json()
-        return [vm_type for vm_type in vm_types_list
-                if vm_type.get('pricing', {}).get(self.provider.region_name)]
+    @dispatch(event="provider.compute.vm_types.get",
+              priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
+    def get(self, vm_type):
+        try:
+            t = self.provider.ec2_conn.meta.client.describe_instance_types(
+                InstanceTypes=[vm_type]).get('InstanceTypes')[0]
+            return AWSVMType(self.provider, t)
+        except ClientError as e:
+            if 'InvalidInstanceType' in e.response.get('Error',
+                                                       {}).get('Code'):
+                return None
+            else:
+                raise e
 
     @dispatch(event="provider.compute.vm_types.list",
               priority=BaseVMTypeService.STANDARD_EVENT_PRIORITY)
     def list(self, limit=None, marker=None):
-        vm_types = [AWSVMType(self.provider, vm_type)
-                    for vm_type in self.instance_data]
-        return ClientPagedResultList(self.provider, vm_types,
+        client = self.provider.ec2_conn.meta.client
+        vmt_list_resp = client.describe_instance_type_offerings(
+            LocationType='availability-zone',
+            Filters=[{'Name': 'location',
+                      'Values': [self.provider.zone_name]}],
+            # MaxResults is set to max value (1000)
+            # and client-side pagination is used
+            **trim_empty_params({'MaxResults': 1000, 'NextToken': None}))
+        vmt_list = vmt_list_resp.get('InstanceTypeOfferings')
+        while vmt_list_resp.get("NextToken"):
+            vmt_list_resp = client.describe_instance_type_offerings(
+                LocationType='availability-zone',
+                Filters=[{'Name': 'location',
+                          'Values': [self.provider.zone_name]}],
+                **trim_empty_params(
+                    {'MaxResults': 1000,
+                     'NextToken': vmt_list_resp.get("NextToken")}))
+            vmt_list.extend(vmt_list_resp.get('InstanceTypeOfferings'))
+
+        vmt_list_names = [x.get("InstanceType")
+                          for x in vmt_list]
+        # describe_instance_types call can get at most 100 types at once
+        chunks = [vmt_list_names[x:x + 100]
+                  for x in range(0, len(vmt_list_names), 100)]
+        raw_types = []
+        for chunk in chunks:
+            raw_chunk = client.describe_instance_types(
+                InstanceTypes=chunk).get('InstanceTypes')
+            raw_types.extend(raw_chunk)
+        cb_types = [AWSVMType(self.provider, t) for t in raw_types]
+        return ClientPagedResultList(self.provider, cb_types,
                                      limit=limit, marker=marker)
 
 

setup.py

@@ -22,7 +22,6 @@ with open(os.path.join('cloudbridge', '__init__.py')) as f:
 REQS_BASE = [
     'six>=1.11',
     'tenacity>=6.0',
-    'cachetools>=2.1.0',
     'deprecation>=2.0.7',
     'pyeventsystem<2'
 ]