Merge branch 'display_id_and_label' of https://github.com/CloudVE/cloudbridge into display_id_and_label

CHANGELOG.rst

@@ -1,3 +1,32 @@
+1.0.0 - Aug ??, 2018. (sha ??)
+-------
+
+* Added Microsoft Azure as a provider
+* Restructured the interface to make it more comprehensible and uniform across
+  all supported providers. See `issue #69 <https://github.com/CloudVE/cloudbridge/issues/69>`_
+  for more details as well as the library layout image for an easy visual
+  reference: https://github.com/CloudVE/cloudbridge#quick-reference.
+* Migrated AWS implementation to use boto3 library from boto (thanks @01000101)
+* Cleaned up use of ``name`` property for resources. Resources now have ``id``,
+  ``name``, and ``label`` properties to represent respectively: a unique
+  identifier supplied by the provider; a descriptive, unchangeable name; and a
+  user-supplied label that can be modified during the existence of a resource.
+* Added enforcement of name and label value: names must be less than 64 chars
+  in length and consist of only lower case letters and dashes
+* Refactored tests and extracted standard interface tests where all resources
+  are being tested using the same code structure. Also, tests will run only
+  for providers that implement a given service.
+* Moved the repository from github.com/gvlproject to github.com/cloudve org
+* When deleting an OpenStack network, clear any ports
+* Added support for launching OpenStack instances into a specific subnet
+* Update image list interface to allow filtering by owner
+* When listing images on AWS, filter only the ones by current account owner
+* Retrieve AWS instance types from a public service to include latest values
+* Instance state uses ``DELETED`` state instead of ``TERMINATED``
+* Return VM type RAM in GB
+* Add implementation for ``generate_url`` on OpenStack
+* General documentation updates
+
 0.3.3 - Aug 7, 2017. (sha 348e1e88935f61f53a83ed8d6a0e012a46621e25)
 -------
 
@@ -12,7 +41,7 @@
 0.3.1 - April 18, 2017. (sha f36a462e886d8444cb2818f6573677ecf0565315)
 -------
 
-* Patch for binary file handling in openstack
+* Patch for binary file handling in OpenStack
 
 0.3.0 - April 11, 2017. (sha 13539ccda9e4809082796574d18b1b9bb3f2c624)
 -------
@@ -23,7 +52,7 @@
 * Added supports for accessing EC2 containers with restricted permissions.
 * Removed exists() method from object store interface. Use get()==None check
   instead.
-* New method (img.min_disk) for geting size of machine image.
+* New method (img.min_disk) for getting size of machine image.
 * Test improvements (flake8 during build, more tests)
 * Misc bug fixes and improvements
 * Changed library to beta state
@@ -66,5 +95,5 @@
 * Support for AWS and OpenStack clouds
 * Basic usage docs and complete API docs
 * 95% test coverage
-* Support for AWS mock test provder (via
+* Support for AWS mock test provider (via
   `moto <https://github.com/spulec/moto>`_)

cloudbridge/cloud/base/helpers.py

@@ -1,4 +1,5 @@
 import fnmatch
+import os
 import re
 import sys
 import traceback
@@ -103,3 +104,28 @@ def cleanup_action(cleanup_func):
     except Exception as e:
         print("Error during cleanup: {0}".format(e))
         traceback.print_exc()
+
+
+def get_env(varname, default_value=None):
+    """
+    Return the value of the environment variable or default_value.
+
+    This is a helper method that wraps ``os.environ.get`` to ensure type
+    compatibility across py2 and py3. For py2, any value obtained from an
+    environment variable, ensure ``unicode`` type and ``str`` for py3. The
+    casting is done only for string variables.
+
+    :type varname: ``str``
+    :param varname: Name of the environment variable for which to check.
+
+    :param default_value: Return this value is the env var is not found.
+                          Defaults to ``None``.
+
+    :return: Value of the supplied environment if found; value of
+             ``default_value`` otherwise.
+    """
+    value = os.environ.get(varname, default_value)
+    if isinstance(value, six.string_types) and not isinstance(
+            value, six.text_type):
+        return six.u(value)
+    return value

cloudbridge/cloud/base/provider.py

@@ -8,6 +8,8 @@ try:
 except ImportError:  # Python 2
     from ConfigParser import SafeConfigParser as ConfigParser
 
+import six
+
 from cloudbridge.cloud.interfaces import CloudProvider
 from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 from cloudbridge.cloud.interfaces.resources import Configuration
@@ -150,11 +152,15 @@ class BaseCloudProvider(CloudProvider):
         """
         log.debug("Getting config key %s, with supplied default value: %s",
                   key, default_value)
+        value = default_value
         if isinstance(self.config, dict) and self.config.get(key):
-            return self.config.get(key, default_value)
+            value = self.config.get(key, default_value)
         elif hasattr(self.config, key) and getattr(self.config, key):
-            return getattr(self.config, key)
+            value = getattr(self.config, key)
         elif (self._config_parser.has_option(self.PROVIDER_ID, key) and
               self._config_parser.get(self.PROVIDER_ID, key)):
-            return self._config_parser.get(self.PROVIDER_ID, key)
-        return default_value
+            value = self._config_parser.get(self.PROVIDER_ID, key)
+        if isinstance(value, six.string_types) and not isinstance(
+                value, six.text_type):
+            return six.u(value)
+        return value

cloudbridge/cloud/base/resources.py

@@ -888,7 +888,7 @@ class BaseRouter(BaseCloudResource, Router):
 class BaseInternetGateway(BaseCloudResource, BaseObjectLifeCycleMixin,
                           InternetGateway):
 
-    CB_DEFAULT_INET_GATEWAY_NAME = os.environ.get(
+    CB_DEFAULT_INET_GATEWAY_NAME = cb_helpers.get_env(
         'CB_DEFAULT_INET_GATEWAY_NAME', 'cloudbridge-inetgateway')
 
     def __init__(self, provider):

cloudbridge/cloud/providers/aws/provider.py

@@ -1,6 +1,5 @@
 """Provider implementation based on boto library for AWS-compatible clouds."""
 import logging as log
-import os
 
 import boto3
 try:
@@ -12,6 +11,7 @@ except ImportError:
     log.debug('[aws provider] moto library not available!')
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
 from .services import AWSComputeService
@@ -34,9 +34,9 @@ class AWSCloudProvider(BaseCloudProvider):
                                                   'us-east-1')
         self.session_cfg = {
             'aws_access_key_id': self._get_config_value(
-                'aws_access_key', os.environ.get('AWS_ACCESS_KEY', None)),
+                'aws_access_key', get_env('AWS_ACCESS_KEY', None)),
             'aws_secret_access_key': self._get_config_value(
-                'aws_secret_key', os.environ.get('AWS_SECRET_KEY', None)),
+                'aws_secret_key', get_env('AWS_SECRET_KEY', None)),
             'aws_session_token': self._get_config_value(
                 'aws_session_token', None)
         }

cloudbridge/cloud/providers/azure/provider.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import uuid
 
 from msrestazure.azure_exceptions import CloudError
@@ -7,6 +6,7 @@ from msrestazure.azure_exceptions import CloudError
 import tenacity
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 from cloudbridge.cloud.providers.azure.azure_client import AzureClient
 from cloudbridge.cloud.providers.azure.services \
@@ -23,25 +23,23 @@ class AzureCloudProvider(BaseCloudProvider):
         super(AzureCloudProvider, self).__init__(config)
 
         # mandatory config values
-        self.subscription_id = self. \
-            _get_config_value('azure_subscription_id',
-                              os.environ.get('AZURE_SUBSCRIPTION_ID', None))
+        self.subscription_id = self._get_config_value(
+            'azure_subscription_id', get_env('AZURE_SUBSCRIPTION_ID'))
         self.client_id = self._get_config_value(
-            'azure_client_id', os.environ.get('AZURE_CLIENT_ID', None))
+            'azure_client_id', get_env('AZURE_CLIENT_ID', None))
         self.secret = self._get_config_value(
-            'azure_secret', os.environ.get('AZURE_SECRET', None))
+            'azure_secret', get_env('AZURE_SECRET', None))
         self.tenant = self._get_config_value(
-            'azure_tenant', os.environ.get('AZURE_TENANT', None))
+            'azure_tenant', get_env('AZURE_TENANT', None))
 
         # optional config values
         self.access_token = self._get_config_value(
-            'azure_access_token', os.environ.get('AZURE_ACCESS_TOKEN', None))
+            'azure_access_token', get_env('AZURE_ACCESS_TOKEN', None))
         self.region_name = self._get_config_value(
-            'azure_region_name', os.environ.get('AZURE_REGION_NAME',
-                                                'eastus'))
+            'azure_region_name', get_env('AZURE_REGION_NAME', 'eastus'))
         self.resource_group = self._get_config_value(
-            'azure_resource_group', os.environ.get('AZURE_RESOURCE_GROUP',
-                                                   'cloudbridge'))
+            'azure_resource_group', get_env('AZURE_RESOURCE_GROUP',
+                                            'cloudbridge'))
         # Storage account name is limited to a max length of 24 alphanum chars
         # and unique across all of Azure. Thus, a uuid is used to generate a
         # unique name for the Storage Account based on the resource group,
@@ -49,19 +47,19 @@ class AzureCloudProvider(BaseCloudProvider):
         # having the same resource group name do not have the same SA name.
         self.storage_account = self._get_config_value(
             'azure_storage_account',
-            os.environ.get(
+            get_env(
                 'AZURE_STORAGE_ACCOUNT',
                 'storacc' + self.subscription_id[-6:] +
                 str(uuid.uuid5(uuid.NAMESPACE_OID,
                                str(self.resource_group)))[-6:]))
 
         self.vm_default_user_name = self._get_config_value(
-            'azure_vm_default_user_name', os.environ.get
-            ('AZURE_VM_DEFAULT_USER_NAME', 'cbuser'))
+            'azure_vm_default_user_name', get_env(
+                'AZURE_VM_DEFAULT_USER_NAME', 'cbuser'))
 
         self.public_key_storage_table_name = self._get_config_value(
-            'azure_public_key_storage_table_name', os.environ.get
-            ('AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME', 'cbcerts'))
+            'azure_public_key_storage_table_name', get_env(
+                'AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME', 'cbcerts'))
 
         self._azure_client = None
 

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,7 +1,6 @@
 """Provider implementation based on OpenStack Python clients for OpenStack."""
 
 import inspect
-import os
 
 from cinderclient import client as cinder_client
 
@@ -19,6 +18,7 @@ from openstack import connection
 from swiftclient import client as swift_client
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.base.helpers import get_env
 
 from .services import OpenStackComputeService
 from .services import OpenStackNetworkingService
@@ -36,21 +36,22 @@ class OpenStackCloudProvider(BaseCloudProvider):
 
         # Initialize cloud connection fields
         self.username = self._get_config_value(
-            'os_username', os.environ.get('OS_USERNAME', None))
+            'os_username', get_env('OS_USERNAME', None))
         self.password = self._get_config_value(
-            'os_password', os.environ.get('OS_PASSWORD', None))
+            'os_password', get_env('OS_PASSWORD', None))
         self.project_name = self._get_config_value(
-            'os_project_name', os.environ.get('OS_PROJECT_NAME', None) or
-            os.environ.get('OS_TENANT_NAME', None))
+            'os_project_name', get_env('OS_PROJECT_NAME', None)
+            or get_env('OS_TENANT_NAME', None))
         self.auth_url = self._get_config_value(
-            'os_auth_url', os.environ.get('OS_AUTH_URL', None))
+            'os_auth_url', get_env('OS_AUTH_URL', None))
         self.region_name = self._get_config_value(
-            'os_region_name', os.environ.get('OS_REGION_NAME', None))
+            'os_region_name', get_env('OS_REGION_NAME', None))
         self.project_domain_name = self._get_config_value(
             'os_project_domain_name',
-            os.environ.get('OS_PROJECT_DOMAIN_NAME', None))
+            get_env('OS_PROJECT_DOMAIN_NAME', None))
         self.user_domain_name = self._get_config_value(
-            'os_user_domain_name', os.environ.get('OS_USER_DOMAIN_NAME', None))
+            'os_user_domain_name',
+            get_env('OS_USER_DOMAIN_NAME', None))
 
         # Service connections, lazily initialized
         self._nova = None
@@ -191,10 +192,10 @@ class OpenStackCloudProvider(BaseCloudProvider):
 
         api_version = self._get_config_value(
             'os_compute_api_version',
-            os.environ.get('OS_COMPUTE_API_VERSION', 2))
+            get_env('OS_COMPUTE_API_VERSION', 2))
         service_name = self._get_config_value(
             'nova_service_name',
-            os.environ.get('NOVA_SERVICE_NAME', None))
+            get_env('NOVA_SERVICE_NAME', None))
 
         if self.config.debug_mode:
             nova_shell.OpenStackComputeShell().setup_debugging(True)
@@ -232,7 +233,7 @@ class OpenStackCloudProvider(BaseCloudProvider):
         """Get an OpenStack Cinder (block storage) client object."""
         api_version = self._get_config_value(
             'os_volume_api_version',
-            os.environ.get('OS_VOLUME_API_VERSION', 2))
+            get_env('OS_VOLUME_API_VERSION', 2))
 
         return cinder_client.Client(api_version,
                                     auth_url=self.auth_url,
@@ -301,9 +302,9 @@ class OpenStackCloudProvider(BaseCloudProvider):
         clean_options = self._clean_options(options,
                                             swift_client.Connection.__init__)
         storage_url = self._get_config_value(
-            'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
+            'os_storage_url', get_env('OS_STORAGE_URL', None))
         auth_token = self._get_config_value(
-            'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
+            'os_auth_token', get_env('OS_AUTH_TOKEN', None))
         if storage_url and auth_token:
             clean_options['preauthurl'] = storage_url
             clean_options['preauthtoken'] = auth_token

docs/topics/release_process.rst

@@ -1,15 +1,18 @@
 Release Process
 ~~~~~~~~~~~~~~~
 
-1. Increment version number in cloudbridge/__init__.py as per semver rules.
+1. Increment version number in ``cloudbridge/__init__.py`` as per
+   `semver rules <https://semver.org/>_.
 
-2. Freeze all library dependencies in setup.py. The version numbers can be a range
-   with the upper limit being the latest known working version, and the lowest being
-   the last known working version. 
+2. Freeze all library dependencies in ``setup.py``. The version numbers can be
+   a range with the upper limit being the latest known working version, and the
+   lowest being the last known working version.
 
-3. Run all tox tests.
+3. Run all ``tox`` tests.
 
-4. Add release notes to CHANGELOG.rst. Also add last commit hash to changelog.
+4. Add release notes to ``CHANGELOG.rst``. Also add last commit hash to
+   changelog. List of commits can be obtained using
+   ``git shortlog <last release hash>..HEAD``
 
 5. Release to PyPi
 
@@ -18,4 +21,4 @@ Release Process
    python setup.py sdist upload
    python setup.py bdist_wheel upload
 
-6. Tag release and make github release.
+6. Tag release and make GitHub release.

test/helpers/__init__.py

@@ -6,8 +6,9 @@ import unittest
 import uuid
 from contextlib import contextmanager
 
-from six import reraise
+import six
 
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.factory import CloudProviderFactory
 from cloudbridge.cloud.interfaces import InstanceState
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
@@ -47,7 +48,7 @@ def cleanup_action(cleanup_func):
         except Exception as e:
             print("Error during exception cleanup: {0}".format(e))
             traceback.print_exc()
-        reraise(ex_class, ex_val, ex_traceback)
+        six.reraise(ex_class, ex_val, ex_traceback)
     try:
         cleanup_func()
     except Exception as e:
@@ -80,9 +81,9 @@ def skipIfNoService(services):
 TEST_DATA_CONFIG = {
     "AWSCloudProvider": {
         # Match the ami value with entry in custom_amis.json for use with moto
-        "image": os.environ.get('CB_IMAGE_AWS', 'ami-aa2ea6d0'),
-        "vm_type": os.environ.get('CB_VM_TYPE_AWS', 't2.nano'),
-        "placement": os.environ.get('CB_PLACEMENT_AWS', 'us-east-1a'),
+        "image": get_env('CB_IMAGE_AWS', 'ami-aa2ea6d0'),
+        "vm_type": get_env('CB_VM_TYPE_AWS', 't2.nano'),
+        "placement": get_env('CB_PLACEMENT_AWS', 'us-east-1a'),
     },
     "OpenStackCloudProvider": {
         "image": os.environ.get('CB_IMAGE_OS',
@@ -92,12 +93,12 @@ TEST_DATA_CONFIG = {
     },
     "AzureCloudProvider": {
         "placement":
-            os.environ.get('CB_PLACEMENT_AZURE', 'eastus'),
+            get_env('CB_PLACEMENT_AZURE', 'eastus'),
         "image":
-            os.environ.get('CB_IMAGE_AZURE',
-                           'Canonical:UbuntuServer:16.04.0-LTS:latest'),
+            get_env('CB_IMAGE_AZURE',
+                    'Canonical:UbuntuServer:16.04.0-LTS:latest'),
         "vm_type":
-            os.environ.get('CB_VM_TYPE_AZURE', 'Basic_A2'),
+            get_env('CB_VM_TYPE_AZURE', 'Basic_A2'),
     }
 }
 
@@ -227,7 +228,7 @@ class ProviderTestBase(unittest.TestCase):
             return 1
 
     def create_provider_instance(self):
-        provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
+        provider_name = get_env("CB_TEST_PROVIDER", "aws")
         use_mock_drivers = parse_bool(
             os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
         factory = CloudProviderFactory()

test/test_cloud_helpers.py

@@ -1,5 +1,8 @@
 import itertools
 
+import six
+
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.base.resources import ClientPagedResultList
 from cloudbridge.cloud.base.resources import ServerPagedResultList
 
@@ -75,3 +78,27 @@ class CloudHelpersTestCase(ProviderTestBase):
                         " lists should return True for server paging.")
         with self.assertRaises(NotImplementedError):
             results.data
+
+    def test_type_validation(self):
+        """
+        Make sure internal type checking implementation properly sets types.
+        """
+        self.provider.config['text_type_check'] = 'test-text'
+        config_value = self.provider._get_config_value('text_type_check', None)
+        self.assertIsInstance(config_value, six.string_types)
+
+        env_value = self.provider._get_config_value(
+            'some_config_value', get_env('MOTO_AMIS_PATH'))
+        self.assertIsInstance(env_value, six.string_types)
+
+        none_value = self.provider._get_config_value(
+            'some_config_value', get_env('MISSING_ENV', None))
+        self.assertIsNone(none_value)
+
+        bool_value = self.provider._get_config_value(
+            'some_config_value', get_env('MISSING_ENV', True))
+        self.assertIsInstance(bool_value, bool)
+
+        int_value = self.provider._get_config_value(
+            'default_result_limit', None)
+        self.assertIsInstance(int_value, int)