Ensure type consistency for config env vars

Resolves #140

cloudbridge/cloud/base/helpers.py

@@ -1,3 +1,4 @@
+import os
 import sys
 import traceback
 from contextlib import contextmanager
@@ -6,7 +7,7 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization as crypt_serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 
-from six import reraise
+import six
 
 
 def generate_key_pair():
@@ -87,9 +88,34 @@ def cleanup_action(cleanup_func):
         except Exception as e:
             print("Error during exception cleanup: {0}".format(e))
             traceback.print_exc()
-        reraise(ex_class, ex_val, ex_traceback)
+        six.reraise(ex_class, ex_val, ex_traceback)
     try:
         cleanup_func()
     except Exception as e:
         print("Error during cleanup: {0}".format(e))
         traceback.print_exc()
+
+
+def get_env(varname, default_value=None):
+    """
+    Return the value of the environment variable or default_value.
+
+    This is a helper method that wraps ``os.environ.get`` to ensure type
+    compatibility across py2 and py3. For py2, any value obtained from an
+    environment variable, ensure ``unicode`` type and ``str`` for py3. The
+    casting is done only for string variables.
+
+    :type varname: ``str``
+    :param varname: Name of the environment variable for which to check.
+
+    :param default_value: Return this value is the env var is not found.
+                          Defaults to ``None``.
+
+    :return: Value of the supplied environment if found; value of
+             ``default_value`` otherwise.
+    """
+    value = os.environ.get(varname, default_value)
+    if isinstance(value, six.string_types) and not isinstance(
+            value, six.text_type):
+        return six.u(value)
+    return value

cloudbridge/cloud/base/resources.py

@@ -4,7 +4,6 @@ Base implementation for data objects exposed through a provider or service
 import inspect
 import itertools
 import logging
-import os
 import re
 import shutil
 import time
@@ -922,8 +921,8 @@ class BaseGatewayContainer(GatewayContainer, BasePageableObjectMixin):
 
 class BaseNetwork(BaseCloudResource, BaseObjectLifeCycleMixin, Network):
 
-    CB_DEFAULT_NETWORK_NAME = os.environ.get('CB_DEFAULT_NETWORK_NAME',
-                                             'cloudbridge-net')
+    CB_DEFAULT_NETWORK_NAME = cb_helpers.get_env('CB_DEFAULT_NETWORK_NAME',
+                                                 'cloudbridge-net')
 
     def __init__(self, provider):
         super(BaseNetwork, self).__init__(provider)
@@ -952,8 +951,8 @@ class BaseNetwork(BaseCloudResource, BaseObjectLifeCycleMixin, Network):
 
 class BaseSubnet(BaseCloudResource, BaseObjectLifeCycleMixin, Subnet):
 
-    CB_DEFAULT_SUBNET_NAME = os.environ.get('CB_DEFAULT_SUBNET_NAME',
-                                            'cloudbridge-subnet')
+    CB_DEFAULT_SUBNET_NAME = cb_helpers.get_env('CB_DEFAULT_SUBNET_NAME',
+                                                'cloudbridge-subnet')
 
     def __init__(self, provider):
         super(BaseSubnet, self).__init__(provider)
@@ -1033,8 +1032,8 @@ class BaseFloatingIP(BaseCloudResource, BaseObjectLifeCycleMixin, FloatingIP):
 
 class BaseRouter(BaseCloudResource, Router):
 
-    CB_DEFAULT_ROUTER_NAME = os.environ.get('CB_DEFAULT_ROUTER_NAME',
-                                            'cloudbridge-router')
+    CB_DEFAULT_ROUTER_NAME = cb_helpers.get_env('CB_DEFAULT_ROUTER_NAME',
+                                                'cloudbridge-router')
 
     def __init__(self, provider):
         super(BaseRouter, self).__init__(provider)
@@ -1053,7 +1052,7 @@ class BaseRouter(BaseCloudResource, Router):
 class BaseInternetGateway(BaseCloudResource, BaseObjectLifeCycleMixin,
                           InternetGateway):
 
-    CB_DEFAULT_INET_GATEWAY_NAME = os.environ.get(
+    CB_DEFAULT_INET_GATEWAY_NAME = cb_helpers.get_env(
         'CB_DEFAULT_INET_GATEWAY_NAME', 'cloudbridge-inetgateway')
 
     def __init__(self, provider):

cloudbridge/cloud/providers/aws/provider.py

@@ -1,6 +1,5 @@
 """Provider implementation based on boto library for AWS-compatible clouds."""
 import logging as log
-import os
 
 import boto3
 try:
@@ -12,6 +11,7 @@ except ImportError:
     log.debug('[aws provider] moto library not available!')
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
 
 from .services import AWSComputeService
@@ -34,9 +34,9 @@ class AWSCloudProvider(BaseCloudProvider):
                                                   'us-east-1')
         self.session_cfg = {
             'aws_access_key_id': self._get_config_value(
-                'aws_access_key', os.environ.get('AWS_ACCESS_KEY', None)),
+                'aws_access_key', get_env('AWS_ACCESS_KEY', None)),
             'aws_secret_access_key': self._get_config_value(
-                'aws_secret_key', os.environ.get('AWS_SECRET_KEY', None)),
+                'aws_secret_key', get_env('AWS_SECRET_KEY', None)),
             'aws_session_token': self._get_config_value(
                 'aws_session_token', None)
         }

cloudbridge/cloud/providers/azure/provider.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import uuid
 
 from msrestazure.azure_exceptions import CloudError
@@ -7,6 +6,7 @@ from msrestazure.azure_exceptions import CloudError
 import tenacity
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.interfaces.exceptions import ProviderConnectionException
 from cloudbridge.cloud.providers.azure.azure_client import AzureClient
 from cloudbridge.cloud.providers.azure.services \
@@ -23,25 +23,23 @@ class AzureCloudProvider(BaseCloudProvider):
         super(AzureCloudProvider, self).__init__(config)
 
         # mandatory config values
-        self.subscription_id = self. \
-            _get_config_value('azure_subscription_id',
-                              os.environ.get('AZURE_SUBSCRIPTION_ID', None))
+        self.subscription_id = self._get_config_value(
+            'azure_subscription_id', get_env('AZURE_SUBSCRIPTION_ID'))
         self.client_id = self._get_config_value(
-            'azure_client_id', os.environ.get('AZURE_CLIENT_ID', None))
+            'azure_client_id', get_env('AZURE_CLIENT_ID', None))
         self.secret = self._get_config_value(
-            'azure_secret', os.environ.get('AZURE_SECRET', None))
+            'azure_secret', get_env('AZURE_SECRET', None))
         self.tenant = self._get_config_value(
-            'azure_tenant', os.environ.get('AZURE_TENANT', None))
+            'azure_tenant', get_env('AZURE_TENANT', None))
 
         # optional config values
         self.access_token = self._get_config_value(
-            'azure_access_token', os.environ.get('AZURE_ACCESS_TOKEN', None))
+            'azure_access_token', get_env('AZURE_ACCESS_TOKEN', None))
         self.region_name = self._get_config_value(
-            'azure_region_name', os.environ.get('AZURE_REGION_NAME',
-                                                'eastus'))
+            'azure_region_name', get_env('AZURE_REGION_NAME', 'eastus'))
         self.resource_group = self._get_config_value(
-            'azure_resource_group', os.environ.get('AZURE_RESOURCE_GROUP',
-                                                   'cloudbridge'))
+            'azure_resource_group', get_env('AZURE_RESOURCE_GROUP',
+                                            'cloudbridge'))
         # Storage account name is limited to a max length of 24 alphanum chars
         # and unique across all of Azure. Thus, a uuid is used to generate a
         # unique name for the Storage Account based on the resource group,
@@ -49,19 +47,19 @@ class AzureCloudProvider(BaseCloudProvider):
         # having the same resource group name do not have the same SA name.
         self.storage_account = self._get_config_value(
             'azure_storage_account',
-            os.environ.get(
+            get_env(
                 'AZURE_STORAGE_ACCOUNT',
                 'storacc' + self.subscription_id[-6:] +
                 str(uuid.uuid5(uuid.NAMESPACE_OID,
                                str(self.resource_group)))[-6:]))
 
         self.vm_default_user_name = self._get_config_value(
-            'azure_vm_default_user_name', os.environ.get
-            ('AZURE_VM_DEFAULT_USER_NAME', 'cbuser'))
+            'azure_vm_default_user_name', get_env(
+                'AZURE_VM_DEFAULT_USER_NAME', 'cbuser'))
 
         self.public_key_storage_table_name = self._get_config_value(
-            'azure_public_key_storage_table_name', os.environ.get
-            ('AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME', 'cbcerts'))
+            'azure_public_key_storage_table_name', get_env(
+                'AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME', 'cbcerts'))
 
         self._azure_client = None
 

cloudbridge/cloud/providers/openstack/provider.py

@@ -1,7 +1,6 @@
 """Provider implementation based on OpenStack Python clients for OpenStack."""
 
 import inspect
-import os
 
 from cinderclient import client as cinder_client
 
@@ -19,6 +18,7 @@ from openstack import connection
 from swiftclient import client as swift_client
 
 from cloudbridge.cloud.base import BaseCloudProvider
+from cloudbridge.cloud.base.helpers import get_env
 
 from .services import OpenStackComputeService
 from .services import OpenStackNetworkingService
@@ -36,21 +36,22 @@ class OpenStackCloudProvider(BaseCloudProvider):
 
         # Initialize cloud connection fields
         self.username = self._get_config_value(
-            'os_username', os.environ.get('OS_USERNAME', None))
+            'os_username', get_env('OS_USERNAME', None))
         self.password = self._get_config_value(
-            'os_password', os.environ.get('OS_PASSWORD', None))
+            'os_password', get_env('OS_PASSWORD', None))
         self.project_name = self._get_config_value(
-            'os_project_name', os.environ.get('OS_PROJECT_NAME', None) or
-            os.environ.get('OS_TENANT_NAME', None))
+            'os_project_name', get_env('OS_PROJECT_NAME', None)
+            or get_env('OS_TENANT_NAME', None))
         self.auth_url = self._get_config_value(
-            'os_auth_url', os.environ.get('OS_AUTH_URL', None))
+            'os_auth_url', get_env('OS_AUTH_URL', None))
         self.region_name = self._get_config_value(
-            'os_region_name', os.environ.get('OS_REGION_NAME', None))
+            'os_region_name', get_env('OS_REGION_NAME', None))
         self.project_domain_name = self._get_config_value(
             'os_project_domain_name',
-            os.environ.get('OS_PROJECT_DOMAIN_NAME', None))
+            get_env('OS_PROJECT_DOMAIN_NAME', None))
         self.user_domain_name = self._get_config_value(
-            'os_user_domain_name', os.environ.get('OS_USER_DOMAIN_NAME', None))
+            'os_user_domain_name',
+            get_env('OS_USER_DOMAIN_NAME', None))
 
         # Service connections, lazily initialized
         self._nova = None
@@ -191,10 +192,10 @@ class OpenStackCloudProvider(BaseCloudProvider):
 
         api_version = self._get_config_value(
             'os_compute_api_version',
-            os.environ.get('OS_COMPUTE_API_VERSION', 2))
+            get_env('OS_COMPUTE_API_VERSION', 2))
         service_name = self._get_config_value(
             'nova_service_name',
-            os.environ.get('NOVA_SERVICE_NAME', None))
+            get_env('NOVA_SERVICE_NAME', None))
 
         if self.config.debug_mode:
             nova_shell.OpenStackComputeShell().setup_debugging(True)
@@ -232,7 +233,7 @@ class OpenStackCloudProvider(BaseCloudProvider):
         """Get an OpenStack Cinder (block storage) client object."""
         api_version = self._get_config_value(
             'os_volume_api_version',
-            os.environ.get('OS_VOLUME_API_VERSION', 2))
+            get_env('OS_VOLUME_API_VERSION', 2))
 
         return cinder_client.Client(api_version,
                                     auth_url=self.auth_url,
@@ -301,9 +302,9 @@ class OpenStackCloudProvider(BaseCloudProvider):
         clean_options = self._clean_options(options,
                                             swift_client.Connection.__init__)
         storage_url = self._get_config_value(
-            'os_storage_url', os.environ.get('OS_STORAGE_URL', None))
+            'os_storage_url', get_env('OS_STORAGE_URL', None))
         auth_token = self._get_config_value(
-            'os_auth_token', os.environ.get('OS_AUTH_TOKEN', None))
+            'os_auth_token', get_env('OS_AUTH_TOKEN', None))
         if storage_url and auth_token:
             clean_options['preauthurl'] = storage_url
             clean_options['preauthtoken'] = auth_token

test/helpers/__init__.py

@@ -6,8 +6,9 @@ import unittest
 import uuid
 from contextlib import contextmanager
 
-from six import reraise
+import six
 
+from cloudbridge.cloud.base.helpers import get_env
 from cloudbridge.cloud.factory import CloudProviderFactory
 from cloudbridge.cloud.interfaces import InstanceState
 from cloudbridge.cloud.interfaces import TestMockHelperMixin
@@ -47,7 +48,7 @@ def cleanup_action(cleanup_func):
         except Exception as e:
             print("Error during exception cleanup: {0}".format(e))
             traceback.print_exc()
-        reraise(ex_class, ex_val, ex_traceback)
+        six.reraise(ex_class, ex_val, ex_traceback)
     try:
         cleanup_func()
     except Exception as e:
@@ -80,24 +81,24 @@ def skipIfNoService(services):
 TEST_DATA_CONFIG = {
     "AWSCloudProvider": {
         # Match the ami value with entry in custom_amis.json for use with moto
-        "image": os.environ.get('CB_IMAGE_AWS', 'ami-aa2ea6d0'),
-        "vm_type": os.environ.get('CB_VM_TYPE_AWS', 't2.nano'),
-        "placement": os.environ.get('CB_PLACEMENT_AWS', 'us-east-1a'),
+        "image": get_env('CB_IMAGE_AWS', 'ami-aa2ea6d0'),
+        "vm_type": get_env('CB_VM_TYPE_AWS', 't2.nano'),
+        "placement": get_env('CB_PLACEMENT_AWS', 'us-east-1a'),
     },
     "OpenStackCloudProvider": {
-        "image": os.environ.get('CB_IMAGE_OS',
-                                'acb53109-941f-4593-9bf8-4a53cb9e0739'),
-        "vm_type": os.environ.get('CB_VM_TYPE_OS', 'm1.tiny'),
-        "placement": os.environ.get('CB_PLACEMENT_OS', 'zone-r1'),
+        "image": get_env('CB_IMAGE_OS',
+                         'acb53109-941f-4593-9bf8-4a53cb9e0739'),
+        "vm_type": get_env('CB_VM_TYPE_OS', 'm1.tiny'),
+        "placement": get_env('CB_PLACEMENT_OS', 'zone-r1'),
     },
     "AzureCloudProvider": {
         "placement":
-            os.environ.get('CB_PLACEMENT_AZURE', 'eastus'),
+            get_env('CB_PLACEMENT_AZURE', 'eastus'),
         "image":
-            os.environ.get('CB_IMAGE_AZURE',
-                           'Canonical:UbuntuServer:16.04.0-LTS:latest'),
+            get_env('CB_IMAGE_AZURE',
+                    'Canonical:UbuntuServer:16.04.0-LTS:latest'),
         "vm_type":
-            os.environ.get('CB_VM_TYPE_AZURE', 'Basic_A2'),
+            get_env('CB_VM_TYPE_AZURE', 'Basic_A2'),
     }
 }
 
@@ -234,7 +235,7 @@ class ProviderTestBase(unittest.TestCase):
             return 1
 
     def create_provider_instance(self):
-        provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
+        provider_name = get_env("CB_TEST_PROVIDER", "aws")
         use_mock_drivers = parse_bool(
             os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
         factory = CloudProviderFactory()