Merge pull request #269 from CloudVE/add_github_actions

Add GitHub actions

.github/workflows/integration.yaml

@@ -0,0 +1,118 @@
+name: Integration
+
+# Run this workflow every time a new commit pushed to your repository
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  lint:
+    name: Lint code
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.8' ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+           python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip dir
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-lint
+
+      - name: Install required packages
+        run: pip install tox
+
+      - name: Run tox
+        run: tox -e lint
+
+  integration:
+    # Name the Job
+    name: Per-cloud integration tests
+    needs: lint
+    # Set the type of machine to run on
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8']
+        cloud-provider: ['aws', 'azure', 'gcp', 'mock', 'openstack']
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+           python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip dir
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('**/setup.py', '**/requirements.txt') }}
+
+      - name: Install required packages
+        run: pip install tox
+
+      - name: Run tox
+        run: tox -e py${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+        env:
+          PYTHONUNBUFFERED: "True"
+          # aws
+          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
+          # azure
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          AZURE_SECRET: ${{ secrets.AZURE_SECRET }}
+          AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
+          AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }}
+          AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+          CB_IMAGE_AZURE: ${{ secrets.CB_IMAGE_AZURE }}
+          CB_VM_TYPE_AZURE: ${{ secrets.CB_VM_TYPE_AZURE }}
+          # gcp
+          GCP_SERVICE_CREDS_DICT: ${{ secrets.GCP_SERVICE_CREDS_DICT }}
+          CB_IMAGE_GCP: ${{ secrets.CB_IMAGE_GCP }}
+          # openstack
+          OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }}
+          OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
+          OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
+          OS_PROJECT_DOMAIN_NAME: ${{ secrets.OS_PROJECT_DOMAIN_NAME }}
+          OS_TENANT_NAME: ${{ secrets.OS_TENANT_NAME }}
+          OS_USERNAME: ${{ secrets.OS_USERNAME }}
+          OS_REGION_NAME: ${{ secrets.OS_REGION_NAME }}
+          OS_USER_DOMAIN_NAME: ${{ secrets.OS_USER_DOMAIN_NAME }}
+          CB_IMAGE_OS: ${{ secrets.CB_IMAGE_OS }}
+          CB_PLACEMENT_OS: ${{ secrets.CB_PLACEMENT_OS }}
+
+      - name: Coveralls
+        uses: AndreMiras/coveralls-python-action@develop
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          flag-name: run-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+          parallel: true
+
+  finish:
+    needs: integration
+    runs-on: ubuntu-latest
+    steps:
+    - name: Coveralls Finished
+      uses: AndreMiras/coveralls-python-action@develop
+      with:
+        github-token: ${{ secrets.github_token }}
+        parallel-finished: true

.travis.yml

@@ -1,82 +0,0 @@
-language: python
-cache:
-  directories:
-  - "$HOME/.cache/pip"
-  - "$TRAVIS_BUILD_DIR/.tox"
-os:
-- linux
-matrix:
-  fast_finish: true
-  allow_failures:
-  - os: osx
-  include:
-  - python: 3.6
-    env: TOX_ENV=py36-aws
-  - python: 3.6
-    env: TOX_ENV=py36-azure
-  - python: 3.6
-    env: TOX_ENV=py36-gcp
-  - python: 3.6
-    env: TOX_ENV=py36-mock
-  - python: 3.6
-    env: TOX_ENV=py36-openstack
-env:
-  global:
-    - PYTHONUNBUFFERED=True
-    - COVERALLS_PARALLEL=true
-before_install:
-    - |
-      case "$TRAVIS_EVENT_TYPE" in
-        push|pull_request)
-           # Check whether we need to run a test for this provider
-           DOCS_REGEX='(\.rst$)|(^(docs))/'
-           FILES_IN_CHANGESET="`git diff --name-only $TRAVIS_COMMIT_RANGE`"
-           echo "$FILES_IN_CHANGESET" | grep -qvE "$DOCS_REGEX" || {
-              echo "Only docs were updated. Stopping build process."
-              exit
-           }
-           echo "$FILES_IN_CHANGESET" | grep -qvE "$DOCS_REGEX|(^(cloudbridge/cloud/providers))" || {
-              echo "Only docs and providers were updated. Checking whether this provider was changed."
-              # Extract env and provider from $TOXENV into $PYENV and $PROVIDER respectively
-              IFS=- read PYENV PROVIDER <<< "$TOX_ENV"
-              echo "$FILES_IN_CHANGESET" | grep -qE "^(cloudbridge/cloud/providers/$PROVIDER)" && {
-                 echo "This provider was affected by this changeset. Running tests."
-              } || {
-                 echo "This provider was not affected by this changeset. Stopping build process."
-                 exit
-              }
-           }
-           ;;
-        *)
-           echo "Build triggered through API or CRON job. Running regardless of changes."
-           ;;
-      esac
-install:
-    - pip install -U pip
-    - pip install -U setuptools
-    - pip install "coverage<5"
-    - pip install tox
-    - pip install coveralls
-    - pip install codecov
-script:
-    - tox -e $TOX_ENV
-after_script:
-    - |
-      case "$TRAVIS_EVENT_TYPE" in
-        push|pull_request)
-           # Don't run coverage if tests or cloudbridge interface was not affected
-           DOCS_REGEX='(\.rst$)|(^(docs))/'
-           FILES_IN_CHANGESET="`git diff --name-only $TRAVIS_COMMIT_RANGE`"
-           echo "$FILES_IN_CHANGESET" | grep -qvE "$DOCS_REGEX|(^(cloudbridge/cloud/providers))" && {
-              coveralls; codecov;
-           } || {
-              echo "Only docs and providers were updated. Not running coverage."
-           }
-           ;;
-        *)
-           echo "Build triggered through API or CRON job. Running regardless of changes"
-           coveralls; codecov;
-           ;;
-      esac
-notifications:
-    webhooks: https://coveralls.io/webhook

README.rst

@@ -9,6 +9,9 @@ Detailed documentation can be found at http://cloudbridge.cloudve.org.
 
 Build Status Tests
 ~~~~~~~~~~~~~~~~~~
+.. image:: https://github.com/CloudVE/cloudbridge/workflows/Integration/badge.svg
+   :target: https://github.com/CloudVE/cloudbridge/actions/
+   :alt: Integration Tests
 
 .. image:: https://codecov.io/gh/CloudVE/cloudbridge/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/CloudVE/cloudbridge

cloudbridge/providers/gcp/provider.py

@@ -9,16 +9,18 @@ import re
 import time
 from string import Template
 
+import google_auth_httplib2
+
 import googleapiclient
 from googleapiclient import discovery
 
-import google_auth_httplib2
 import httplib2
 
 from oauth2client.client import GoogleCredentials
 from oauth2client.service_account import ServiceAccountCredentials
 
 from google.auth.credentials import with_scopes_if_required
+
 from google.oauth2.service_account import Credentials
 
 from cloudbridge.base import BaseCloudProvider
@@ -34,6 +36,7 @@ log = logging.getLogger(__name__)
 
 CLOUD_SCOPES = ['https://www.googleapis.com/auth/cloud-platform']
 
+
 class GCPResourceUrl(object):
 
     def __init__(self, resource, connection):
@@ -201,7 +204,6 @@ class GCPResources(object):
 
 
 class GCPCloudProvider(BaseCloudProvider):
-
     PROVIDER_ID = 'gcp'
 
     def __init__(self, config):
@@ -210,15 +212,15 @@ class GCPCloudProvider(BaseCloudProvider):
         # Disable warnings about file_cache not being available when using
         # oauth2client >= 4.0.0.
         logging.getLogger('googleapiclient.discovery_cache').setLevel(
-                logging.ERROR)
+            logging.ERROR)
 
         # Initialize cloud connection fields
         self.credentials_file = self._get_config_value(
-                'gcp_service_creds_file',
-                os.getenv('GCP_SERVICE_CREDS_FILE'))
+            'gcp_service_creds_file',
+            os.getenv('GCP_SERVICE_CREDS_FILE'))
         self.credentials_dict = self._get_config_value(
-                'gcp_service_creds_dict',
-                json.loads(os.getenv('GCP_SERVICE_CREDS_DICT', '{}')))
+            'gcp_service_creds_dict',
+            json.loads(os.getenv('GCP_SERVICE_CREDS_DICT', '{}')))
         self.credentials_obj = self._get_config_value('gcp_credentials_obj')
         self.vm_default_user_name = self._get_config_value(
             'gcp_vm_default_username',
@@ -304,10 +306,10 @@ class GCPCloudProvider(BaseCloudProvider):
     def _compute_resources(self):
         if not self._compute_resources_cache:
             self._compute_resources_cache = GCPResources(
-                    self.gcp_compute,
-                    project=self.project_name,
-                    region=self.region_name,
-                    zone=self.zone_name)
+                self.gcp_compute,
+                project=self.project_name,
+                region=self.region_name,
+                zone=self.zone_name)
         return self._compute_resources_cache
 
     @property
@@ -329,11 +331,11 @@ class GCPCloudProvider(BaseCloudProvider):
         if not self.credentials_obj:
             if self.credentials_dict:
                 self.credentials_obj = (
-                        ServiceAccountCredentials.from_json_keyfile_dict(
-                                self.credentials_dict))
+                    ServiceAccountCredentials.from_json_keyfile_dict(
+                        self.credentials_dict))
             else:
                 self.credentials_obj = (
-                        GoogleCredentials.get_application_default())
+                    GoogleCredentials.get_application_default())
         return self.credentials_obj
 
     def sign_blob(self, string_to_sign):
@@ -344,13 +346,18 @@ class GCPCloudProvider(BaseCloudProvider):
         return self._credentials.service_account_email
 
     def _get_build_request(self):
-        credentials = Credentials.from_service_account_info(self.credentials_dict)
+        credentials = Credentials.from_service_account_info(
+            self.credentials_dict)
         credentials = with_scopes_if_required(credentials, list(CLOUD_SCOPES))
-        # FROM: https://github.com/googleapis/google-api-python-client/blob/master/docs/thread_safety.md
+
+        # FROM: https://github.com/googleapis/google-api-python-client/blob/
+        # master/docs/thread_safety.md
         # Create a new Http() object for every request
         def build_request(http, *args, **kwargs):
-          new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=httplib2.Http())
-          return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)
+            new_http = google_auth_httplib2.AuthorizedHttp(
+                credentials, http=httplib2.Http())
+            return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)
+
         return build_request
 
     def _connect_gcp_storage(self):
@@ -399,12 +406,12 @@ class GCPCloudProvider(BaseCloudProvider):
         if not url_or_name:
             return None
         resource_url = (
-            self._compute_resources.get_resource_url_with_default(
-                resource, url_or_name, **kwargs) or
-            self._storage_resources.get_resource_url_with_default(
-                resource, url_or_name, **kwargs) or
-            self._dns_resources.get_resource_url_with_default(
-                resource, url_or_name, **kwargs))
+                self._compute_resources.get_resource_url_with_default(
+                    resource, url_or_name, **kwargs) or
+                self._storage_resources.get_resource_url_with_default(
+                    resource, url_or_name, **kwargs) or
+                self._dns_resources.get_resource_url_with_default(
+                    resource, url_or_name, **kwargs))
         if resource_url is None:
             return None
         try:

cloudbridge/providers/mock/provider.py

@@ -10,8 +10,6 @@ from moto import mock_ec2
 from moto import mock_route53
 from moto import mock_s3
 
-import responses
-
 from ..aws import AWSCloudProvider
 from ...interfaces.provider import TestMockHelperMixin
 

requirements.txt

@@ -1 +1,4 @@
+# needed by moto
+sshpubkeys
+git+https://github.com/CloudVE/moto@fix_unknown_instance_type
 -e ".[dev]"

tests/test_cloud_helpers.py

@@ -86,11 +86,6 @@ class CloudHelpersTestCase(ProviderTestBase):
         config_value = self.provider._get_config_value('text_type_check', None)
         self.assertIsInstance(config_value, six.string_types)
 
-        # pylint:disable=protected-access
-        env_value = self.provider._get_config_value(
-            'some_config_value', get_env('MOTO_AMIS_PATH'))
-        self.assertIsInstance(env_value, six.string_types)
-
         # pylint:disable=protected-access
         none_value = self.provider._get_config_value(
             'some_config_value', get_env('MISSING_ENV', None))

tox.ini

@@ -6,14 +6,12 @@
 # running the tests.
 
 [tox]
-envlist = {py38,pypy}-{aws,azure,gcp,openstack,mock}
+envlist = {py3.8,pypy}-{aws,azure,gcp,openstack,mock},lint
 
 [testenv]
-commands = flake8 cloudbridge tests setup.py
-           # see setup.cfg for options sent to nosetests and coverage
+commands = # see setup.cfg for options sent to nosetests and coverage
            nosetests -v --nocapture --nologcapture --logging-format='%(asctime)s [%(levelname)s] %(name)s: %(message)s' {posargs}
 setenv =
-    MOTO_AMIS_PATH=./tests/fixtures/custom_amis.json
     # Fix for moto import issue: https://github.com/travis-ci/travis-ci/issues/7940
     BOTO_CONFIG=/dev/null
     aws: CB_TEST_PROVIDER=aws
@@ -21,6 +19,8 @@ setenv =
     gcp: CB_TEST_PROVIDER=gcp
     openstack: CB_TEST_PROVIDER=openstack
     mock: CB_TEST_PROVIDER=mock
+    # https://github.com/nedbat/coveragepy/issues/883#issuecomment-650562896
+    COVERAGE_FILE=.coverage.{envname}
 passenv =
     PYTHONUNBUFFERED
     aws: CB_IMAGE_AWS CB_INSTANCE_TYPE_AWS CB_PLACEMENT_AWS AWS_ACCESS_KEY AWS_SECRET_KEY
@@ -31,3 +31,7 @@ passenv =
 deps =
     -rrequirements.txt
     coverage<5
+
+[testenv:lint]
+commands = flake8 cloudbridge tests setup.py
+deps = flake8