Merge pull request #275 from CloudVE/update_packages_09_21

Upgrade dependencies to latest

cloudbridge/providers/gcp/helpers.py

@@ -1,5 +1,12 @@
+import binascii
+import collections
+import datetime
+import hashlib
 import re
 
+import six
+from six.moves.urllib.parse import quote
+
 from googleapiclient.errors import HttpError
 
 import tenacity
@@ -186,3 +193,87 @@ def change_label(resource, key, value, res_att, request):
             response, zone=getattr(resource, 'zone_name', None))
     finally:
         resource.refresh()
+
+
+# https://cloud.google.com/storage/docs/access-control/signing-urls-manually#python-sample
+def generate_signed_url(credentials, bucket_name, object_name,
+                        subresource=None, expiration=604800, http_method='GET',
+                        query_parameters=None, headers=None):
+
+    if expiration > 604800:
+        # max allowed expiration time is 7 days
+        expiration = 604800
+
+    escaped_object_name = quote(six.ensure_binary(object_name), safe=b'/~')
+    canonical_uri = '/{}'.format(escaped_object_name)
+
+    datetime_now = datetime.datetime.utcnow()
+    request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ')
+    datestamp = datetime_now.strftime('%Y%m%d')
+
+    client_email = credentials.service_account_email
+    credential_scope = '{}/auto/storage/goog4_request'.format(datestamp)
+    credential = '{}/{}'.format(client_email, credential_scope)
+
+    if headers is None:
+        headers = dict()
+    host = '{}.storage.googleapis.com'.format(bucket_name)
+    headers['host'] = host
+
+    canonical_headers = ''
+    ordered_headers = collections.OrderedDict(sorted(headers.items()))
+    for k, v in ordered_headers.items():
+        lower_k = str(k).lower()
+        strip_v = str(v).lower()
+        canonical_headers += '{}:{}\n'.format(lower_k, strip_v)
+
+    signed_headers = ''
+    for k, _ in ordered_headers.items():
+        lower_k = str(k).lower()
+        signed_headers += '{};'.format(lower_k)
+    signed_headers = signed_headers[:-1]  # remove trailing ';'
+
+    if query_parameters is None:
+        query_parameters = dict()
+    query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256'
+    query_parameters['X-Goog-Credential'] = credential
+    query_parameters['X-Goog-Date'] = request_timestamp
+    query_parameters['X-Goog-Expires'] = expiration
+    query_parameters['X-Goog-SignedHeaders'] = signed_headers
+    if subresource:
+        query_parameters[subresource] = ''
+
+    canonical_query_string = ''
+    ordered_query_parameters = collections.OrderedDict(
+        sorted(query_parameters.items()))
+    for k, v in ordered_query_parameters.items():
+        encoded_k = quote(str(k), safe='')
+        encoded_v = quote(str(v), safe='')
+        canonical_query_string += '{}={}&'.format(encoded_k, encoded_v)
+    canonical_query_string = canonical_query_string[:-1]  # remove trailing '&'
+
+    canonical_request = '\n'.join([http_method,
+                                   canonical_uri,
+                                   canonical_query_string,
+                                   canonical_headers,
+                                   signed_headers,
+                                   'UNSIGNED-PAYLOAD'])
+
+    canonical_request_hash = hashlib.sha256(
+        canonical_request.encode()).hexdigest()
+
+    string_to_sign = '\n'.join(['GOOG4-RSA-SHA256',
+                                request_timestamp,
+                                credential_scope,
+                                canonical_request_hash])
+
+    # signer.sign() signs using RSA-SHA256 with PKCS1v15 padding
+    signature = binascii.hexlify(
+        credentials.signer.sign(string_to_sign)
+    ).decode()
+
+    scheme_and_host = '{}://{}'.format('https', host)
+    signed_url = '{}{}?{}&x-goog-signature={}'.format(
+        scheme_and_host, canonical_uri, canonical_query_string, signature)
+
+    return signed_url

cloudbridge/providers/gcp/provider.py

@@ -9,6 +9,7 @@ import re
 import time
 from string import Template
 
+import google.auth
 import google_auth_httplib2
 
 import googleapiclient
@@ -16,9 +17,6 @@ from googleapiclient import discovery
 
 import httplib2
 
-from oauth2client.client import GoogleCredentials
-from oauth2client.service_account import ServiceAccountCredentials
-
 from google.auth.credentials import with_scopes_if_required
 
 from google.oauth2.service_account import Credentials
@@ -330,25 +328,19 @@ class GCPCloudProvider(BaseCloudProvider):
     def _credentials(self):
         if not self.credentials_obj:
             if self.credentials_dict:
-                self.credentials_obj = (
-                    ServiceAccountCredentials.from_json_keyfile_dict(
-                        self.credentials_dict))
+                self.credentials_obj = Credentials.from_service_account_info(
+                    self.credentials_dict)
             else:
-                self.credentials_obj = (
-                    GoogleCredentials.get_application_default())
+                self.credentials_obj, _ = google.auth.default()
         return self.credentials_obj
 
-    def sign_blob(self, string_to_sign):
-        return self._credentials.sign_blob(string_to_sign)[1]
-
     @property
     def client_id(self):
         return self._credentials.service_account_email
 
     def _get_build_request(self):
-        credentials = Credentials.from_service_account_info(
-            self.credentials_dict)
-        credentials = with_scopes_if_required(credentials, list(CLOUD_SCOPES))
+        credentials = with_scopes_if_required(
+            self._credentials, list(CLOUD_SCOPES))
 
         # FROM: https://github.com/googleapis/google-api-python-client/blob/
         # master/docs/thread_safety.md

cloudbridge/providers/gcp/resources.py

@@ -1,15 +1,12 @@
 """
 DataTypes used by this provider
 """
-import base64
-import calendar
 import hashlib
 import inspect
 import io
 import logging
 import math
 import re
-import time
 import uuid
 from collections import namedtuple
 
@@ -1982,18 +1979,10 @@ class GCPBucketObject(BaseBucketObject):
         """
         Generates a signed URL accessible to everyone.
         """
-        expiration = calendar.timegm(time.gmtime()) + expires_in
-        signed_signature = self._provider.sign_blob(
-            'GET\n\n\n%d\n/%s/%s' %
-            (expiration, self._obj['bucket'], self.name))
-        encoded_signature = base64.b64encode(signed_signature).decode("utf-8")
-        url_encoded_signature = (encoded_signature.replace('+', '%2B')
-                                                  .replace('/', '%2F'))
-        return ('https://storage.googleapis.com/%s/%s?GoogleAccessId=%s'
-                '&Expires=%d&Signature=%s' % (self._obj['bucket'], self.name,
-                                              self._provider.client_id,
-                                              expiration,
-                                              url_encoded_signature))
+        # pylint:disable=protected-access
+        return helpers.generate_signed_url(
+            self._provider._credentials, self._obj['bucket'], self.name,
+            expiration=expires_in)
 
     def refresh(self):
         # pylint:disable=protected-access

setup.py

@@ -25,34 +25,33 @@ REQS_BASE = [
     'pyeventsystem<2'
 ]
 REQS_AWS = [
-    'boto3>=1.9.86,<1.20'
+    'boto3>=1.9.86,<2.0.0'
 ]
 # Install azure>=3.0.0 package to find which of the azure libraries listed
 # below are compatible with each other. List individual libraries instead
 # of using the azure umbrella package to speed up installation.
 REQS_AZURE = [
-    'msrestazure==0.6.4',
-    'azure-identity==1.7.0',
-    'azure-common==1.1.27',
-    'azure-mgmt-devtestlabs==9.0.0',
-    'azure-mgmt-resource==19.0.0',
-    'azure-mgmt-compute==23.1.0',
-    'azure-mgmt-network==19.2.0',
-    'azure-mgmt-storage==19.0.0',
-    'azure-storage-blob==12.9.0',
-    'azure-cosmosdb-table==1.0.6',
-    'pysftp==0.2.9'
+    'msrestazure<1.0.0',
+    'azure-identity<2.0.0',
+    'azure-common<2.0.0',
+    'azure-mgmt-devtestlabs<10.0.0',
+    'azure-mgmt-resource<20.0.0',
+    'azure-mgmt-compute<24.0.0',
+    'azure-mgmt-network<20.0.0',
+    'azure-mgmt-storage<20.0.0',
+    'azure-storage-blob<13.0.0',
+    'azure-cosmosdb-table<2.0.0',
+    'pysftp<1.0.0'
 ]
 REQS_GCP = [
-    'google-api-python-client>=1.7.8,<1.13',
-    'oauth2client<4.2'
+    'google-api-python-client>=2.0,<3.0.0'
 ]
 REQS_OPENSTACK = [
-    'openstacksdk>=0.12.0,<0.53',
-    'python-novaclient>=7.0.0,<17.3',
-    'python-swiftclient>=3.2.0,<3.11',
-    'python-neutronclient>=6.0.0,<7.3',
-    'python-keystoneclient>=3.13.0,<4.2'
+    'openstacksdk>=0.12.0,<1.0.0',
+    'python-novaclient>=7.0.0,<18.0',
+    'python-swiftclient>=3.2.0,<4.0',
+    'python-neutronclient>=6.0.0,<8.0',
+    'python-keystoneclient>=3.13.0,<5.0'
 ]
 REQS_SIMPLE = REQS_BASE + REQS_AWS + REQS_GCP + REQS_OPENSTACK
 REQS_AZURE = REQS_SIMPLE + REQS_AZURE