Type the GCP provider (pragmatic tier) + reconcile its inconsistencies

Annotate cloudbridge/providers/gcp/ (~415 callables) and add providers.gcp.*
to the pragmatic mypy tier. Conform implementations to the corrected interface
and apply the user's decisions for the new inconsistencies GCP surfaced.

Interface/base widenings (genuinely-optional values that providers expose):
- VMFirewallRule.protocol -> str | None (a rule may cover all protocols).
- Instance.key_pair_id -> str | None (an instance may have no key pair).
- AttachmentInfo.device -> str | None (GCP disks expose no device name);
  BaseAttachmentInfo accepts device: str | None.
- Router.subnets -> Iterable[Subnet] (was list[Subnet]) so a provider may
  return its SubnetSubService; AWS/OpenStack lists remain valid covariant
  returns.

GCP conformance:
- Required getters raise ProviderInternalException when absent
  (Instance.vm_type, Instance.image_id) rather than returning None.
- create_image and all *Service.create error paths raise instead of returning
  None (a create returns the resource or raises).
- Implement GCPInstance.start() (mirrors stop()); it was missing, which had
  forced type: ignore[abstract] at the instantiation sites.
- start/stop/reboot/delete/attach*/detach* -> None; upload* -> BucketObject;
  find() wraps in ClientPagedResultList; DnsRecord.data -> list[str];
  Volume.attachments -> AttachmentInfo | None.
- GCP-specific provider members reached via cast("GCPCloudProvider", ...).

Verified: mypy green (43 files), flake8 clean, GCP modules import under the GCP
SDK env. (No AWS runtime change this round; interface/base edits are
annotation-only.)

cloudbridge/base/resources.py

@@ -468,7 +468,8 @@ class BaseMachineImage(
 
 class BaseAttachmentInfo(AttachmentInfo):
 
-    def __init__(self, volume: Volume, instance_id: str, device: str) -> None:
+    def __init__(self, volume: Volume, instance_id: str,
+                 device: str | None) -> None:
         self._volume = volume
         self._instance_id = instance_id
         self._device = device
@@ -482,7 +483,7 @@ class BaseAttachmentInfo(AttachmentInfo):
         return self._instance_id
 
     @property
-    def device(self) -> str:
+    def device(self) -> str | None:
         return self._device
 
 

cloudbridge/interfaces/resources.py

@@ -691,7 +691,7 @@ class Instance(ObjectLifeCycleMixin, LabeledCloudResource):
         pass
 
     @abstractproperty
-    def key_pair_id(self) -> str:
+    def key_pair_id(self) -> str | None:
         """
         Get the id of the key pair associated with this instance.
 
@@ -1268,7 +1268,7 @@ class Router(LabeledCloudResource):
         pass
 
     @abstractproperty
-    def subnets(self) -> list[Subnet]:
+    def subnets(self) -> Iterable[Subnet]:
         """
         List of subnets attached to this router.
 
@@ -1490,7 +1490,7 @@ class AttachmentInfo(object):
         pass
 
     @abstractproperty
-    def device(self) -> str:
+    def device(self) -> str | None:
         """
         Get the device the volume is mapped as.
 
@@ -2082,7 +2082,7 @@ class VMFirewallRule(CloudResource):
         pass
 
     @abstractproperty
-    def protocol(self) -> str:
+    def protocol(self) -> str | None:
         """
         IP protocol used. Either ``tcp`` | ``udp`` | ``icmp``.
 

cloudbridge/providers/gcp/helpers.py

@@ -3,6 +3,10 @@ import collections
 import datetime
 import hashlib
 import re
+from typing import Any
+from typing import Callable
+from typing import Iterator
+from typing import TYPE_CHECKING
 from urllib.parse import quote
 
 from googleapiclient.errors import HttpError
@@ -11,12 +15,15 @@ import tenacity
 
 from cloudbridge.interfaces.exceptions import ProviderInternalException
 
+if TYPE_CHECKING:
+    from .provider import GCPCloudProvider
 
-def gcp_projects(provider):
+
+def gcp_projects(provider: "GCPCloudProvider") -> Any:
     return provider.gcp_compute.projects()
 
 
-def iter_all(resource, **kwargs):
+def iter_all(resource: Any, **kwargs: Any) -> Iterator[Any]:
     token = None
     while True:
         response = resource.list(pageToken=token, **kwargs).execute()
@@ -27,7 +34,7 @@ def iter_all(resource, **kwargs):
         token = response['nextPageToken']
 
 
-def get_common_metadata(provider):
+def get_common_metadata(provider: "GCPCloudProvider") -> Any:
     """
     Get a project's commonInstanceMetadata entry
     """
@@ -36,7 +43,7 @@ def get_common_metadata(provider):
     return metadata["commonInstanceMetadata"]
 
 
-def __if_fingerprint_differs(e):
+def __if_fingerprint_differs(e: BaseException) -> bool:
     # return True if the CloudError exception is due to subnet being in use
     if isinstance(e, HttpError):
         expected_message = 'Supplied fingerprint does not match current ' \
@@ -50,7 +57,8 @@ def __if_fingerprint_differs(e):
                 retry=tenacity.retry_if_exception(__if_fingerprint_differs),
                 wait=tenacity.wait_exponential(max=10),
                 reraise=True)
-def gcp_metadata_save_op(provider, callback):
+def gcp_metadata_save_op(provider: "GCPCloudProvider",
+                         callback: Callable[[Any], Any]) -> None:
     """
     Carries out a metadata save operation. In GCP, a fingerprint based
     locking mechanism is used to prevent lost updates. A new fingerprint
@@ -59,7 +67,7 @@ def gcp_metadata_save_op(provider, callback):
     metadata, and saves the metadata using the original fingerprint
     immediately afterwards, ensuring that update conflicts can be detected.
     """
-    def _save_common_metadata(provider):
+    def _save_common_metadata(provider: "GCPCloudProvider") -> None:
         # get the latest metadata (so we get the latest fingerprint)
         metadata = get_common_metadata(provider)
         # allow callback to do processing on it
@@ -73,8 +81,9 @@ def gcp_metadata_save_op(provider, callback):
     _save_common_metadata(provider)
 
 
-def modify_or_add_metadata_item(provider, key, value):
-    def _update_metadata_key(metadata):
+def modify_or_add_metadata_item(provider: "GCPCloudProvider", key: str,
+                                value: str) -> None:
+    def _update_metadata_key(metadata: Any) -> None:
         entries = [item for item in metadata.get('items', [])
                    if item['key'] == key]
         if entries:
@@ -92,8 +101,9 @@ def modify_or_add_metadata_item(provider, key, value):
 # This function will raise an HttpError with message containing
 # "Metadata has duplicate key" if it's not unique, unlike the previous
 # method which either adds or updates the value corresponding to that key
-def add_metadata_item(provider, key, value):
-    def _add_metadata_key(metadata):
+def add_metadata_item(provider: "GCPCloudProvider", key: str,
+                      value: str) -> None:
+    def _add_metadata_key(metadata: Any) -> None:
         entry = {'key': key, 'value': value}
         entries = metadata.get('items', [])
         entries.append(entry)
@@ -104,7 +114,9 @@ def add_metadata_item(provider, key, value):
     gcp_metadata_save_op(provider, _add_metadata_key)
 
 
-def find_matching_metadata_items(provider, key_regex):
+def find_matching_metadata_items(provider: "GCPCloudProvider",
+                                 key_regex: str | re.Pattern[str]
+                                 ) -> list[Any]:
     metadata = get_common_metadata(provider)
     items = metadata.get('items', [])
     if not items:
@@ -113,7 +125,7 @@ def find_matching_metadata_items(provider, key_regex):
             if re.search(key_regex, item['key'])]
 
 
-def get_metadata_item_value(provider, key):
+def get_metadata_item_value(provider: "GCPCloudProvider", key: str) -> Any:
     metadata = get_common_metadata(provider)
     entries = [item['value'] for item in metadata.get('items', [])
                if item['key'] == key]
@@ -123,8 +135,8 @@ def get_metadata_item_value(provider, key):
         return None
 
 
-def remove_metadata_item(provider, key):
-    def _remove_metadata_by_key(metadata):
+def remove_metadata_item(provider: "GCPCloudProvider", key: str) -> bool:
+    def _remove_metadata_by_key(metadata: Any) -> bool | None:
         items = metadata.get('items', [])
         # No metadata to delete
         if not items:
@@ -144,12 +156,13 @@ def remove_metadata_item(provider, key):
 
             else:
                 metadata['items'] = entries
+                return None
 
     gcp_metadata_save_op(provider, _remove_metadata_by_key)
     return True
 
 
-def __if_label_fingerprint_differs(e):
+def __if_label_fingerprint_differs(e: BaseException) -> bool:
     # return True if the CloudError exception is due to subnet being in use
     if isinstance(e, HttpError):
         expected_message = 'Labels fingerprint either invalid or ' \
@@ -164,7 +177,8 @@ def __if_label_fingerprint_differs(e):
                     __if_label_fingerprint_differs),
                 wait=tenacity.wait_exponential(max=10),
                 reraise=True)
-def change_label(resource, key, value, res_att, request):
+def change_label(resource: Any, key: str, value: str, res_att: str,
+                 request: Any) -> None:
     resource.assert_valid_resource_label(value)
     labels = getattr(resource, res_att).get("labels", {})
     labels[key] = str(value)
@@ -188,9 +202,11 @@ def change_label(resource, key, value, res_att, request):
 
 
 # https://cloud.google.com/storage/docs/access-control/signing-urls-manually#python-sample
-def generate_signed_url(credentials, bucket_name, object_name,
-                        subresource=None, expiration=604800, http_method='GET',
-                        query_parameters=None, headers=None):
+def generate_signed_url(credentials: Any, bucket_name: str, object_name: str,
+                        subresource: str | None = None,
+                        expiration: int = 604800, http_method: str = 'GET',
+                        query_parameters: dict[str, Any] | None = None,
+                        headers: dict[str, str] | None = None) -> str:
 
     if expiration > 604800:
         # max allowed expiration time is 7 days

cloudbridge/providers/gcp/provider.py

@@ -8,8 +8,13 @@ import os
 import re
 import time
 from string import Template
+from typing import Any
+from typing import Callable
 
 import google.auth
+from google.auth.credentials import with_scopes_if_required
+from google.oauth2.service_account import Credentials
+
 import google_auth_httplib2
 
 import googleapiclient
@@ -17,12 +22,13 @@ from googleapiclient import discovery
 
 import httplib2
 
-from google.auth.credentials import with_scopes_if_required
-
-from google.oauth2.service_account import Credentials
-
 from cloudbridge.base import BaseCloudProvider
 from cloudbridge.interfaces.exceptions import ProviderConnectionException
+from cloudbridge.interfaces.services import ComputeService
+from cloudbridge.interfaces.services import DnsService
+from cloudbridge.interfaces.services import NetworkingService
+from cloudbridge.interfaces.services import SecurityService
+from cloudbridge.interfaces.services import StorageService
 
 from .services import GCPComputeService
 from .services import GCPDnsService
@@ -37,12 +43,12 @@ CLOUD_SCOPES = ['https://www.googleapis.com/auth/cloud-platform']
 
 class GCPResourceUrl(object):
 
-    def __init__(self, resource, connection):
+    def __init__(self, resource: str, connection: Any) -> None:
         self._resource = resource
         self._connection = connection
-        self.parameters = {}
+        self.parameters: dict[str, Any] = {}
 
-    def get_resource(self):
+    def get_resource(self) -> Any:
         """
         The format of the returned resource is explained in details in
         https://cloud.google.com/compute/docs/reference/latest/ and
@@ -71,7 +77,7 @@ class GCPResourceUrl(object):
 
 class GCPResources(object):
 
-    def __init__(self, connection, **kwargs):
+    def __init__(self, connection: Any, **kwargs: Any) -> None:
         self._connection = connection
         self._parameter_defaults = kwargs
 
@@ -116,7 +122,7 @@ class GCPResources(object):
         self.RESOURCE_REGEX = re.compile(
             r"(https://.*\.googleapis\.com/{0})(.*)".format(
                 desc['servicePath']))
-        self._resources = {}
+        self._resources: dict[str, dict[str, Any]] = {}
 
         # We will not mutate self._desc; it's OK to use items() in Python 2.x.
         for resource, resource_desc in desc['resources'].items():
@@ -149,7 +155,7 @@ class GCPResources(object):
             self._resources[resource] = {'parameters': parameters,
                                          'pattern': re.compile(pattern)}
 
-    def parse_url(self, url):
+    def parse_url(self, url: str) -> "GCPResourceUrl | None":
         """
         Build a GCPResourceUrl from a resource's URL string. One can then call
         the get() method on the returned object to fetch resource details from
@@ -180,8 +186,11 @@ class GCPResources(object):
             for index, parameter in enumerate(desc['parameters']):
                 out.parameters[parameter] = m.group(index + 1)
             return out
+        return None
 
-    def get_resource_url_with_default(self, resource, url_or_name, **kwargs):
+    def get_resource_url_with_default(
+            self, resource: str, url_or_name: str,
+            **kwargs: Any) -> "GCPResourceUrl | None":
         """
         Build a GCPResourceUrl from a service's name and resource url or name.
         If the url_or_name is a valid GCP resource URL, then we build the
@@ -210,7 +219,7 @@ class GCPResources(object):
 class GCPCloudProvider(BaseCloudProvider):
     PROVIDER_ID = 'gcp'
 
-    def __init__(self, config):
+    def __init__(self, config: dict[str, Any]) -> None:
         super(GCPCloudProvider, self).__init__(config)
 
         # Disable warnings about file_cache not being available when using
@@ -248,12 +257,12 @@ class GCPCloudProvider(BaseCloudProvider):
             self.project_name = os.environ.get('GCP_PROJECT_NAME')
 
         # service connections, lazily initialized
-        self._gcp_compute = None
-        self._gcp_storage = None
-        self._gcp_dns = None
-        self._compute_resources_cache = None
-        self._storage_resources_cache = None
-        self._dns_resources_cache = None
+        self._gcp_compute: Any = None
+        self._gcp_storage: Any = None
+        self._gcp_dns: Any = None
+        self._compute_resources_cache: GCPResources | None = None
+        self._storage_resources_cache: GCPResources | None = None
+        self._dns_resources_cache: GCPResources | None = None
 
         # Initialize provider services
         self._compute = GCPComputeService(self)
@@ -265,49 +274,49 @@ class GCPCloudProvider(BaseCloudProvider):
     # Override base class implementation because it will cause
     # an infinite loop
     @property
-    def zone_name(self):
+    def zone_name(self) -> str | None:
         return self._zone_name
 
     @property
-    def compute(self):
+    def compute(self) -> ComputeService:
         return self._compute
 
     @property
-    def networking(self):
+    def networking(self) -> NetworkingService:
         return self._networking
 
     @property
-    def security(self):
+    def security(self) -> SecurityService:
         return self._security
 
     @property
-    def storage(self):
+    def storage(self) -> StorageService:
         return self._storage
 
     @property
-    def dns(self):
+    def dns(self) -> DnsService:
         return self._dns
 
     @property
-    def gcp_compute(self):
+    def gcp_compute(self) -> Any:
         if not self._gcp_compute:
             self._gcp_compute = self._connect_gcp_compute()
         return self._gcp_compute
 
     @property
-    def gcp_storage(self):
+    def gcp_storage(self) -> Any:
         if not self._gcp_storage:
             self._gcp_storage = self._connect_gcp_storage()
         return self._gcp_storage
 
     @property
-    def gcp_dns(self):
+    def gcp_dns(self) -> Any:
         if not self._gcp_dns:
             self._gcp_dns = self._connect_gcp_dns()
         return self._gcp_dns
 
     @property
-    def _compute_resources(self):
+    def _compute_resources(self) -> GCPResources:
         if not self._compute_resources_cache:
             self._compute_resources_cache = GCPResources(
                 self.gcp_compute,
@@ -317,13 +326,13 @@ class GCPCloudProvider(BaseCloudProvider):
         return self._compute_resources_cache
 
     @property
-    def _storage_resources(self):
+    def _storage_resources(self) -> GCPResources:
         if not self._storage_resources_cache:
             self._storage_resources_cache = GCPResources(self.gcp_storage)
         return self._storage_resources_cache
 
     @property
-    def _dns_resources(self):
+    def _dns_resources(self) -> GCPResources:
         if not self._dns_resources_cache:
             self._dns_resources_cache = GCPResources(
                 self.gcp_dns,
@@ -331,7 +340,7 @@ class GCPCloudProvider(BaseCloudProvider):
         return self._dns_resources_cache
 
     @property
-    def _credentials(self):
+    def _credentials(self) -> Any:
         if not self.credentials_obj:
             if self.credentials_dict:
                 self.credentials_obj = Credentials.from_service_account_info(
@@ -341,42 +350,43 @@ class GCPCloudProvider(BaseCloudProvider):
         return self.credentials_obj
 
     @property
-    def client_id(self):
+    def client_id(self) -> str:
         return self._credentials.service_account_email
 
-    def _get_build_request(self):
+    def _get_build_request(self) -> Callable[..., Any]:
         credentials = with_scopes_if_required(
             self._credentials, list(CLOUD_SCOPES))
 
         # FROM: https://github.com/googleapis/google-api-python-client/blob/
         # master/docs/thread_safety.md
         # Create a new Http() object for every request
-        def build_request(http, *args, **kwargs):
+        def build_request(http: Any, *args: Any, **kwargs: Any) -> Any:
             new_http = google_auth_httplib2.AuthorizedHttp(
                 credentials, http=httplib2.Http())
             return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)
 
         return build_request
 
-    def _connect_gcp_storage(self):
+    def _connect_gcp_storage(self) -> Any:
         return discovery.build('storage', 'v1', credentials=self._credentials,
                                cache_discovery=False,
                                requestBuilder=self._get_build_request()
                                )
 
-    def _connect_gcp_compute(self):
+    def _connect_gcp_compute(self) -> Any:
         return discovery.build('compute', 'v1', credentials=self._credentials,
                                cache_discovery=False,
                                requestBuilder=self._get_build_request()
                                )
 
-    def _connect_gcp_dns(self):
+    def _connect_gcp_dns(self) -> Any:
         return discovery.build('dns', 'v1', credentials=self._credentials,
                                cache_discovery=False,
                                requestBuilder=self._get_build_request()
                                )
 
-    def wait_for_operation(self, operation, region=None, zone=None):
+    def wait_for_operation(self, operation: Any, region: str | None = None,
+                           zone: str | None = None) -> Any:
         args = {'project': self.project_name, 'operation': operation['name']}
         if not region and not zone:
             operations = self.gcp_compute.globalOperations()
@@ -396,11 +406,12 @@ class GCPCloudProvider(BaseCloudProvider):
 
             time.sleep(0.5)
 
-    def parse_url(self, url):
+    def parse_url(self, url: str) -> "GCPResourceUrl | None":
         out = self._compute_resources.parse_url(url)
         return out if out else self._storage_resources.parse_url(url)
 
-    def get_resource(self, resource, url_or_name, **kwargs):
+    def get_resource(self, resource: str, url_or_name: str,
+                     **kwargs: Any) -> Any:
         if not url_or_name:
             return None
         resource_url = (
@@ -424,7 +435,7 @@ class GCPCloudProvider(BaseCloudProvider):
             else:
                 raise
 
-    def authenticate(self):
+    def authenticate(self) -> bool:
         try:
             self.gcp_compute
             return True

cloudbridge/providers/gcp/subservices.py

@@ -6,6 +6,12 @@ from cloudbridge.base.subservices import BaseFloatingIPSubService
 from cloudbridge.base.subservices import BaseGatewaySubService
 from cloudbridge.base.subservices import BaseSubnetSubService
 from cloudbridge.base.subservices import BaseVMFirewallRuleSubService
+from cloudbridge.interfaces.provider import CloudProvider
+from cloudbridge.interfaces.resources import Bucket
+from cloudbridge.interfaces.resources import DnsZone
+from cloudbridge.interfaces.resources import Gateway
+from cloudbridge.interfaces.resources import Network
+from cloudbridge.interfaces.resources import VMFirewall
 
 
 log = logging.getLogger(__name__)
@@ -13,34 +19,34 @@ log = logging.getLogger(__name__)
 
 class GCPBucketObjectSubService(BaseBucketObjectSubService):
 
-    def __init__(self, provider, bucket):
+    def __init__(self, provider: CloudProvider, bucket: Bucket) -> None:
         super(GCPBucketObjectSubService, self).__init__(provider, bucket)
 
 
 class GCPGatewaySubService(BaseGatewaySubService):
-    def __init__(self, provider, network):
+    def __init__(self, provider: CloudProvider, network: Network) -> None:
         super(GCPGatewaySubService, self).__init__(provider, network)
 
 
 class GCPVMFirewallRuleSubService(BaseVMFirewallRuleSubService):
 
-    def __init__(self, provider, firewall):
+    def __init__(self, provider: CloudProvider, firewall: VMFirewall) -> None:
         super(GCPVMFirewallRuleSubService, self).__init__(provider, firewall)
 
 
 class GCPFloatingIPSubService(BaseFloatingIPSubService):
 
-    def __init__(self, provider, gateway):
+    def __init__(self, provider: CloudProvider, gateway: Gateway) -> None:
         super(GCPFloatingIPSubService, self).__init__(provider, gateway)
 
 
 class GCPSubnetSubService(BaseSubnetSubService):
 
-    def __init__(self, provider, network):
+    def __init__(self, provider: CloudProvider, network: Network) -> None:
         super(GCPSubnetSubService, self).__init__(provider, network)
 
 
 class GCPDnsRecordSubService(BaseDnsRecordSubService):
 
-    def __init__(self, provider, dns_zone):
+    def __init__(self, provider: CloudProvider, dns_zone: DnsZone) -> None:
         super(GCPDnsRecordSubService, self).__init__(provider, dns_zone)

pyproject.toml

@@ -152,6 +152,7 @@ module = [
     "cloudbridge.providers.aws.*",
     "cloudbridge.providers.mock.*",
     "cloudbridge.providers.openstack.*",
+    "cloudbridge.providers.gcp.*",
 ]
 ignore_errors = false
 warn_return_any = false