2
0
Эх сурвалжийг харах

Fix Deployer Manager RPC connection leak

The deployer manager service created new rpc connections on every deployment
check, basically causing an RPC connection leak which also caused task hanging.
This fixes the connection leak from within the deployer service.
Daniel Vincze 5 сар өмнө
parent
commit
def53f841a

+ 1 - 1
coriolis/constants.py

@@ -293,7 +293,7 @@ MINION_MACHINE_LOCK_NAME_FORMAT = "minion-pool-%s-machine-%s"
 
 EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP = {
     EXECUTION_TYPE_TRANSFER_EXECUTION: TRANSFER_LOCK_NAME_FORMAT,
-    EXECUTION_TYPE_DEPLOYMENT: TRANSFER_LOCK_NAME_FORMAT,
+    EXECUTION_TYPE_DEPLOYMENT: DEPLOYMENT_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_UPDATE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_DISKS_DELETE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_MINION_POOL_MAINTENANCE: MINION_POOL_LOCK_NAME_FORMAT,

+ 2 - 8
coriolis/deployer_manager/rpc/server.py

@@ -54,7 +54,7 @@ class DeployerManagerServerEndpoint:
             LOG.info(
                 f"Deployment '{deployment_id}' is still '{deployment_status}'")
             i += 1
-            time.sleep(5)
+            time.sleep(1)
 
         raise exception.InvalidDeploymentState(
             f"Timed out waiting for deployment '{deployment_id}' to be out of "
@@ -120,7 +120,6 @@ class DeployerManagerServerEndpoint:
                 self._admin_ctx, deployment_id, str(ex))
 
     def _loop(self):
-        greenthreads = []
         while True:
             try:
                 deployments = self._rpc_conductor_client.get_deployments(
@@ -128,12 +127,7 @@ class DeployerManagerServerEndpoint:
                     include_task_info=False)
                 for d in deployments:
                     if d['last_execution_status'] == PENDING_STATUS:
-                        greenthreads.append(
-                            eventlet.spawn(
-                                self._check_deployer_status, d['id']))
-
-                for gt in greenthreads:
-                    gt.wait()
+                        self._check_deployer_status(d['id'])
             except Exception:
                 LOG.warning(
                     f"Deployer manager failed to list pending deployments. "