瀏覽代碼

Fix Deployer Manager RPC connection leak

The deployer manager service created new rpc connections on every deployment
check, basically causing an RPC connection leak which also caused task hanging.
This fixes the connection leak from within the deployer service.
Daniel Vincze 9 月之前
父節點
當前提交
def53f841a
共有 2 個文件被更改,包括 3 次插入9 次删除
  1. 1 1
      coriolis/constants.py
  2. 2 8
      coriolis/deployer_manager/rpc/server.py

+ 1 - 1
coriolis/constants.py

@@ -293,7 +293,7 @@ MINION_MACHINE_LOCK_NAME_FORMAT = "minion-pool-%s-machine-%s"
 
 
 EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP = {
 EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP = {
     EXECUTION_TYPE_TRANSFER_EXECUTION: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_EXECUTION: TRANSFER_LOCK_NAME_FORMAT,
-    EXECUTION_TYPE_DEPLOYMENT: TRANSFER_LOCK_NAME_FORMAT,
+    EXECUTION_TYPE_DEPLOYMENT: DEPLOYMENT_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_UPDATE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_UPDATE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_DISKS_DELETE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_DISKS_DELETE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_MINION_POOL_MAINTENANCE: MINION_POOL_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_MINION_POOL_MAINTENANCE: MINION_POOL_LOCK_NAME_FORMAT,

+ 2 - 8
coriolis/deployer_manager/rpc/server.py

@@ -54,7 +54,7 @@ class DeployerManagerServerEndpoint:
             LOG.info(
             LOG.info(
                 f"Deployment '{deployment_id}' is still '{deployment_status}'")
                 f"Deployment '{deployment_id}' is still '{deployment_status}'")
             i += 1
             i += 1
-            time.sleep(5)
+            time.sleep(1)
 
 
         raise exception.InvalidDeploymentState(
         raise exception.InvalidDeploymentState(
             f"Timed out waiting for deployment '{deployment_id}' to be out of "
             f"Timed out waiting for deployment '{deployment_id}' to be out of "
@@ -120,7 +120,6 @@ class DeployerManagerServerEndpoint:
                 self._admin_ctx, deployment_id, str(ex))
                 self._admin_ctx, deployment_id, str(ex))
 
 
     def _loop(self):
     def _loop(self):
-        greenthreads = []
         while True:
         while True:
             try:
             try:
                 deployments = self._rpc_conductor_client.get_deployments(
                 deployments = self._rpc_conductor_client.get_deployments(
@@ -128,12 +127,7 @@ class DeployerManagerServerEndpoint:
                     include_task_info=False)
                     include_task_info=False)
                 for d in deployments:
                 for d in deployments:
                     if d['last_execution_status'] == PENDING_STATUS:
                     if d['last_execution_status'] == PENDING_STATUS:
-                        greenthreads.append(
-                            eventlet.spawn(
-                                self._check_deployer_status, d['id']))
-
-                for gt in greenthreads:
-                    gt.wait()
+                        self._check_deployer_status(d['id'])
             except Exception:
             except Exception:
                 LOG.warning(
                 LOG.warning(
                     f"Deployer manager failed to list pending deployments. "
                     f"Deployer manager failed to list pending deployments. "