Просмотр исходного кода

Fix Deployer Manager RPC connection leak

The deployer manager service created new rpc connections on every deployment
check, basically causing an RPC connection leak which also caused task hanging.
This fixes the connection leak from within the deployer service.
Daniel Vincze 6 месяцев назад
Родитель
Сommit
def53f841a
2 измененных файлов с 3 добавлено и 9 удалено
  1. 1 1
      coriolis/constants.py
  2. 2 8
      coriolis/deployer_manager/rpc/server.py

+ 1 - 1
coriolis/constants.py

@@ -293,7 +293,7 @@ MINION_MACHINE_LOCK_NAME_FORMAT = "minion-pool-%s-machine-%s"
 
 
 EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP = {
 EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP = {
     EXECUTION_TYPE_TRANSFER_EXECUTION: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_EXECUTION: TRANSFER_LOCK_NAME_FORMAT,
-    EXECUTION_TYPE_DEPLOYMENT: TRANSFER_LOCK_NAME_FORMAT,
+    EXECUTION_TYPE_DEPLOYMENT: DEPLOYMENT_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_UPDATE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_UPDATE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_DISKS_DELETE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_TRANSFER_DISKS_DELETE: TRANSFER_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_MINION_POOL_MAINTENANCE: MINION_POOL_LOCK_NAME_FORMAT,
     EXECUTION_TYPE_MINION_POOL_MAINTENANCE: MINION_POOL_LOCK_NAME_FORMAT,

+ 2 - 8
coriolis/deployer_manager/rpc/server.py

@@ -54,7 +54,7 @@ class DeployerManagerServerEndpoint:
             LOG.info(
             LOG.info(
                 f"Deployment '{deployment_id}' is still '{deployment_status}'")
                 f"Deployment '{deployment_id}' is still '{deployment_status}'")
             i += 1
             i += 1
-            time.sleep(5)
+            time.sleep(1)
 
 
         raise exception.InvalidDeploymentState(
         raise exception.InvalidDeploymentState(
             f"Timed out waiting for deployment '{deployment_id}' to be out of "
             f"Timed out waiting for deployment '{deployment_id}' to be out of "
@@ -120,7 +120,6 @@ class DeployerManagerServerEndpoint:
                 self._admin_ctx, deployment_id, str(ex))
                 self._admin_ctx, deployment_id, str(ex))
 
 
     def _loop(self):
     def _loop(self):
-        greenthreads = []
         while True:
         while True:
             try:
             try:
                 deployments = self._rpc_conductor_client.get_deployments(
                 deployments = self._rpc_conductor_client.get_deployments(
@@ -128,12 +127,7 @@ class DeployerManagerServerEndpoint:
                     include_task_info=False)
                     include_task_info=False)
                 for d in deployments:
                 for d in deployments:
                     if d['last_execution_status'] == PENDING_STATUS:
                     if d['last_execution_status'] == PENDING_STATUS:
-                        greenthreads.append(
-                            eventlet.spawn(
-                                self._check_deployer_status, d['id']))
-
-                for gt in greenthreads:
-                    gt.wait()
+                        self._check_deployer_status(d['id'])
             except Exception:
             except Exception:
                 LOG.warning(
                 LOG.warning(
                     f"Deployer manager failed to list pending deployments. "
                     f"Deployer manager failed to list pending deployments. "