Merge pull request #92 from gabriel-samfira/add-diagnostics

Add diagnostics

coriolis/api/v1/diagnostics.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import logging
+
+from coriolis import exception
+from coriolis.api import wsgi as api_wsgi
+from coriolis.api.v1.views import diagnostic_view
+from coriolis.diagnostics import api
+from coriolis.policies import diagnostics
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DiagnosticsController(api_wsgi.Controller):
+    def __init__(self):
+        self._diag_api = api.API()
+        super(DiagnosticsController, self).__init__()
+
+    def index(self, req):
+        context = req.environ['coriolis.context']
+        context.can(
+            diagnostics.get_diagnostics_policy_label("get"))
+
+        return diagnostic_view.collection(
+            req, self._diag_api.get(context))
+
+
+def create_resource():
+    return api_wsgi.Resource(DiagnosticsController())

coriolis/api/v1/router.py

@@ -4,6 +4,7 @@
 from oslo_log import log as logging
 
 from coriolis import api
+from coriolis.api.v1 import diagnostics
 from coriolis.api.v1 import endpoint_actions
 from coriolis.api.v1 import endpoint_destination_options
 from coriolis.api.v1 import endpoint_instances
@@ -147,3 +148,8 @@ class APIRouter(api.APIRouter):
                         controller=self.resources['replica_schedules'],
                         collection={'index': 'GET'},
                         member={'action': 'POST'})
+
+        diag = diagnostics.create_resource()
+        self.resources['diagnostics'] = diag
+        mapper.resource('diagnostics', 'diagnostics',
+                        controller=self.resources['diagnostics'])

coriolis/api/v1/views/diagnostic_view.py

@@ -0,0 +1,10 @@
+# Copyright 2019 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+
+def single(req, diag):
+    return {"diagnostic":  diag}
+
+
+def collection(req, diag):
+    return {'diagnostics': diag}

coriolis/conductor/rpc/client.py

@@ -292,3 +292,9 @@ class ConductorClient(object):
             ctxt, 'update_replica',
             replica_id=replica_id,
             properties=properties)
+
+    def get_diagnostics(self, ctxt):
+        return self._client.call(ctxt, 'get_diagnostics')
+
+    def get_all_diagnostics(self, ctxt):
+        return self._client.call(ctxt, 'get_all_diagnostics')

coriolis/conductor/rpc/server.py

@@ -105,6 +105,16 @@ class ConductorServerEndpoint(object):
         self._rpc_worker_client = rpc_worker_client.WorkerClient()
         self._replica_cron_client = rpc_cron_client.ReplicaCronClient()
 
+    def get_all_diagnostics(self, ctxt):
+        conductor = self.get_diagnostics(ctxt)
+        cron = self._replica_cron_client.get_diagnostics(ctxt)
+        worker = self._rpc_worker_client.get_diagnostics(ctxt)
+        return [
+            conductor,
+            cron,
+            worker,
+        ]
+
     def _check_delete_reservation_for_transfer(self, transfer_action):
         action_id = transfer_action.base_id
         if not self._licensing_client:
@@ -1294,3 +1304,6 @@ class ConductorServerEndpoint(object):
                   execution.id)
         self._begin_tasks(ctxt, execution, replica.info)
         return self.get_replica_tasks_execution(ctxt, replica_id, execution.id)
+
+    def get_diagnostics(self, ctxt):
+        return utils.get_diagnostics_info()

coriolis/diagnostics/api.py

@@ -0,0 +1,18 @@
+from coriolis import utils
+
+from coriolis.conductor.rpc import client as conductor_rpc
+from coriolis.replica_cron.rpc import client as cron_rpc
+from coriolis.worker.rpc import client as worker_rpc
+
+
+class API(object):
+    def __init__(self):
+        self._conductor_cli = conductor_rpc.ConductorClient()
+        self._cron_cli = cron_rpc.ReplicaCronClient()
+        self._worker_cli = worker_rpc.WorkerClient()
+
+    def get(self, ctxt):
+        diag = self._conductor_cli.get_all_diagnostics(ctxt)
+        api = utils.get_diagnostics_info()
+        diag.append(api)
+        return diag

coriolis/policies/diagnostics.py

@@ -0,0 +1,35 @@
+# Copyright 2018 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from oslo_policy import policy
+
+from coriolis.policies import base
+
+
+DIAGNOSTICS_POLICY_PREFIX = "%s:diagnostics" % (
+    base.CORIOLIS_POLICIES_PREFIX)
+DIAGNOSTICS_POLICY_DEFAULT_RULE = "rule:admin_or_owner"
+
+
+def get_diagnostics_policy_label(rule_label):
+    return "%s:%s" % (
+        DIAGNOSTICS_POLICY_PREFIX, rule_label)
+
+
+DIAGNOSTICS_POLICY_DEFAULT_RULES = [
+    policy.DocumentedRuleDefault(
+        get_diagnostics_policy_label('get'),
+        DIAGNOSTICS_POLICY_DEFAULT_RULE,
+        "Get diagnostics information from coriolis components",
+        [
+            {
+                "path": "/diagnostics",
+                "method": "GET"
+            }
+        ]
+    )
+]
+
+
+def list_rules():
+    return DIAGNOSTICS_POLICY_DEFAULT_RULES

coriolis/policy.py

@@ -10,6 +10,7 @@ from oslo_policy import policy
 from coriolis import exception
 from coriolis import utils
 from coriolis.policies import base
+from coriolis.policies import diagnostics
 from coriolis.policies import endpoints
 from coriolis.policies import general
 from coriolis.policies import migrations
@@ -25,7 +26,7 @@ _ENFORCER = None
 
 DEFAULT_POLICIES_MODULES = [
     base, endpoints, general, migrations, replicas, replica_schedules,
-    replica_tasks_executions]
+    replica_tasks_executions, diagnostics]
 
 
 def reset():

coriolis/replica_cron/rpc/client.py

@@ -19,3 +19,6 @@ class ReplicaCronClient(object):
 
     def unregister(self, ctxt, schedule):
         self._client.call(ctxt, 'unregister', schedule=schedule)
+    
+    def get_diagnostics(self, ctxt):
+        return self._client.call(ctxt, 'get_diagnostics')

coriolis/replica_cron/rpc/server.py

@@ -3,7 +3,7 @@ import json
 from coriolis.conductor.rpc import client as rpc_client
 from coriolis import context
 from coriolis import exception
-# from coriolis import utils
+from coriolis import utils
 from coriolis.replica_cron import cron
 
 from oslo_log import log as logging
@@ -89,3 +89,6 @@ class ReplicaCronServerEndpoint(object):
         schedule_id = schedule["id"]
         LOG.debug("removing schedule %s" % schedule_id)
         self._cron.unregister(schedule_id)
+
+    def get_diagnostics(self, ctxt):
+        return utils.get_diagnostics_info()

coriolis/utils.py

@@ -11,6 +11,7 @@ import io
 import json
 import os
 import pickle
+import platform
 import re
 import socket
 import string
@@ -18,6 +19,7 @@ import subprocess
 import time
 import traceback
 import uuid
+import __main__ as main
 
 from io import StringIO
 
@@ -25,7 +27,15 @@ import OpenSSL
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+
+import netifaces
 import paramiko
+# NOTE(gsamfira): I am aware that this is not ideal, but pip
+# developers have decided to move all logic inside an _internal
+# package, and I really don't want to do an exec call to pip
+# just to get installed packages and their versions, when I can
+# simply call a function.
+from pip._internal.operations import freeze
 from six.moves.urllib import parse
 from webob import exc
 
@@ -77,6 +87,46 @@ exec %(cmdline)s
 """
 
 
+def _get_local_ips():
+    ifaces = netifaces.interfaces()
+    ret = []
+    for iface in ifaces:
+        if iface == "lo":
+            continue
+        addrs = netifaces.ifaddresses(iface)
+        ret.append(
+            {
+                iface: {
+                    "ipv4": addrs.get(netifaces.AF_INET),
+                    "ipv6": addrs.get(netifaces.AF_INET6),
+                },
+            }
+        )
+    return ret
+
+
+def _get_host_os_info():
+    info = None
+    # This exists on all modern Linux systems
+    if os.path.isfile("/etc/os-release"):
+        with open("/etc/os-release") as fd:
+            info = fd.read().split('\n')
+    return info
+
+
+def get_diagnostics_info():
+    # TODO(gsamfira): decide if we want any other kind of
+    # diagnostics.
+    packages = list(freeze.freeze())
+    return {
+        "application": os.path.basename(main.__file__),
+        "packages": packages,
+        "os_info": _get_host_os_info(),
+        "hostname": platform.node(),
+        "ip_addresses": _get_local_ips(),
+    }
+
+
 def setup_logging():
     logging.setup(CONF, 'coriolis')
 
@@ -394,40 +444,6 @@ def to_dict(obj, max_depth=10):
     return jsonutils.loads(jsonutils.dumps(obj, default=_to_primitive))
 
 
-def topological_graph_sorting(items, id="id", depends_on="depends_on",
-                              sort_key=None):
-    """Kahn's algorithm"""
-    if sort_key:
-        # Sort siblings
-        items = sorted(items, key=lambda t: t[sort_key], reverse=True)
-
-    a = []
-    for i in items:
-        a.append({"id": i[id],
-                  "depends_on": list(i[depends_on] or []),
-                  "item": i})
-
-    s = []
-    l = []
-    for n in a:
-        if not n["depends_on"]:
-            s.append(n)
-    while s:
-        n = s.pop()
-        l.append(n["item"])
-
-        for m in a:
-            if n["id"] in m["depends_on"]:
-                m["depends_on"].remove(n["id"])
-                if not m["depends_on"]:
-                    s.append(m)
-
-    if len(l) != len(a):
-        raise ValueError("The graph contains cycles")
-
-    return l
-
-
 def load_class(class_path):
     LOG.debug('Loading class \'%s\'' % class_path)
     parts = class_path.rsplit('.', 1)

coriolis/worker/rpc/client.py

@@ -125,3 +125,6 @@ class WorkerClient(object):
             ctxt, 'get_provider_schemas',
             platform_name=platform_name,
             provider_type=provider_type)
+
+    def get_diagnostics(self, ctxt):
+        return self._client.call(ctxt, 'get_diagnostics')

coriolis/worker/rpc/server.py

@@ -429,6 +429,9 @@ class WorkerServerEndpoint(object):
             schemas["source_environment_schema"] = schema
 
         return schemas
+    
+    def get_diagnostics(self, ctxt):
+        return utils.get_diagnostics_info()
 
 
 def _get_task_export_path(task_id, create=False):