feat/metrics improvement with avg, min and max (#2618)

* feat: add avg, min, max charts for metrics

* chore: minor fixes

* chore: add aggregated metrics for cluster metrics

Co-authored-by: Soham Parekh <sohamparekh@Sohams-MacBook-Pro-2.local>
Co-authored-by: Soham Parekh <sohamparekh@soham.parekh-macOS>

dashboard/package-lock.json

@@ -1713,6 +1713,12 @@
         "@types/node": "*"
       }
     },
+    "@types/chroma-js": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/@types/chroma-js/-/chroma-js-2.1.4.tgz",
+      "integrity": "sha512-l9hWzP7cp7yleJUI7P2acmpllTJNYf5uU6wh50JzSIZt3fFHe+w2FM6w9oZGBTYzjjm2qHdnQvI+fF/JF/E5jQ==",
+      "dev": true
+    },
     "@types/color": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/color/-/color-3.0.3.tgz",
@@ -3551,6 +3557,11 @@
       "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
       "dev": true
     },
+    "chroma-js": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chroma-js/-/chroma-js-2.4.2.tgz",
+      "integrity": "sha512-U9eDw6+wt7V8z5NncY2jJfZa+hUH8XEj8FQHgFJTrUFnJfXYf4Ml4adI2vXZOjqRDpFWtYVWypDfZwnJ+HIR4A=="
+    },
     "chrome-trace-event": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz",
@@ -8927,6 +8938,11 @@
       "integrity": "sha512-sDl4qMFpijcGw22U5w63KmD3cZJfBuFlVNbVMKje2keoKML7X2UzWbc4XrmEbDwg0NXJc3yv4/ox7b+JWb57kQ==",
       "dev": true
     },
+    "simple-statistics": {
+      "version": "7.8.0",
+      "resolved": "https://registry.npmjs.org/simple-statistics/-/simple-statistics-7.8.0.tgz",
+      "integrity": "sha512-lTWbfJc0u6GZhBojLOrlHJMTHu6PdUjSsYLrpiH902dVBiYJyWlN/LdSoG8b5VvfG1D30gIBgarqMNeNmU5nAA=="
+    },
     "simple-swizzle": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",

dashboard/package.json

@@ -25,6 +25,7 @@
     "anser": "^2.0.1",
     "axios": "^0.21.2",
     "brace": "^0.11.1",
+    "chroma-js": "^2.4.2",
     "clipboard": "^2.0.8",
     "cohere-js": "^1.0.19",
     "cohere-sentry": "^1.0.1",
@@ -60,6 +61,7 @@
     "react-transition-group": "^4.4.2",
     "regenerator-runtime": "^0.13.9",
     "semver": "^7.3.5",
+    "simple-statistics": "^7.8.0",
     "stacktrace-js": "^2.0.2",
     "styled-components": "^5.2.0",
     "traverse": "^0.6.7",
@@ -81,6 +83,7 @@
     "@testing-library/jest-dom": "^4.2.4",
     "@testing-library/react": "^9.3.2",
     "@testing-library/user-event": "^7.1.2",
+    "@types/chroma-js": "^2.1.4",
     "@types/color": "^3.0.3",
     "@types/d3-array": "^2.9.0",
     "@types/d3-time-format": "^3.0.0",

dashboard/src/main/home/cluster-dashboard/dashboard/Metrics.tsx

@@ -10,6 +10,7 @@ import ParentSize from "@visx/responsive/lib/components/ParentSize";
 import AreaChart from "../expanded-chart/metrics/AreaChart";
 import {
   AvailableMetrics,
+  GenericMetricResponse,
   NormalizedMetricsData,
 } from "../expanded-chart/metrics/types";
 import SelectRow from "../../../../components/form-components/SelectRow";
@@ -18,6 +19,7 @@ import {
   resolutions,
   secondsBeforeNow,
 } from "../expanded-chart/metrics/MetricsSection";
+import AggregatedDataLegend from "../expanded-chart/metrics/AggregatedDataLegend";
 
 const Metrics: React.FC = () => {
   const { currentProject, currentCluster, setCurrentError } = useContext(
@@ -36,6 +38,9 @@ const Metrics: React.FC = () => {
   );
   const [selectedPercentile, setSelectedPercentile] = useState("0.99");
   const [data, setData] = useState<NormalizedMetricsData[]>([]);
+  const [aggregatedData, setAggregatedData] = useState<
+    Record<string, NormalizedMetricsData[]>
+  >({});
   const [showMetricsSettings, setShowMetricsSettings] = useState(false);
   const [isLoading, setIsLoading] = useState(0);
   const [hpaData, setHpaData] = useState([]);
@@ -218,6 +223,39 @@ const Metrics: React.FC = () => {
 
       setIsLoading((prev) => prev + 1);
       setData([]);
+      setAggregatedData({});
+
+      const allPodsRes = await api.getMetrics(
+        "<token>",
+        {
+          metric: selectedMetric,
+          shouldsum: false,
+          kind: "Ingress",
+          namespace: selectedIngress?.namespace || "default",
+          percentile:
+            selectedMetric == "nginx:latency-histogram"
+              ? parseFloat(selectedPercentile)
+              : undefined,
+          startrange: start,
+          endrange: end,
+          resolution: resolutions[selectedRange],
+          pods: [],
+          name: selectedIngress?.name,
+        },
+        {
+          id: currentProject.id,
+          cluster_id: currentCluster.id,
+        }
+      );
+
+      const allPodsData: GenericMetricResponse[] = allPodsRes.data ?? [];
+      const allPodsMetrics = allPodsData.flatMap((d) => d.results);
+      const allPodsMetricsNormalized = new MetricNormalizer(
+        [{ results: allPodsMetrics }],
+        selectedMetric as AvailableMetrics
+      );
+      setAggregatedData(allPodsMetricsNormalized.getAggregatedData());
+      //
 
       const res = await api.getMetrics(
         "<token>",
@@ -320,11 +358,13 @@ const Metrics: React.FC = () => {
       )}
       {data.length > 0 && isLoading === 0 && (
         <>
+          <AggregatedDataLegend data={data} />
           <ParentSize>
             {({ width, height }) => (
               <AreaChart
                 dataKey={selectedMetricLabel}
                 data={data}
+                aggregatedData={aggregatedData}
                 hpaData={hpaData}
                 hpaEnabled={false}
                 width={width}

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/AggregatedDataLegend.tsx

@@ -0,0 +1,57 @@
+import React from "react";
+import * as stats from "simple-statistics";
+import styled from "styled-components";
+import chroma from "chroma-js";
+import { NormalizedMetricsData } from "./types";
+import { AggregatedDataColors } from "./utils";
+
+interface AggregatedDataLegendProps {
+  data: NormalizedMetricsData[];
+}
+
+const AggregatedDataLegend = ({ data }: AggregatedDataLegendProps) => {
+  const min = stats.min(data.map((d) => d.value));
+  const avg = stats.mean(data.map((d) => d.value));
+  const max = stats.max(data.map((d) => d.value));
+
+  const aggregatedData = {
+    min,
+    avg,
+    max,
+  };
+
+  return (
+    <AggregatedDataContainer>
+      {Object.entries(aggregatedData).map(([key, value]) => (
+        <AggregatedDataItem>
+          <DataBar color={AggregatedDataColors[key]} />
+          {key.toUpperCase()}: {Math.round(value * 10000) / 10000}
+        </AggregatedDataItem>
+      ))}
+    </AggregatedDataContainer>
+  );
+};
+
+export default AggregatedDataLegend;
+
+const AggregatedDataContainer = styled.div`
+  width: 200px;
+  display: flex;
+  flex-direction: column;
+  margin-block: 8px;
+`;
+
+const AggregatedDataItem = styled.div`
+  display: flex;
+  flex-direction: row;
+  height: 20px;
+  align-items: center;
+  gap: 4px;
+`;
+
+const DataBar = styled.div<{ color: string }>`
+  height: 100%;
+  width: 5px;
+  border: 1px solid ${(props) => props.color};
+  background-color: ${(props) => chroma(props.color).alpha(0.6).css()};
+`;

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/AreaChart.tsx

@@ -1,4 +1,8 @@
+import _ from "lodash";
 import React, { useCallback, useMemo, useRef } from "react";
+import chroma from "chroma-js";
+import * as stats from "simple-statistics";
+import styled from "styled-components";
 import { AreaClosed, Bar, Line, LinePath } from "@visx/shape";
 import { curveMonotoneX } from "@visx/curve";
 import { scaleLinear, scaleTime } from "@visx/scale";
@@ -13,6 +17,7 @@ import { LinearGradient } from "@visx/gradient";
 import { bisector, extent, max } from "d3-array";
 import { timeFormat } from "d3-time-format";
 import { NormalizedMetricsData } from "./types";
+import { AggregatedDataColors } from "./utils";
 
 var globalData: NormalizedMetricsData[];
 
@@ -46,6 +51,7 @@ const bisectDate = bisector<NormalizedMetricsData, Date>(
 
 export type AreaProps = {
   data: NormalizedMetricsData[];
+  aggregatedData?: Record<string, NormalizedMetricsData[]>;
   dataKey: string;
   hpaEnabled?: boolean;
   hpaData?: NormalizedMetricsData[];
@@ -57,6 +63,7 @@ export type AreaProps = {
 
 const AreaChart: React.FunctionComponent<AreaProps> = ({
   data,
+  aggregatedData = {},
   dataKey,
   hpaEnabled = false,
   hpaData = [],
@@ -76,6 +83,7 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
   } = useTooltip<{
     data: NormalizedMetricsData;
     tooltipHpaData: NormalizedMetricsData;
+    aggregatedData?: Record<string, NormalizedMetricsData>;
   }>();
 
   const svgContainer = useRef();
@@ -103,13 +111,42 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
         domain: [
           0,
           1.25 *
-            max([...globalData, ...(isHpaEnabled ? hpaData : [])], getValue),
+            max(
+              [
+                ...globalData,
+                ...Object.values(aggregatedData).flat(),
+                ...(isHpaEnabled ? hpaData : []),
+              ],
+              getValue
+            ),
         ],
         nice: true,
       }),
     [margin.top, width, height, data, hpaData, isHpaEnabled]
   );
 
+  const getAggregatedDataTooltip = (x0: Date) => {
+    let aggregatedTooltipData: Record<string, NormalizedMetricsData> = {};
+    for (let [key, values] of Object.entries(aggregatedData)) {
+      const index = bisectDate(values, x0, 1);
+      const d0 = values[index - 1];
+      const d1 = values[index];
+      let d = d0;
+
+      if (d1 && getDate(d1)) {
+        d =
+          x0.valueOf() - getDate(d0).valueOf() >
+          getDate(d1).valueOf() - x0.valueOf()
+            ? d1
+            : d0;
+      }
+
+      aggregatedTooltipData[key] = d;
+    }
+
+    return aggregatedTooltipData;
+  };
+
   // tooltip handler
   const handleTooltip = useCallback(
     (
@@ -139,7 +176,11 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
 
       if (!isHpaEnabled || hpaIndex !== hpaIndex2) {
         showTooltip({
-          tooltipData: { data: d, tooltipHpaData: undefined },
+          tooltipData: {
+            data: d,
+            tooltipHpaData: undefined,
+            aggregatedData: getAggregatedDataTooltip(x0),
+          },
           tooltipLeft: x || 0,
           tooltipTop: valueScale(getValue(d)) || 0,
         });
@@ -168,7 +209,11 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
       point = point?.matrixTransform(container.getScreenCTM().inverse());
 
       showTooltip({
-        tooltipData: { data: d, tooltipHpaData },
+        tooltipData: {
+          data: d,
+          tooltipHpaData,
+          aggregatedData: getAggregatedDataTooltip(x0),
+        },
         tooltipLeft: x || 0,
         tooltipTop: point.y || 0,
       });
@@ -221,6 +266,14 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
           to={accentColor}
           toOpacity={0}
         />
+        {Object.entries(AggregatedDataColors).map(([dataKey, color]) => (
+          <LinearGradient
+            id={`area-gradient-${dataKey}`}
+            from={color}
+            to={color}
+            toOpacity={0}
+          />
+        ))}
         <GridRows
           left={margin.left}
           scale={valueScale}
@@ -250,6 +303,18 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
           fill="url(#area-gradient)"
           curve={curveMonotoneX}
         />
+        {Object.entries(aggregatedData).map(([key, data]) => (
+          <LinePath<NormalizedMetricsData>
+            data={data}
+            x={(d) => dateScale(getDate(d)) ?? 0}
+            y={(d) => valueScale(getValue(d)) ?? 0}
+            height={innerHeight}
+            strokeWidth={1}
+            stroke={AggregatedDataColors[key]}
+            // fill={`url(#area-gradient-${key})`}
+            curve={curveMonotoneX}
+          />
+        ))}
         {isHpaEnabled && (
           <LinePath<NormalizedMetricsData>
             stroke="#ffffff"
@@ -322,6 +387,19 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
               strokeWidth={2}
               pointerEvents="none"
             />
+            {Object.values(tooltipData.aggregatedData)?.map((d) => (
+              <circle
+                cx={tooltipLeft}
+                cy={valueScale(getValue(d)) + 1}
+                r={4}
+                fill="black"
+                fillOpacity={0.1}
+                stroke="black"
+                strokeOpacity={0.1}
+                strokeWidth={2}
+                pointerEvents="none"
+              />
+            ))}
             <circle
               cx={tooltipLeft}
               cy={dataGraphTooltipGlyphPosition}
@@ -331,6 +409,17 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
               strokeWidth={2}
               pointerEvents="none"
             />
+            {Object.values(tooltipData.aggregatedData)?.map((d) => (
+              <circle
+                cx={tooltipLeft}
+                cy={valueScale(getValue(d))}
+                r={4}
+                fill={accentColorDark}
+                stroke="white"
+                strokeWidth={2}
+                pointerEvents="none"
+              />
+            ))}
             {isHpaEnabled && hpaGraphTooltipGlyphPosition !== null && (
               <>
                 <circle
@@ -368,13 +457,17 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
               ...defaultStyles,
               background: "#26272f",
               color: "#aaaabb",
-              textAlign: "center",
             }}
           >
-            {formatDate(getDate(tooltipData.data))}
-            <div style={{ color: accentColor }}>
+            <TooltipDate>{formatDate(getDate(tooltipData.data))}</TooltipDate>
+            <TooltipDataRow>
               {dataKey}: {getValue(tooltipData.data)}
-            </div>
+            </TooltipDataRow>
+            {Object.entries(tooltipData.aggregatedData).map(([key, value]) => (
+              <TooltipDataRow color={AggregatedDataColors[key]}>
+                {`${key.toUpperCase()}. ${dataKey}`}: {getValue(value)}
+              </TooltipDataRow>
+            ))}
             {isHpaEnabled && hpaGraphTooltipGlyphPosition !== null && (
               <div style={{ color: "#FFF" }}>
                 Autoscaling Threshold: {getValue(tooltipData.tooltipHpaData)}
@@ -388,3 +481,13 @@ const AreaChart: React.FunctionComponent<AreaProps> = ({
 };
 
 export default AreaChart;
+
+const TooltipDate = styled.div`
+  text-align: center;
+  margin-bottom: 8px;
+`;
+
+const TooltipDataRow = styled.div<{ color?: string }>`
+  color: ${(props) => props.color ?? accentColor};
+  margin-bottom: 4px;
+`;

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/MetricNormalizer.ts

@@ -1,3 +1,5 @@
+import * as stats from "simple-statistics";
+import _ from "lodash";
 import {
   GenericMetricResponse,
   MetricsCPUDataResponse,
@@ -52,6 +54,40 @@ export class MetricNormalizer {
     return [];
   }
 
+  getAggregatedData(): Record<string, NormalizedMetricsData[]> {
+    const groupedByDate = _.groupBy(this.getParsedData(), "date");
+
+    const avg = Object.keys(groupedByDate).map((date) => {
+      const values = groupedByDate[date].map((d) => d.value);
+      return {
+        date: Number(date),
+        value: stats.mean(values),
+      };
+    });
+
+    const min = Object.keys(groupedByDate).map((date) => {
+      const values = groupedByDate[date].map((d) => d.value);
+      return {
+        date: Number(date),
+        value: stats.min(values),
+      };
+    });
+
+    const max = Object.keys(groupedByDate).map((date) => {
+      const values = groupedByDate[date].map((d) => d.value);
+      return {
+        date: Number(date),
+        value: stats.max(values),
+      };
+    });
+
+    return {
+      min,
+      avg,
+      max,
+    };
+  }
+
   private parseCPUMetrics(arr: MetricsCPUDataResponse["results"]) {
     return arr.map((d) => {
       return {

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/MetricsSection.tsx

@@ -12,8 +12,13 @@ import Loading from "components/Loading";
 import SelectRow from "components/form-components/SelectRow";
 import AreaChart from "./AreaChart";
 import { MetricNormalizer } from "./MetricNormalizer";
-import { AvailableMetrics, NormalizedMetricsData } from "./types";
+import {
+  AvailableMetrics,
+  GenericMetricResponse,
+  NormalizedMetricsData,
+} from "./types";
 import CheckboxRow from "components/form-components/CheckboxRow";
+import AggregatedDataLegend from "./AggregatedDataLegend";
 
 type PropsType = {
   currentChart: ChartTypeWithExtendedConfig;
@@ -49,6 +54,9 @@ const MetricsSection: React.FunctionComponent<PropsType> = ({
   );
   const [dropdownExpanded, setDropdownExpanded] = useState(false);
   const [data, setData] = useState<NormalizedMetricsData[]>([]);
+  const [aggregatedData, setAggregatedData] = useState<
+    Record<string, NormalizedMetricsData[]>
+  >({});
   const [showMetricsSettings, setShowMetricsSettings] = useState(false);
   const [metricsOptions, setMetricsOptions] = useState([
     { value: "cpu", label: "CPU Utilization (vCPUs)" },
@@ -291,6 +299,36 @@ const MetricsSection: React.FunctionComponent<PropsType> = ({
 
       setIsLoading((prev) => prev + 1);
       setData([]);
+      setAggregatedData({});
+
+      // Get aggregated metrics
+      const allPodsRes = await api.getMetrics(
+        "<token>",
+        {
+          metric: selectedMetric,
+          shouldsum: false,
+          kind: selectedController?.kind,
+          name: selectedController?.metadata.name,
+          namespace: namespace,
+          startrange: start,
+          endrange: end,
+          resolution: resolutions[selectedRange],
+          pods: [],
+        },
+        {
+          id: currentProject.id,
+          cluster_id: currentCluster.id,
+        }
+      );
+
+      const allPodsData: GenericMetricResponse[] = allPodsRes.data ?? [];
+      const allPodsMetrics = allPodsData.flatMap((d) => d.results);
+      const allPodsMetricsNormalized = new MetricNormalizer(
+        [{ results: allPodsMetrics }],
+        selectedMetric as AvailableMetrics
+      );
+      setAggregatedData(allPodsMetricsNormalized.getAggregatedData());
+      //
 
       const res = await api.getMetrics(
         "<token>",
@@ -489,6 +527,7 @@ const MetricsSection: React.FunctionComponent<PropsType> = ({
       )}
       {data.length > 0 && isLoading === 0 && (
         <>
+          <AggregatedDataLegend data={data} />
           {currentChart?.config?.autoscaling?.enabled &&
             ["cpu", "memory"].includes(selectedMetric) && (
               <CheckboxRow
@@ -501,6 +540,7 @@ const MetricsSection: React.FunctionComponent<PropsType> = ({
             {({ width, height }) => (
               <AreaChart
                 dataKey={selectedMetricLabel}
+                aggregatedData={aggregatedData}
                 data={data}
                 hpaData={hpaData}
                 hpaEnabled={

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/utils.ts

@@ -0,0 +1,5 @@
+export const AggregatedDataColors: Record<string, string> = {
+  min: "#4C4F6B",
+  avg: "#F2C94C",
+  max: "#B04649",
+};