basic metrics visualization done

dashboard/package.json

@@ -13,6 +13,7 @@
     "@types/material-ui": "^0.21.8",
     "@types/qs": "^6.9.5",
     "@types/random-words": "^1.1.0",
+    "@visx/axis": "^1.6.1",
     "@visx/curve": "^1.0.0",
     "@visx/event": "^1.3.0",
     "@visx/gradient": "^1.0.0",

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/AreaChart.tsx

@@ -1,14 +1,18 @@
 import React, { useMemo, useCallback } from "react";
 import { AreaClosed, Line, Bar } from "@visx/shape";
-import appleStock, { AppleStock } from "@visx/mock-data/lib/mocks/appleStock";
 import { curveMonotoneX } from "@visx/curve";
 import { scaleTime, scaleLinear } from "@visx/scale";
+import { AxisLeft, AxisBottom } from '@visx/axis';
+
 import {
   withTooltip,
   Tooltip,
   TooltipWithBounds,
   defaultStyles,
 } from "@visx/tooltip";
+
+import { GridRows, GridColumns } from '@visx/grid';
+
 import { WithTooltipProvidedProps } from "@visx/tooltip/lib/enhancers/withTooltip";
 import { localPoint } from "@visx/event";
 import { LinearGradient } from "@visx/gradient";
@@ -20,13 +24,16 @@ export const accentColor = '#f5cb42';
 export const accentColorDark = '#949eff';
 */
 
-interface MetricsData {
+
+export type MetricsData = {
   date: number; // unix timestamp
   value: number; // value 
 }
 
 type TooltipData = MetricsData;
 
+var globalData : MetricsData[]
+
 export const background = "#3b697800";
 export const background2 = "#20405100";
 export const accentColor = "#949eff";
@@ -41,13 +48,26 @@ const tooltipStyles = {
 // util
 const formatDate = timeFormat("%H:%M:%S %b %d, '%y");
 
+const hourFormat = timeFormat("%H:%M");
+const dayFormat = timeFormat("%b %d");
+
+// map resolutions to formats
+const formats : { [range: string]: (date: Date) => string } = {
+  "1H": hourFormat,
+  "6H": hourFormat,
+  "1D": hourFormat,
+  "1M": dayFormat,
+}
+
 // accessors
 const getDate = (d: MetricsData) => new Date(d.date*1000);
 const getValue = (d: MetricsData) => d.value;
+
 const bisectDate = bisector<MetricsData, Date>((d) => new Date(d.date*1000)).left;
 
 export type AreaProps = {
-  data: MetricsData[],
+  data: MetricsData[];
+  resolution: string;
   width: number;
   height: number;
   margin?: { top: number; right: number; bottom: number; left: number };
@@ -56,6 +76,7 @@ export type AreaProps = {
 export default withTooltip<AreaProps, TooltipData>(
   ({
     data,
+    resolution,
     width,
     height,
     margin = { top: 0, right: 0, bottom: 0, left: 0 },
@@ -65,10 +86,14 @@ export default withTooltip<AreaProps, TooltipData>(
     tooltipTop = 0,
     tooltipLeft = 0,
   }: AreaProps & WithTooltipProvidedProps<TooltipData>) => {
-    if (width < 10) return null;
+    globalData = data
+
+    if (width == 0 || height == 0 || width < 10) {
+      return null
+    }
 
     // bounds
-    const innerWidth = width - margin.left - margin.right;
+    const innerWidth = width - margin.left - margin.right - 40;
     const innerHeight = height - margin.top - margin.bottom - 20;
 
     // scales
@@ -76,18 +101,18 @@ export default withTooltip<AreaProps, TooltipData>(
       () =>
         scaleTime({
           range: [margin.left, innerWidth + margin.left],
-          domain: extent(data, getDate) as [Date, Date],
+          domain: extent(globalData, getDate) as [Date, Date],
         }),
-      [innerWidth, margin.left]
+      [innerWidth, margin.left, width, height, data]
     );
-    const stockValueScale = useMemo(
+    const valueScale = useMemo(
       () =>
         scaleLinear({
           range: [innerHeight + margin.top, margin.top],
-          domain: [0, 1.25 * max(data, getValue)],
+          domain: [0, 1.25 * max(globalData, getValue)],
           nice: true,
         }),
-      [margin.top, innerHeight]
+      [margin.top, innerHeight, width, height, data]
     );
 
     // tooltip handler
@@ -99,10 +124,11 @@ export default withTooltip<AreaProps, TooltipData>(
       ) => {
         const { x } = localPoint(event) || { x: 0 };
         const x0 = dateScale.invert(x);
-        const index = bisectDate(data, x0, 1);
-        const d0 = data[index - 1];
-        const d1 = data[index];
+        const index = bisectDate(globalData, x0, 1);
+        const d0 = globalData[index - 1];
+        const d1 = globalData[index];
         let d = d0;
+
         if (d1 && getDate(d1)) {
           d =
             x0.valueOf() - getDate(d0).valueOf() >
@@ -110,14 +136,15 @@ export default withTooltip<AreaProps, TooltipData>(
               ? d1
               : d0;
         }
+
         showTooltip({
           tooltipData: d,
-          tooltipLeft: x,
-          tooltipTop: stockValueScale(getValue(d)),
+          tooltipLeft: x || 0,
+          tooltipTop: valueScale(getValue(d)) || 0,
         });
       },
-      [showTooltip, stockValueScale, dateScale]
-    );
+      [showTooltip, valueScale, dateScale, width, height, data]
+    )
 
     return (
       <div>
@@ -141,16 +168,61 @@ export default withTooltip<AreaProps, TooltipData>(
             to={accentColor}
             toOpacity={0}
           />
+          <GridRows
+            left={margin.left}
+            scale={valueScale}
+            width={innerWidth}
+            strokeDasharray="1,3"
+            stroke="white"
+            strokeOpacity={0.2}
+            pointerEvents="none"
+          />
+          <GridColumns
+            top={margin.top}
+            scale={dateScale}
+            height={innerHeight}
+            strokeDasharray="1,3"
+            stroke="white"
+            strokeOpacity={0.2}
+            pointerEvents="none"
+          />
           <AreaClosed<MetricsData>
             data={data}
             x={(d) => dateScale(getDate(d)) ?? 0}
-            y={(d) => stockValueScale(getValue(d)) ?? 0}
-            yScale={stockValueScale}
+            y={(d) => valueScale(getValue(d)) ?? 0}
+            height={innerHeight}
+            yScale={valueScale}
             strokeWidth={1}
             stroke="url(#area-gradient)"
             fill="url(#area-gradient)"
             curve={curveMonotoneX}
           />
+          <AxisLeft
+            left={10}
+            scale={valueScale}
+            hideAxisLine={true}
+            hideTicks={true}
+            tickLabelProps={() => ({
+              fill: "white",
+              fontSize: 11,
+              textAnchor: 'start',
+              fillOpacity: 0.4,
+              dy: 0,
+            })}
+          />
+          <AxisBottom
+            top={height - 20}
+            scale={dateScale}
+            tickFormat={formats[resolution]}
+            hideAxisLine={true}
+            hideTicks={true}
+            tickLabelProps={() => ({
+              fill: "white",
+              fontSize: 11,
+              textAnchor: 'middle',
+              fillOpacity: 0.4,
+            })}
+          />
           <Bar
             x={margin.left}
             y={margin.top}

dashboard/src/main/home/cluster-dashboard/expanded-chart/metrics/MetricsSection.tsx

@@ -4,43 +4,77 @@ import ParentSize from "@visx/responsive/lib/components/ParentSize";
 
 import api from "shared/api";
 import { Context } from "shared/Context";
-import { ChartType } from "shared/types";
+import { ChartType, StorageType } from "shared/types";
 
 import TabSelector from "components/TabSelector";
-import AreaChart from "./AreaChart";
+import AreaChart, { MetricsData } from "./AreaChart";
 
 type PropsType = {
   currentChart: ChartType;
 };
 
 type StateType = {
+  controllers: any[];
+  selectedController: any;
+  pods: string[];
+  selectedPod: string;
   selectedRange: string;
+  selectedMetric: string;
   selectedMetricLabel: string;
+  controllerDropdownExpanded: boolean;
+  podDropdownExpanded: boolean;
   dropdownExpanded: boolean;
+  data: MetricsData[];
 };
 
-var fakeData = [{
-  date: 1613512500,
-  value: 0.00017923172010701633,
-},
-{
-  date: 1613513100,
-  value: 0.00018,
-},
-{
-  date: 1613513700,
-  value: 0.0001923,
-}]
-
-export default class ListSection extends Component<PropsType, StateType> {
+type MetricsCPUDataResponse = {
+  pod?: string,
+  results: {
+    date: number,
+    cpu: string,
+  }[],
+}[]
+
+type MetricsMemoryDataResponse = {
+  pod?: string,
+  results: {
+    date: number,
+    memory: string,
+  }[],
+}[]
+
+const resolutions : { [range: string]: string } = {
+  "1H": "15s",
+  "6H": "15s",
+  "1D": "15s",
+  "1M": "5h",
+}
+
+const secondsBeforeNow : { [range: string]: number } = {
+  "1H": 60 * 60,
+  "6H": 60 * 60 * 6,
+  "1D": 60 * 60 * 24,
+  "1M": 60 * 60 * 24 * 30,
+}
+
+export default class MetricsSection extends Component<PropsType, StateType> {
   state = {
+    pods: [] as string[],
+    selectedPod: "",
+    controllers: [] as any[],
+    selectedController: null as any,
     selectedRange: "1H",
-    selectedMetricLabel: "CPU Utilization",
+    selectedMetric: "cpu",
+    selectedMetricLabel: "CPU Utilization (vCPUs)",
     dropdownExpanded: false,
+    podDropdownExpanded: false,
+    controllerDropdownExpanded: false,
+    data: [] as MetricsData[],
   };
 
   componentDidMount() {
-    const { selectors, currentChart } = this.props;
+    // get all controllers and read in a list of pods
+    let { currentChart } = this.props;
     let { currentCluster, currentProject, setCurrentError } = this.context;
 
     api
@@ -58,11 +92,161 @@ export default class ListSection extends Component<PropsType, StateType> {
         }
       )
       .then((res) => {
-        this.setState({ controllers: res.data, loading: false });
+        // TODO -- check at least one controller returned
+
+        // iterate through the controllers to get the list of pods
+        this.setState({ controllers: res.data, selectedController: res.data[0] });
+        
+        this.getPods()
       })
       .catch((err) => {
         setCurrentError(JSON.stringify(err));
-        this.setState({ controllers: [], loading: false });
+        this.setState({ controllers: [] });
+      });      
+  }
+
+  componentDidUpdate(prevProps: PropsType, prevState: StateType) {
+    // if resolution, data kind, controllers, or pods have changed, update data
+    if (this.state.selectedMetric != prevState.selectedMetric) {
+      this.getMetrics()
+    }
+
+    if (this.state.selectedRange != prevState.selectedRange) {
+      this.getMetrics()
+    }
+
+    if (this.state.selectedPod != prevState.selectedPod) {
+      this.getMetrics()
+    }
+
+    if (this.state.selectedController?.metadata?.name != prevState.selectedController?.metadata?.name) {
+      this.getMetrics()
+    }
+  }
+
+  getMetrics = () => {
+    if (this.state.pods.length == 0) {
+      return
+    }
+
+    let { currentChart } = this.props;
+    let { currentCluster, currentProject, setCurrentError } = this.context;
+    let kind = this.state.selectedMetric
+    let shouldsum = true
+
+    // calculate start and end range
+    var d = new Date();
+    var end = Math.round(d.getTime() / 1000);
+    var start = end - secondsBeforeNow[this.state.selectedRange]
+
+    let pods = this.state.pods
+
+    if (this.state.selectedPod != "All") {
+      pods = [this.state.selectedPod]
+    }
+
+    api
+      .getMetrics(
+        "<token>",
+        {
+          cluster_id: currentCluster.id,
+          metric: kind,
+          shouldsum: shouldsum,
+          pods: pods,
+          namespace: currentChart.namespace,
+          startrange: start,
+          endrange: end,
+          resolution: resolutions[this.state.selectedRange],
+        },
+        {
+          id: currentProject.id,
+        }
+      )
+      .then((res) => {
+        // transform the metrics to expected form
+        if (kind == "cpu") {
+          let data = res.data as MetricsCPUDataResponse
+          
+          // if summed, just look at the first data
+            let tData = data[0].results.map(
+              (d: {
+                date: number,
+                cpu: string,
+              }, i: number) => {
+                return {
+                  date: d.date,
+                  value: parseFloat(d.cpu),
+                }
+              }
+            )
+
+            this.setState({ data: tData })
+        } else if (kind == "memory") {
+          let data = res.data as MetricsMemoryDataResponse
+
+          let tData = data[0].results.map(
+            (d: {
+              date: number,
+              memory: string,
+            }, i: number) => {
+              return {
+                date: d.date,
+                value: parseFloat(d.memory) / (1024 * 1024), // put units in Mi
+              }
+            }
+          )
+
+          this.setState({ data: tData })
+        }
+      })
+      .catch((err) => {
+        setCurrentError(JSON.stringify(err));
+        // this.setState({ controllers: [], loading: false });
+      });
+  }
+
+  getPods = () => {
+    let { selectedController } = this.state;
+    let { currentCluster, currentProject, setCurrentError } = this.context;
+
+    let selectors = [] as string[];
+    let ml =
+      selectedController?.spec?.selector?.matchLabels || selectedController?.spec?.selector;
+    let i = 1;
+    let selector = "";
+    for (var key in ml) {
+      selector += key + "=" + ml[key];
+      if (i != Object.keys(ml).length) {
+        selector += ",";
+      }
+      i += 1;
+    }
+    selectors.push(selector);
+
+    api
+      .getMatchingPods(
+        "<token>",
+        {
+          cluster_id: currentCluster.id,
+          selectors,
+        },
+        {
+          id: currentProject.id,
+        }
+      )
+      .then((res) => {
+        let pods = res?.data?.map((pod: any) => {
+          return pod?.metadata?.name
+        });
+
+        this.setState({ pods, selectedPod: "All" });
+
+        this.getMetrics()
+      })
+      .catch((err) => {
+        console.log(err);
+        setCurrentError(JSON.stringify(err));
+        return;
       });
   }
 
@@ -85,18 +269,105 @@ export default class ListSection extends Component<PropsType, StateType> {
     }
   };
 
+  renderPodDropdown = () => {
+    if (this.state.podDropdownExpanded) {
+      return (
+        <>
+          <DropdownOverlay
+            onClick={() => this.setState({ podDropdownExpanded: false })}
+          />
+          <Dropdown
+            dropdownWidth="400px"
+            dropdownMaxHeight="200px"
+            onClick={() => this.setState({ podDropdownExpanded: false })}
+          >
+            {this.renderPodOptionList()}
+          </Dropdown>
+        </>
+      );
+    }
+  };
+
+  renderPodOptionList = () => {
+    let allPod = [(
+      <Option
+        key={0}
+        selected={"All" === this.state.selectedPod}
+        onClick={() => this.setState({ selectedPod: "All" })}
+        lastItem={false}
+      >
+        All (summed)
+      </Option>
+    )];
+
+    let podOptions = this.state.pods.map(
+      (option: string, i: number) => {
+        return (
+          <Option
+            key={i + 1}
+            selected={option === this.state.selectedPod}
+            onClick={() => this.setState({ selectedPod: option })}
+            lastItem={i === this.state.pods.length - 1}
+          >
+            {option}
+          </Option>
+        );
+      }
+    )
+
+    return allPod.concat(podOptions)
+  };
+
+  renderControllerDropdown = () => {
+    if (this.state.controllerDropdownExpanded) {
+      return (
+        <>
+          <DropdownOverlay
+            onClick={() => this.setState({ controllerDropdownExpanded: false })}
+          />
+          <Dropdown
+            dropdownWidth="300px"
+            dropdownMaxHeight="200px"
+            onClick={() => this.setState({ controllerDropdownExpanded: false })}
+          >
+            {this.renderControllerOptionList()}
+          </Dropdown>
+        </>
+      );
+    }
+  };
+
+  renderControllerOptionList = () => {
+    return this.state.controllers.map(
+      (controller: any, i: number) => {
+        let name = controller?.metadata?.name
+
+        return (
+          <Option
+            key={i}
+            selected={name === this.state.selectedController?.metadata?.name}
+            onClick={() => this.setState({ selectedController: controller })}
+            lastItem={i === this.state.controllers.length - 1}
+          >
+            {name}
+          </Option>
+        );
+      }
+    )
+  };
+
   renderOptionList = () => {
     let metricOptions = [
-      { value: "cpu", label: "CPU Utilization" },
-      { value: "ram", label: "RAM Utilization" },
+      { value: "cpu", label: "CPU Utilization (vCPUs)" },
+      { value: "memory", label: "RAM Utilization (Mi)" },
     ];
     return metricOptions.map(
       (option: { value: string; label: string }, i: number) => {
         return (
           <Option
             key={i}
-            selected={option.label === this.state.selectedMetricLabel}
-            onClick={() => this.setState({ selectedMetricLabel: option.label })}
+            selected={option.value === this.state.selectedMetric}
+            onClick={() => this.setState({ selectedMetric: option.value, selectedMetricLabel: option.label })}
             lastItem={i === metricOptions.length - 1}
           >
             {option.label}
@@ -110,7 +381,13 @@ export default class ListSection extends Component<PropsType, StateType> {
     return (
       <StyledMetricsSection>
         <ParentSize>
-          {({ width, height }) => <AreaChart data={fakeData} width={width} height={height} />}
+          {({ width, height }) => <AreaChart 
+            data={this.state.data} 
+            width={width} 
+            height={height} 
+            resolution={this.state.selectedRange}
+            margin={{ top: 70, right: 0, bottom: 0, left: 50 }}
+          />}
         </ParentSize>
         <MetricSelector
           onClick={() =>
@@ -121,15 +398,31 @@ export default class ListSection extends Component<PropsType, StateType> {
           <i className="material-icons">arrow_drop_down</i>
           {this.renderDropdown()}
         </MetricSelector>
+        <ControllerSelector
+          onClick={() =>
+            this.setState({ controllerDropdownExpanded: !this.state.controllerDropdownExpanded })
+          }
+        >
+          {this.state.selectedController?.metadata?.name}
+          <i className="material-icons">arrow_drop_down</i>
+          {this.renderControllerDropdown()}
+        </ControllerSelector>
+        <PodSelector
+          onClick={() =>
+            this.setState({ podDropdownExpanded: !this.state.podDropdownExpanded })
+          }
+        >
+          {this.state.selectedPod}
+          <i className="material-icons">arrow_drop_down</i>
+          {this.renderPodDropdown()}
+        </PodSelector>
         <RangeWrapper>
           <TabSelector
             options={[
               { value: "1H", label: "1H" },
+              { value: "6H", label: "6H" },
               { value: "1D", label: "1D" },
               { value: "1M", label: "1M" },
-              { value: "3M", label: "3M" },
-              { value: "1Y", label: "1Y" },
-              { value: "ALL", label: "ALL" },
             ]}
             currentTab={this.state.selectedRange}
             setCurrentTab={(x: string) => this.setState({ selectedRange: x })}
@@ -140,7 +433,7 @@ export default class ListSection extends Component<PropsType, StateType> {
   }
 }
 
-ListSection.contextType = Context;
+MetricsSection.contextType = Context;
 
 const DropdownOverlay = styled.div`
   position: fixed;
@@ -192,21 +485,22 @@ const Dropdown = styled.div`
   box-shadow: 0 4px 8px 0px #00000088;
 `;
 
+
 const RangeWrapper = styled.div`
   position: absolute;
-  bottom: 10px;
-  font-weight: bold;
+  top: 20px;
   left: 0;
-  width: 100%;
+  font-weight: bold;
+  width: calc(100% - 36px);
 `;
 
 const MetricSelector = styled.div`
-  font-size: 16px;
+  font-size: 13px;
   font-weight: 500;
   color: #ffffff;
   position: absolute;
   top: 0;
-  left: 5px;
+  left: 0;
   display: flex;
   align-items: center;
   cursor: pointer;
@@ -224,6 +518,14 @@ const MetricSelector = styled.div`
   }
 `;
 
+const ControllerSelector = styled(MetricSelector)`
+  left: 200px;
+`
+
+const PodSelector = styled(MetricSelector)`
+  left: 500px;
+`
+
 const StyledMetricsSection = styled.div`
   width: 100%;
   height: 100%;

dashboard/src/shared/api.tsx

@@ -348,6 +348,22 @@ const getMatchingPods = baseApi<
   return `/api/projects/${pathParams.id}/k8s/pods`;
 });
 
+const getMetrics = baseApi<
+  {
+    cluster_id: number;
+    metric: string,
+    shouldsum: boolean,
+    pods: string[],
+    namespace: string,
+    startrange: number,
+    endrange: number,
+    resolution: string,
+  },
+  { id: number }
+>("GET", (pathParams) => {
+  return `/api/projects/${pathParams.id}/k8s/metrics`;
+});
+
 const getNamespaces = baseApi<
   {
     cluster_id: number;
@@ -598,6 +614,7 @@ export default {
   getIngress,
   getInvites,
   getMatchingPods,
+  getMetrics,
   getNamespaces,
   getOAuthIds,
   getProjectClusters,

internal/kubernetes/prometheus/metrics.go

@@ -30,13 +30,13 @@ func GetPrometheusService(clientset kubernetes.Interface) (*v1.Service, bool, er
 }
 
 type QueryOpts struct {
-	Metric     string   `json:"metric"`
-	ShouldSum  bool     `json:"should_sum"`
-	PodList    []string `json:"pods"`
-	Namespace  string   `json:"namespace"`
-	StartRange uint     `json:"start_range"`
-	EndRange   uint     `json:"end_range"`
-	Resolution string   `json:"resolution"`
+	Metric     string   `schema:"metric"`
+	ShouldSum  bool     `schema:"shouldsum"`
+	PodList    []string `schema:"pods"`
+	Namespace  string   `schema:"namespace"`
+	StartRange uint     `schema:"startrange"`
+	EndRange   uint     `schema:"endrange"`
+	Resolution string   `schema:"resolution"`
 }
 
 func QueryPrometheus(

internal/models/gitrepo.go

@@ -14,7 +14,7 @@ type GitRepo struct {
 	ProjectID uint `json:"project_id"`
 
 	// The username/organization that this repo integration is linked to
-	RepoEntity string `json:"repo_entity"`
+	RepoEntity string `json:"repo_entity" gorm:"unique"`
 
 	// The various auth mechanisms available to the integration
 	OAuthIntegrationID uint

server/api/k8s_handler.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 
 	"github.com/go-chi/chi"
+	"github.com/gorilla/schema"
 	"github.com/gorilla/websocket"
 	"github.com/porter-dev/porter/internal/forms"
 	"github.com/porter-dev/porter/internal/kubernetes"
@@ -346,7 +347,10 @@ func (app *App) HandleGetPodMetrics(w http.ResponseWriter, r *http.Request) {
 	form.K8sForm.PopulateK8sOptionsFromQueryParams(vals, app.Repo.Cluster)
 
 	// decode from JSON to form value
-	if err := json.NewDecoder(r.Body).Decode(form.QueryOpts); err != nil {
+	decoder := schema.NewDecoder()
+	decoder.IgnoreUnknownKeys(true)
+
+	if err := decoder.Decode(form.QueryOpts, vals); err != nil {
 		app.handleErrorFormDecoding(err, ErrProjectDecode, w)
 		return
 	}
@@ -370,19 +374,19 @@ func (app *App) HandleGetPodMetrics(w http.ResponseWriter, r *http.Request) {
 	promSvc, found, err := prometheus.GetPrometheusService(agent.Clientset)
 
 	if err != nil {
-		app.handleErrorFormValidation(err, ErrK8sValidate, w)
+		app.handleErrorInternal(err, w)
 		return
 	}
 
 	if !found {
-		app.handleErrorFormValidation(err, ErrK8sValidate, w)
+		app.handleErrorInternal(err, w)
 		return
 	}
 
 	rawQuery, err := prometheus.QueryPrometheus(agent.Clientset, promSvc, form.QueryOpts)
 
 	if err != nil {
-		app.handleErrorFormValidation(err, ErrK8sValidate, w)
+		app.handleErrorInternal(err, w)
 		return
 	}