Merge branch 'develop' into r2k1

.gitignore

@@ -5,6 +5,7 @@
 ui/.parcel-cache
 ui/.cache
 ui/dist
+ui/.env
 ui/node_modules/
 cmd/costmodel/costmodel
 cmd/costmodel/costmodel-amd64

pkg/cloudcost/querier.go

@@ -29,7 +29,7 @@ const DefaultChartItemsLength int = 10
 // ViewQuerier defines a contract for return View types to the QueryService to service the View Api
 type ViewQuerier interface {
 	QueryViewGraph(ViewQueryRequest, context.Context) (ViewGraphData, error)
-	QueryViewTotals(ViewQueryRequest, context.Context) (*ViewTableRow, int, error)
+	QueryViewTotals(ViewQueryRequest, context.Context) (*ViewTotals, error)
 	QueryViewTable(ViewQueryRequest, context.Context) (ViewTableRows, error)
 }
 

pkg/cloudcost/queryservice.go

@@ -106,11 +106,6 @@ func (s *QueryService) GetCloudCostViewGraphHandler() func(w http.ResponseWriter
 	}
 }
 
-type CloudCostViewTotalsResponse struct {
-	NumResults int           `json:"numResults"`
-	Combined   *ViewTableRow `json:"combined"`
-}
-
 func (s *QueryService) GetCloudCostViewTotalsHandler() func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	// Return valid handler func
 	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
@@ -136,17 +131,12 @@ func (s *QueryService) GetCloudCostViewTotalsHandler() func(w http.ResponseWrite
 			return
 		}
 
-		totals, count, err := s.ViewQuerier.QueryViewTotals(*request, ctx)
+		resp, err := s.ViewQuerier.QueryViewTotals(*request, ctx)
 		if err != nil {
 			http.Error(w, fmt.Sprintf("Internal server error: %s", err), http.StatusInternalServerError)
 			return
 		}
 
-		resp := CloudCostViewTotalsResponse{
-			NumResults: count,
-			Combined:   totals,
-		}
-
 		_, spanResp := tracer.Start(ctx, "write response")
 		w.Header().Set("Content-Type", "application/json")
 		protocol.WriteData(w, resp)

pkg/cloudcost/queryservice_helper.go

@@ -38,6 +38,11 @@ func ParseCloudCostRequest(qp httputil.QueryParams) (*QueryRequest, error) {
 		aggregateBy = append(aggregateBy, prop)
 	}
 
+	// if we're aggregating by nothing (aka `item` on the frontend) then aggregate by all
+	if len(aggregateBy) == 0 {
+		aggregateBy = []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp}
+	}
+
 	accumulate := kubecost.ParseAccumulate(qp.Get("accumulate", ""))
 
 	var filter filter21.Filter

pkg/cloudcost/queryservice_helper_test.go

@@ -41,7 +41,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  "",
 				Filter:      nil,
 			},
@@ -77,7 +77,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  kubecost.AccumulateOptionWeek,
 				Filter:      nil,
 			},
@@ -91,7 +91,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  kubecost.AccumulateOptionNone,
 				Filter:      nil,
 			},
@@ -105,7 +105,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  kubecost.AccumulateOptionNone,
 				Filter:      validFilter,
 			},

pkg/cloudcost/repositoryquerier.go

@@ -114,42 +114,45 @@ func (rq *RepositoryQuerier) QueryViewGraph(request ViewQueryRequest, ctx contex
 	return sets, nil
 }
 
-func (rq *RepositoryQuerier) QueryViewTotals(request ViewQueryRequest, ctx context.Context) (*ViewTableRow, int, error) {
+func (rq *RepositoryQuerier) QueryViewTotals(request ViewQueryRequest, ctx context.Context) (*ViewTotals, error) {
 	ccasr, err := rq.Query(request.QueryRequest, ctx)
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: query failed: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: query failed: %w", err)
 	}
 	acc, err := ccasr.AccumulateAll()
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: accumulate failed: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: accumulate failed: %w", err)
 	}
 	if acc.IsEmpty() {
-		return nil, 0, nil
+		return nil, nil
 	}
 	count := len(acc.CloudCosts)
 
 	total, err := acc.Aggregate([]string{})
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: aggregate total failed: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: aggregate total failed: %w", err)
 	}
 
 	if total.IsEmpty() {
-		return nil, -1, fmt.Errorf("QueryViewTotals: missing total: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: missing total: %w", err)
 	}
 
 	if len(total.CloudCosts) != 1 {
-		return nil, -1, fmt.Errorf("QueryViewTotals: total did not aggregate: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: total did not aggregate: %w", err)
 	}
 
 	cm, err := total.CloudCosts[""].GetCostMetric(request.CostMetricName)
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: failed to retrieve cost metric: %w", err)
-	}
-	return &ViewTableRow{
-		Name:              "Totals",
-		KubernetesPercent: cm.KubernetesPercent,
-		Cost:              cm.Cost,
-	}, count, nil
+		return nil, fmt.Errorf("QueryViewTotals: failed to retrieve cost metric: %w", err)
+	}
+	return &ViewTotals{
+		NumResults: count,
+		Combined: &ViewTableRow{
+			Name:              "Totals",
+			KubernetesPercent: cm.KubernetesPercent,
+			Cost:              cm.Cost,
+		},
+	}, nil
 }
 
 func (rq *RepositoryQuerier) QueryViewTable(request ViewQueryRequest, ctx context.Context) (ViewTableRows, error) {
@@ -168,8 +171,14 @@ func (rq *RepositoryQuerier) QueryViewTable(request ViewQueryRequest, ctx contex
 		if err2 != nil {
 			return nil, fmt.Errorf("QueryViewTable: failed to retrieve cost metric: %w", err)
 		}
+		var labels map[string]string
+		if cloudCost.Properties != nil {
+			labels = cloudCost.Properties.Labels
+		}
+
 		vtr := &ViewTableRow{
 			Name:              key,
+			Labels:            labels,
 			KubernetesPercent: costMetric.KubernetesPercent,
 			Cost:              costMetric.Cost,
 		}

pkg/cloudcost/view.go

@@ -31,9 +31,10 @@ func (vtrs ViewTableRows) Equal(that ViewTableRows) bool {
 }
 
 type ViewTableRow struct {
-	Name              string  `json:"name"`
-	KubernetesPercent float64 `json:"kubernetesPercent"`
-	Cost              float64 `json:"cost"`
+	Name              string            `json:"name"`
+	Labels            map[string]string `json:"labels"`
+	KubernetesPercent float64           `json:"kubernetesPercent"`
+	Cost              float64           `json:"cost"`
 }
 
 func (vtr *ViewTableRow) Equal(that *ViewTableRow) bool {
@@ -41,6 +42,20 @@ func (vtr *ViewTableRow) Equal(that *ViewTableRow) bool {
 		return false
 	}
 
+	if len(vtr.Labels) != len(that.Labels) {
+		return false
+	}
+
+	for key, value := range vtr.Labels {
+		thatValue, ok := that.Labels[key]
+		if !ok {
+			return false
+		}
+		if value != thatValue {
+			return false
+		}
+	}
+
 	if !mathutil.Approximately(vtr.KubernetesPercent, that.KubernetesPercent) {
 		return false
 	}
@@ -105,3 +120,8 @@ func (vgdsi ViewGraphDataSetItem) Equal(that ViewGraphDataSetItem) bool {
 
 	return true
 }
+
+type ViewTotals struct {
+	NumResults int           `json:"numResults"`
+	Combined   *ViewTableRow `json:"combined"`
+}

pkg/costmodel/costmodel.go

@@ -1114,7 +1114,60 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			gpuc = 0.0
 		}
 
-		if newCnode.GPU != "" && newCnode.GPUCost == "" {
+		// Special case for SUSE rancher, since it won't behave with normal
+		// calculations, courtesy of the instance type not being "real" (a
+		// recognizable AWS instance type.)
+		if newCnode.InstanceType == "rke2" {
+			log.Infof(
+				"Found a SUSE Rancher node %s, defaulting and skipping math",
+				cp.GetKey(nodeLabels, n).Features(),
+			)
+
+			defaultCPUCorePrice, err := strconv.ParseFloat(cfg.CPU, 64)
+			if err != nil {
+				log.Errorf("Could not parse default cpu price")
+				defaultCPUCorePrice = 0
+			}
+			if math.IsNaN(defaultCPUCorePrice) {
+				log.Warnf("defaultCPU parsed as NaN. Setting to 0.")
+				defaultCPUCorePrice = 0
+			}
+
+			defaultRAMPrice, err := strconv.ParseFloat(cfg.RAM, 64)
+			if err != nil {
+				log.Errorf("Could not parse default ram price")
+				defaultRAMPrice = 0
+			}
+			if math.IsNaN(defaultRAMPrice) {
+				log.Warnf("defaultRAM parsed as NaN. Setting to 0.")
+				defaultRAMPrice = 0
+			}
+
+			defaultGPUPrice, err := strconv.ParseFloat(cfg.GPU, 64)
+			if err != nil {
+				log.Errorf("Could not parse default gpu price")
+				defaultGPUPrice = 0
+			}
+			if math.IsNaN(defaultGPUPrice) {
+				log.Warnf("defaultGPU parsed as NaN. Setting to 0.")
+				defaultGPUPrice = 0
+			}
+			// Just say no to doing the ratios!
+			cpuCost := defaultCPUCorePrice * cpu
+			gpuCost := defaultGPUPrice * gpuc
+			ramCost := defaultRAMPrice * ram
+			nodeCost := cpuCost + gpuCost + ramCost
+
+			newCnode.Cost = fmt.Sprintf("%f", nodeCost)
+			newCnode.VCPUCost = fmt.Sprintf("%f", cpuCost)
+			newCnode.GPUCost = fmt.Sprintf("%f", gpuCost)
+			newCnode.RAMCost = fmt.Sprintf("%f", ramCost)
+			newCnode.RAMBytes = fmt.Sprintf("%f", ram)
+
+		} else if newCnode.GPU != "" && newCnode.GPUCost == "" {
+			// was the big thing to investigate. All the funky ratio math
+			// we were doing was messing with their default pricing. for SUSE Rancher.
+
 			// We couldn't find a gpu cost, so fix cpu and ram, then accordingly
 			log.Infof("GPU without cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
 
@@ -2501,9 +2554,8 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 					}
 
 					if totals == nil {
-						log.Errorf("unable to locate asset totals for allocation %s", key)
-						return nil, fmt.Errorf("unable to locate allocation totals for allocation")
-
+						log.Errorf("unable to locate asset totals for allocation %s, corresponding PARC is being skipped", key)
+						continue
 					}
 
 					parc.CPUTotalCost = totals.CPUCost

ui/Dockerfile.cross

@@ -5,7 +5,7 @@ ENV API_SERVER=0.0.0.0
 ENV UI_PORT=9090
 
 COPY ./dist /var/www
-COPY default.nginx.conf /etc/nginx/conf.d/
+COPY default.nginx.conf.template /etc/nginx/conf.d/default.nginx.conf.template
 COPY nginx.conf /etc/nginx/
 COPY ./docker-entrypoint.sh /usr/local/bin/
 

ui/src/cloudCost/cloudCostDetails.js

@@ -33,7 +33,7 @@ const CloudCostDetails = ({
 
   const nextFilters = [
     ...(filters ?? []),
-    { property: "providerIds", value: selectedProviderId },
+    { property: "providerID", value: selectedProviderId },
   ];
 
   async function fetchData() {
@@ -122,7 +122,7 @@ const CloudCostDetails = ({
         title={`Costs over the last ${window}`}
         style={{ margin: "10%" }}
       >
-        <Paper>
+        <Paper style={{ padding: 20 }}>
           <Typography style={{ marginTop: "1rem" }} variant="body1">
             {selectedItem}
           </Typography>

ui/src/cloudCostReports.js

@@ -175,7 +175,6 @@ const CloudCostReports = () => {
       return {
         property,
         value,
-        name: aggMap[property] || property,
       };
     });
     setFilters(newFilters);
@@ -267,6 +266,7 @@ const CloudCostReports = () => {
               aggregationOptions={aggregationOptions}
               aggregateBy={aggregateBy}
               setAggregateBy={(agg) => {
+                setFilters([])
                 searchParams.set("agg", agg);
                 routerHistory.push({
                   search: `?${searchParams.toString()}`,

ui/src/services/cloudCostDayTotals.js

@@ -1,5 +1,5 @@
 import axios from "axios";
-import { getCloudFilters } from "../util";
+import { parseFilters } from "../util";
 import { costMetricToPropName } from "../cloudCost/tokens";
 
 function formatItemsForCost({ data, costType }) {
@@ -21,12 +21,11 @@ class CloudCostDayTotalsService {
     if (this.BASE_URL.includes("PLACEHOLDER_BASE_URL")) {
       this.BASE_URL = `http://localhost:9090/model`;
     }
-
     if (aggregate.includes("item")) {
       const resp = await axios.get(
         `${
           this.BASE_URL
-        }/cloudCost?window=${window}&costMetric=${costMetric}${getCloudFilters(
+        }/cloudCost?window=${window}&costMetric=${costMetric}&filter=${parseFilters(
           filters
         )}`
       );

ui/src/services/cloudCostTop.js

@@ -1,5 +1,5 @@
 import axios from "axios";
-import { getCloudFilters, formatSampleItemsForGraph } from "../util";
+import { formatSampleItemsForGraph, parseFilters } from "../util";
 
 class CloudCostTopService {
   BASE_URL = process.env.BASE_URL || "{PLACEHOLDER_BASE_URL}";
@@ -13,7 +13,7 @@ class CloudCostTopService {
       window,
       aggregate,
       costMetric,
-      filters,
+      filter: parseFilters(filters ?? []),
       limit: 1000,
     };
 
@@ -21,7 +21,7 @@ class CloudCostTopService {
       const resp = await axios.get(
         `${
           this.BASE_URL
-        }/cloudCost?window=${window}&costMetric=${costMetric}${getCloudFilters(
+        }/cloudCost?window=${window}&costMetric=${costMetric}&filter=${parseFilters(
           filters
         )}`
       );

ui/src/util.js

@@ -344,35 +344,6 @@ export function checkCustomWindow(window) {
   return customDateRegex.test(window);
 }
 
-export function getCloudFilters(filters) {
-  const filterNamesMap = {
-    "invoice entity": "filterInvoiceEntityIDs",
-    provider: "filterProviders",
-    providerids: "filterProviderIDs",
-    service: "filterServices",
-    account: "filterAccountIDs",
-  };
-  const params = new URLSearchParams();
-  const labelFilters = [];
-
-  for (let filter of filters) {
-    const mapped = filterNamesMap[filter.property.toLowerCase()];
-
-    if (mapped) {
-      params.set(mapped, filter.value);
-    } else if (filter.property === "Labels") {
-      labelFilters.push(filter.value);
-    } else if (filter.property.startsWith(":")) {
-      labelFilters.push(`${filter.property.slice(6)}:${filter.value}`);
-    }
-  }
-  if (labelFilters.length) {
-    params.set("filterLabels", labelFilters.join(","));
-  }
-
-  return `&${params.toString()}`;
-}
-
 export function formatSampleItemsForGraph({ data, costMetric }) {
   const costMetricPropName = costMetric
     ? costMetricToPropName[costMetric]
@@ -412,29 +383,31 @@ export function formatSampleItemsForGraph({ data, costMetric }) {
         cloudCostItem[costMetricPropName].kubernetesPercent;
     });
   });
-  const tableRows = Object.entries(accumulator).map(
-    ([
-      name,
-      {
+  const tableRows = Object.entries(accumulator)
+    .map(
+      ([
+        name,
+        {
+          cost,
+          start,
+          end,
+          providerID,
+          kubernetesCost,
+          kubernetesPercent,
+          labelName,
+        },
+      ]) => ({
         cost,
+        name,
+        kubernetesCost,
+        kubernetesPercent,
         start,
         end,
         providerID,
-        kubernetesCost,
-        kubernetesPercent,
         labelName,
-      },
-    ]) => ({
-      cost,
-      name,
-      kubernetesCost,
-      kubernetesPercent,
-      start,
-      end,
-      providerID,
-      labelName,
-    })
-  );
+      })
+    )
+    .sort((a, b) => (a.cost > b.cost ? -1 : 1));
 
   const tableTotal = tableRows.reduce(
     (tr1, tr2) => ({
@@ -457,6 +430,18 @@ export function formatSampleItemsForGraph({ data, costMetric }) {
   return { graphData, tableRows, tableTotal };
 }
 
+export function parseFilters(filters) {
+  if (typeof filters === "string") {
+    return filters;
+  }
+  // remove dups (via context ) and format
+  return (
+    [...new Set(filters.map((f) => `${f.property}:"${f.value}"`))].join(
+      encodeURIComponent("+")
+    ) || ""
+  );
+}
+
 export default {
   rangeToCumulative,
   cumulativeToTotals,