add historical labels

costmodel/costmodel.go

@@ -440,6 +440,7 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 		}
 	}
 	missingNodes := make(map[string]*costAnalyzerCloud.Node)
+	missingContainers := make(map[string]*CostData)
 	for key := range containers {
 		if _, ok := containerNameCost[key]; ok {
 			continue // because ordering is important for the allocation model (all PV's applied to the first), just dedupe if it's already been added.
@@ -602,35 +603,119 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 					missingNodes[c.NodeName] = node
 				}
 			}
+			namespacelabels, ok := namespaceLabelsMapping[c.Namespace]
+			if !ok {
+				klog.V(3).Infof("Missing data for namespace %s", c.Namespace)
+			}
 			costs := &CostData{
-				Name:      c.ContainerName,
-				PodName:   c.PodName,
-				NodeName:  c.NodeName,
-				NodeData:  node,
-				Namespace: c.Namespace,
-				RAMReq:    RAMReqV,
-				RAMUsed:   RAMUsedV,
-				CPUReq:    CPUReqV,
-				CPUUsed:   CPUUsedV,
-				GPUReq:    GPUReqV,
+				Name:            c.ContainerName,
+				PodName:         c.PodName,
+				NodeName:        c.NodeName,
+				NodeData:        node,
+				Namespace:       c.Namespace,
+				RAMReq:          RAMReqV,
+				RAMUsed:         RAMUsedV,
+				CPUReq:          CPUReqV,
+				CPUUsed:         CPUUsedV,
+				GPUReq:          GPUReqV,
+				NamespaceLabels: namespacelabels,
 			}
 			costs.CPUAllocation = getContainerAllocation(costs.CPUReq, costs.CPUUsed)
 			costs.RAMAllocation = getContainerAllocation(costs.RAMReq, costs.RAMUsed)
 			if filterNamespace == "" {
 				containerNameCost[key] = costs
+				missingContainers[key] = costs
 			} else if costs.Namespace == filterNamespace {
 				containerNameCost[key] = costs
+				missingContainers[key] = costs
 			}
 		}
 	}
 	err = findDeletedNodeInfo(cli, missingNodes, window)
 
+	if err != nil {
+		return nil, err
+	}
+	err = findDeletedPodInfo(cli, missingContainers, window)
 	if err != nil {
 		return nil, err
 	}
 	return containerNameCost, err
 }
 
+func findDeletedPodInfo(cli prometheusClient.Client, missingContainers map[string]*CostData, window string) error {
+	if len(missingContainers) > 0 {
+		q := make([]string, 0, len(missingContainers))
+		for key := range missingContainers {
+			cm, _ := NewContainerMetricFromKey(key)
+			q = append(q, cm.PodName)
+		}
+		l := strings.Join(q, "|")
+		queryHistoricalPodLabels := fmt.Sprintf(`kube_pod_labels{pod=~"%s"}[%s]`, l, window)
+
+		podLabelsResult, err := query(cli, queryHistoricalPodLabels)
+		if err != nil {
+			return fmt.Errorf("Error fetching historical pod labels: " + err.Error())
+		}
+		podLabels, err := labelsFromPrometheusQuery(podLabelsResult)
+		for key, costData := range missingContainers {
+			cm, _ := NewContainerMetricFromKey(key)
+			labels, ok := podLabels[cm.PodName]
+			if !ok {
+				klog.V(1).Infof("Unable to find historical data for pod '%s'", cm.PodName)
+			}
+			for k, v := range costData.NamespaceLabels {
+				if _, ok := labels[k]; !ok {
+					labels[k] = v
+				}
+			}
+			costData.Labels = labels
+		}
+	}
+
+	return nil
+}
+
+func labelsFromPrometheusQuery(qr interface{}) (map[string]map[string]string, error) {
+	toReturn := make(map[string]map[string]string)
+	for _, val := range qr.(map[string]interface{})["data"].(map[string]interface{})["result"].([]interface{}) {
+		metricInterface, ok := val.(map[string]interface{})["metric"]
+		if !ok {
+			return nil, fmt.Errorf("Metric field does not exist in data result vector")
+		}
+		metricMap, ok := metricInterface.(map[string]interface{})
+		if !ok {
+			return nil, fmt.Errorf("Metric field is improperly formatted")
+		}
+		pod, ok := metricMap["pod"]
+		if !ok {
+			return nil, fmt.Errorf("pod field does not exist in data result vector")
+		}
+		podName, ok := pod.(string)
+		if !ok {
+			return nil, fmt.Errorf("pod field is improperly formatted")
+		}
+
+		for labelName, labelValue := range metricMap {
+			parsedLabelName := labelName
+			parsedLv, ok := labelValue.(string)
+			if !ok {
+				return nil, fmt.Errorf("label value is improperly formatted")
+			}
+			if strings.HasPrefix(parsedLabelName, "label_") {
+				l := strings.Replace(parsedLabelName, "label_", "", 1)
+				if podLabels, ok := toReturn[podName]; ok {
+					podLabels[l] = parsedLv
+				} else {
+					toReturn[podName] = make(map[string]string)
+					toReturn[podName][l] = parsedLv
+				}
+			}
+		}
+	}
+	return toReturn, nil
+}
+
 func findDeletedNodeInfo(cli prometheusClient.Client, missingNodes map[string]*costAnalyzerCloud.Node, window string) error {
 	if len(missingNodes) > 0 {
 		q := make([]string, 0, len(missingNodes))
@@ -1206,7 +1291,7 @@ func (cm *CostModel) ComputeCostDataRange(cli prometheusClient.Client, clientset
 	}
 
 	missingNodes := make(map[string]*costAnalyzerCloud.Node)
-
+	missingContainers := make(map[string]*CostData)
 	for key := range containers {
 		if _, ok := containerNameCost[key]; ok {
 			continue // because ordering is important for the allocation model (all PV's applied to the first), just dedupe if it's already been added.
@@ -1366,35 +1451,48 @@ func (cm *CostModel) ComputeCostDataRange(cli prometheusClient.Client, clientset
 					missingNodes[c.NodeName] = node
 				}
 			}
+			namespacelabels, ok := namespaceLabelsMapping[c.Namespace]
+			if !ok {
+				klog.V(3).Infof("Missing data for namespace %s", c.Namespace)
+			}
 			costs := &CostData{
-				Name:      c.ContainerName,
-				PodName:   c.PodName,
-				NodeName:  c.NodeName,
-				NodeData:  node,
-				Namespace: c.Namespace,
-				RAMReq:    RAMReqV,
-				RAMUsed:   RAMUsedV,
-				CPUReq:    CPUReqV,
-				CPUUsed:   CPUUsedV,
-				GPUReq:    GPUReqV,
+				Name:            c.ContainerName,
+				PodName:         c.PodName,
+				NodeName:        c.NodeName,
+				NodeData:        node,
+				Namespace:       c.Namespace,
+				RAMReq:          RAMReqV,
+				RAMUsed:         RAMUsedV,
+				CPUReq:          CPUReqV,
+				CPUUsed:         CPUUsedV,
+				GPUReq:          GPUReqV,
+				NamespaceLabels: namespacelabels,
 			}
 			costs.CPUAllocation = getContainerAllocation(costs.CPUReq, costs.CPUUsed)
 			costs.RAMAllocation = getContainerAllocation(costs.RAMReq, costs.RAMUsed)
 			if filterNamespace == "" {
 				containerNameCost[key] = costs
+				missingContainers[key] = costs
 			} else if costs.Namespace == filterNamespace {
 				containerNameCost[key] = costs
+				missingContainers[key] = costs
 			}
 		}
 	}
 
 	w := end.Sub(start)
+	w += window
 	if w.Minutes() > 0 {
 		wStr := fmt.Sprintf("%dm", int(w.Minutes()))
 		err = findDeletedNodeInfo(cli, missingNodes, wStr)
 		if err != nil {
 			return nil, err
 		}
+		klog.Infof("Finding deleted pod info from range query:")
+		err = findDeletedPodInfo(cli, missingContainers, wStr)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return containerNameCost, err