Merge branch 'develop' of github.com:kubecost/cost-model into neal/aggregate-by-label

Merge branch develop into branch neal/aggregate-by-label

pkg/costmodel/costmodel.go

@@ -177,10 +177,10 @@ const (
 		)
 	) by (namespace,container_name,pod_name,node,cluster_id)
 	* on (pod_name, namespace, cluster_id) group_left(container) label_replace(avg(avg_over_time(kube_pod_status_phase{phase="Running"}[%s] %s)) by (pod,namespace,cluster_id), "pod_name","$1","pod","(.+)")`
-	queryPVRequestsStr = `avg(avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, namespace, volumename, cluster_id)
+	queryPVRequestsStr = `avg(avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, namespace, volumename, cluster_id, kubernetes_node)
 	*
-	on (persistentvolumeclaim, namespace, cluster_id) group_right(storageclass, volumename)
-	sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{volumename != ""}) by (persistentvolumeclaim, namespace, cluster_id, kubernetes_name)) by (persistentvolumeclaim, storageclass, namespace, volumename, cluster_id)`
+	on (persistentvolumeclaim, namespace, cluster_id, kubernetes_node) group_right(storageclass, volumename)
+	sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{}) by (persistentvolumeclaim, namespace, cluster_id, kubernetes_node, kubernetes_name)) by (persistentvolumeclaim, storageclass, namespace, cluster_id, volumename, kubernetes_node)`
 	// queryRAMAllocationByteHours yields the total byte-hour RAM allocation over the given
 	// window, aggregated by container.
 	//  [line 3]  sum_over_time(each byte) = [byte*scrape] by metric

pkg/costmodel/metrics.go

@@ -950,44 +950,45 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				} else {
 					containerSeen[labelKey] = false
 				}
+			}
 
-				storageClasses := cmme.KubeClusterCache.GetAllStorageClasses()
-				storageClassMap := make(map[string]map[string]string)
-				for _, storageClass := range storageClasses {
-					params := storageClass.Parameters
-					storageClassMap[storageClass.ObjectMeta.Name] = params
-					if storageClass.GetAnnotations()["storageclass.kubernetes.io/is-default-class"] == "true" || storageClass.GetAnnotations()["storageclass.beta.kubernetes.io/is-default-class"] == "true" {
-						storageClassMap["default"] = params
-						storageClassMap[""] = params
-					}
+			storageClasses := cmme.KubeClusterCache.GetAllStorageClasses()
+			storageClassMap := make(map[string]map[string]string)
+			for _, storageClass := range storageClasses {
+				params := storageClass.Parameters
+				storageClassMap[storageClass.ObjectMeta.Name] = params
+				if storageClass.GetAnnotations()["storageclass.kubernetes.io/is-default-class"] == "true" || storageClass.GetAnnotations()["storageclass.beta.kubernetes.io/is-default-class"] == "true" {
+					storageClassMap["default"] = params
+					storageClassMap[""] = params
 				}
+			}
 
-				pvs := cmme.KubeClusterCache.GetAllPersistentVolumes()
-				for _, pv := range pvs {
-					parameters, ok := storageClassMap[pv.Spec.StorageClassName]
-					if !ok {
-						klog.V(4).Infof("Unable to find parameters for storage class \"%s\". Does pv \"%s\" have a storageClassName?", pv.Spec.StorageClassName, pv.Name)
-					}
-					var region string
-					if r, ok := pv.Labels[v1.LabelZoneRegion]; ok {
-						region = r
-					} else {
-						region = defaultRegion
-					}
-					cacPv := &cloud.PV{
-						Class:      pv.Spec.StorageClassName,
-						Region:     region,
-						Parameters: parameters,
-					}
-
-					// TODO: GetPVCost should be a method in CostModel?
-					GetPVCost(cacPv, pv, cmme.CloudProvider, region)
-					c, _ := strconv.ParseFloat(cacPv.Cost, 64)
-					cmme.PersistentVolumePriceRecorder.WithLabelValues(pv.Name, pv.Name, cacPv.ProviderID).Set(c)
-					labelKey := getKeyFromLabelStrings(pv.Name, pv.Name)
-					pvSeen[labelKey] = true
+			pvs := cmme.KubeClusterCache.GetAllPersistentVolumes()
+			for _, pv := range pvs {
+				parameters, ok := storageClassMap[pv.Spec.StorageClassName]
+				if !ok {
+					klog.V(4).Infof("Unable to find parameters for storage class \"%s\". Does pv \"%s\" have a storageClassName?", pv.Spec.StorageClassName, pv.Name)
+				}
+				var region string
+				if r, ok := pv.Labels[v1.LabelZoneRegion]; ok {
+					region = r
+				} else {
+					region = defaultRegion
 				}
+				cacPv := &cloud.PV{
+					Class:      pv.Spec.StorageClassName,
+					Region:     region,
+					Parameters: parameters,
+				}
+
+				// TODO: GetPVCost should be a method in CostModel?
+				GetPVCost(cacPv, pv, cmme.CloudProvider, region)
+				c, _ := strconv.ParseFloat(cacPv.Cost, 64)
+				cmme.PersistentVolumePriceRecorder.WithLabelValues(pv.Name, pv.Name, cacPv.ProviderID).Set(c)
+				labelKey := getKeyFromLabelStrings(pv.Name, pv.Name)
+				pvSeen[labelKey] = true
 			}
+
 			for labelString, seen := range nodeSeen {
 				if !seen {
 					klog.V(4).Infof("Removing %s from nodes", labelString)

pkg/kubecost/asset.go

@@ -135,7 +135,7 @@ type AssetLabels map[string]string
 
 // Clone returns a cloned map of labels
 func (al AssetLabels) Clone() AssetLabels {
-	clone := AssetLabels{}
+	clone := make(AssetLabels, len(al))
 
 	for label, value := range al {
 		clone[label] = value

pkg/kubecost/kubecost_codecs.go

@@ -14,11 +14,10 @@ package kubecost
 import (
 	"encoding"
 	"fmt"
+	util "github.com/kubecost/cost-model/pkg/util"
 	"reflect"
 	"strings"
 	"time"
-
-	util "github.com/kubecost/cost-model/pkg/util"
 )
 
 const (
@@ -26,7 +25,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 5
+	CodecVersion uint8 = 6
 )
 
 //--------------------------------------------------------------------------

pkg/kubecost/properties.go

@@ -65,7 +65,7 @@ func (p *Properties) Clone() Properties {
 		return nil
 	}
 
-	clone := Properties{}
+	clone := make(Properties, len(*p))
 	for k, v := range *p {
 		clone[k] = v
 	}
@@ -707,8 +707,8 @@ func (p *Properties) UnmarshalBinary(data []byte) error {
 
 	// LabelProp
 	if buff.ReadUInt8() == 1 { // read nil byte
-		labels := map[string]string{}
 		length := buff.ReadInt() // read map len
+		labels := make(map[string]string, length)
 		for idx := 0; idx < length; idx++ {
 			key := buff.ReadString()
 			val := buff.ReadString()
@@ -719,8 +719,8 @@ func (p *Properties) UnmarshalBinary(data []byte) error {
 
 	// AnnotationProp
 	if buff.ReadUInt8() == 1 { // read nil byte
-		annotations := map[string]string{}
 		length := buff.ReadInt() // read map len
+		annotations := make(map[string]string, length)
 		for idx := 0; idx < length; idx++ {
 			key := buff.ReadString()
 			val := buff.ReadString()
@@ -731,11 +731,11 @@ func (p *Properties) UnmarshalBinary(data []byte) error {
 
 	// ServiceProp
 	if buff.ReadUInt8() == 1 { // read nil byte
-		services := []string{}
 		length := buff.ReadInt() // read map len
+		services := make([]string, length)
 		for idx := 0; idx < length; idx++ {
 			val := buff.ReadString()
-			services = append(services, val)
+			services[idx] = val
 		}
 		p.SetServices(services)
 	}