|
|
@@ -111,6 +111,7 @@ type CostData struct {
|
|
|
CPUAllocation []*Vector `json:"cpuallocated,omitempty"`
|
|
|
GPUReq []*Vector `json:"gpureq,omitempty"`
|
|
|
PVCData []*PersistentVolumeClaimData `json:"pvcData,omitempty"`
|
|
|
+ NetworkData []*Vector `json:"network,omitempty"`
|
|
|
Labels map[string]string `json:"labels,omitempty"`
|
|
|
NamespaceLabels map[string]string `json:"namespaceLabels,omitempty"`
|
|
|
}
|
|
|
@@ -172,7 +173,10 @@ const (
|
|
|
*
|
|
|
on (persistentvolumeclaim, namespace) group_right(storageclass, volumename)
|
|
|
sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace)`
|
|
|
- normalizationStr = `max(count_over_time(kube_pod_container_resource_requests_memory_bytes{}[%s] %s))`
|
|
|
+ queryZoneNetworkUsage = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s] %s)) by (namespace,pod_name) / 1024 / 1024 / 1024`
|
|
|
+ queryRegionNetworkUsage = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s] %s)) by (namespace,pod_name) / 1024 / 1024 / 1024`
|
|
|
+ queryInternetNetworkUsage = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s] %s)) by (namespace,pod_name) / 1024 / 1024 / 1024`
|
|
|
+ normalizationStr = `max(count_over_time(kube_pod_container_resource_requests_memory_bytes{}[%s] %s))`
|
|
|
)
|
|
|
|
|
|
type PrometheusMetadata struct {
|
|
|
@@ -288,10 +292,13 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
|
|
|
queryCPUUsage := fmt.Sprintf(queryCPUUsageStr, window, offset)
|
|
|
queryGPURequests := fmt.Sprintf(queryGPURequestsStr, window, offset, window, offset)
|
|
|
queryPVRequests := fmt.Sprintf(queryPVRequestsStr)
|
|
|
+ queryNetZoneRequests := fmt.Sprintf(queryZoneNetworkUsage, window, "")
|
|
|
+ queryNetRegionRequests := fmt.Sprintf(queryRegionNetworkUsage, window, "")
|
|
|
+ queryNetInternetRequests := fmt.Sprintf(queryInternetNetworkUsage, window, "")
|
|
|
normalization := fmt.Sprintf(normalizationStr, window, offset)
|
|
|
|
|
|
var wg sync.WaitGroup
|
|
|
- wg.Add(8)
|
|
|
+ wg.Add(11)
|
|
|
|
|
|
var promErr error
|
|
|
var resultRAMRequests interface{}
|
|
|
@@ -324,6 +331,21 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
|
|
|
resultPVRequests, promErr = query(cli, queryPVRequests)
|
|
|
defer wg.Done()
|
|
|
}()
|
|
|
+ var resultNetZoneRequests interface{}
|
|
|
+ go func() {
|
|
|
+ resultNetZoneRequests, promErr = query(cli, queryNetZoneRequests)
|
|
|
+ defer wg.Done()
|
|
|
+ }()
|
|
|
+ var resultNetRegionRequests interface{}
|
|
|
+ go func() {
|
|
|
+ resultNetRegionRequests, promErr = query(cli, queryNetRegionRequests)
|
|
|
+ defer wg.Done()
|
|
|
+ }()
|
|
|
+ var resultNetInternetRequests interface{}
|
|
|
+ go func() {
|
|
|
+ resultNetInternetRequests, promErr = query(cli, queryNetInternetRequests)
|
|
|
+ defer wg.Done()
|
|
|
+ }()
|
|
|
var normalizationResult interface{}
|
|
|
go func() {
|
|
|
normalizationResult, promErr = query(cli, normalization)
|
|
|
@@ -385,6 +407,12 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ networkUsageMap, err := getNetworkUsageData(resultNetZoneRequests, resultNetRegionRequests, resultNetInternetRequests, false)
|
|
|
+ if err != nil {
|
|
|
+ klog.V(1).Infof("Unable to get Network Cost Data: %s", err.Error())
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+
|
|
|
containerNameCost := make(map[string]*CostData)
|
|
|
containers := make(map[string]bool)
|
|
|
|
|
|
@@ -481,6 +509,16 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ var podNetCosts []*Vector
|
|
|
+ if usage, ok := networkUsageMap[ns+","+podName]; ok {
|
|
|
+ netCosts, err := getNetworkCost(usage, cloud)
|
|
|
+ if err != nil {
|
|
|
+ klog.V(3).Infof("Error pulling network costs: %s", err.Error())
|
|
|
+ } else {
|
|
|
+ podNetCosts = netCosts
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
var podServices []string
|
|
|
if _, ok := podServicesMapping[ns]; ok {
|
|
|
if svcs, ok := podServicesMapping[ns][pod.GetObjectMeta().GetName()]; ok {
|
|
|
@@ -523,8 +561,10 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
|
|
|
}
|
|
|
|
|
|
var pvReq []*PersistentVolumeClaimData
|
|
|
+ var netReq []*Vector
|
|
|
if i == 0 { // avoid duplicating by just assigning all claims to the first container.
|
|
|
pvReq = podPVs
|
|
|
+ netReq = podNetCosts
|
|
|
}
|
|
|
|
|
|
costs := &CostData{
|
|
|
@@ -544,6 +584,7 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
|
|
|
CPUUsed: CPUUsedV,
|
|
|
GPUReq: GPUReqV,
|
|
|
PVCData: pvReq,
|
|
|
+ NetworkData: netReq,
|
|
|
Labels: podLabels,
|
|
|
NamespaceLabels: nsLabels,
|
|
|
}
|
|
|
@@ -1422,6 +1463,22 @@ type PersistentVolumeClaimData struct {
|
|
|
Values []*Vector `json:"values"`
|
|
|
}
|
|
|
|
|
|
+// NetworkUsageVNetworkUsageDataector contains the network usage values for egress network traffic
|
|
|
+type NetworkUsageData struct {
|
|
|
+ PodName string
|
|
|
+ Namespace string
|
|
|
+ NetworkZoneEgress []*Vector
|
|
|
+ NetworkRegionEgress []*Vector
|
|
|
+ NetworkInternetEgress []*Vector
|
|
|
+}
|
|
|
+
|
|
|
+// NetworkUsageVector contains a network usage vector for egress network traffic
|
|
|
+type NetworkUsageVector struct {
|
|
|
+ PodName string
|
|
|
+ Namespace string
|
|
|
+ Values []*Vector
|
|
|
+}
|
|
|
+
|
|
|
func getCost(qr interface{}) (map[string][]*Vector, error) {
|
|
|
toReturn := make(map[string][]*Vector)
|
|
|
for _, val := range qr.(map[string]interface{})["data"].(map[string]interface{})["result"].([]interface{}) {
|
|
|
@@ -1928,6 +1985,282 @@ func GetContainerMetricVectors(qr interface{}, normalize bool, normalizationValu
|
|
|
return containerData, nil
|
|
|
}
|
|
|
|
|
|
+func getNetworkUsageData(zr interface{}, rr interface{}, ir interface{}, isRange bool) (map[string]*NetworkUsageData, error) {
|
|
|
+ var vectorFn func(interface{}) (map[string]*NetworkUsageVector, error)
|
|
|
+
|
|
|
+ if isRange {
|
|
|
+ vectorFn = getNetworkUsageVectors
|
|
|
+ } else {
|
|
|
+ vectorFn = getNetworkUsageVector
|
|
|
+ }
|
|
|
+
|
|
|
+ zoneNetworkMap, err := vectorFn(zr)
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+
|
|
|
+ regionNetworkMap, err := vectorFn(rr)
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+
|
|
|
+ internetNetworkMap, err := vectorFn(ir)
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+
|
|
|
+ usageData := make(map[string]*NetworkUsageData)
|
|
|
+ for k, v := range zoneNetworkMap {
|
|
|
+ existing, ok := usageData[k]
|
|
|
+ if !ok {
|
|
|
+ usageData[k] = &NetworkUsageData{
|
|
|
+ PodName: v.PodName,
|
|
|
+ Namespace: v.Namespace,
|
|
|
+ NetworkZoneEgress: v.Values,
|
|
|
+ }
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ existing.NetworkZoneEgress = v.Values
|
|
|
+ }
|
|
|
+
|
|
|
+ for k, v := range regionNetworkMap {
|
|
|
+ existing, ok := usageData[k]
|
|
|
+ if !ok {
|
|
|
+ usageData[k] = &NetworkUsageData{
|
|
|
+ PodName: v.PodName,
|
|
|
+ Namespace: v.Namespace,
|
|
|
+ NetworkRegionEgress: v.Values,
|
|
|
+ }
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ existing.NetworkRegionEgress = v.Values
|
|
|
+ }
|
|
|
+
|
|
|
+ for k, v := range internetNetworkMap {
|
|
|
+ existing, ok := usageData[k]
|
|
|
+ if !ok {
|
|
|
+ usageData[k] = &NetworkUsageData{
|
|
|
+ PodName: v.PodName,
|
|
|
+ Namespace: v.Namespace,
|
|
|
+ NetworkInternetEgress: v.Values,
|
|
|
+ }
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ existing.NetworkInternetEgress = v.Values
|
|
|
+ }
|
|
|
+
|
|
|
+ return usageData, nil
|
|
|
+}
|
|
|
+
|
|
|
+func getNetworkUsageVector(qr interface{}) (map[string]*NetworkUsageVector, error) {
|
|
|
+ ncdmap := make(map[string]*NetworkUsageVector)
|
|
|
+ data, ok := qr.(map[string]interface{})["data"]
|
|
|
+ if !ok {
|
|
|
+ e, err := wrapPrometheusError(qr)
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ return nil, fmt.Errorf(e)
|
|
|
+ }
|
|
|
+ d, ok := data.(map[string]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Data field improperly formatted in prometheus repsonse")
|
|
|
+ }
|
|
|
+ result, ok := d["result"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Result field not present in prometheus response")
|
|
|
+ }
|
|
|
+ results, ok := result.([]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Result field improperly formatted in prometheus response")
|
|
|
+ }
|
|
|
+ for _, val := range results {
|
|
|
+ metricInterface, ok := val.(map[string]interface{})["metric"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Metric field does not exist in data result vector")
|
|
|
+ }
|
|
|
+ metricMap, ok := metricInterface.(map[string]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Metric field is improperly formatted")
|
|
|
+ }
|
|
|
+
|
|
|
+ podName, ok := metricMap["pod_name"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Pod Name does not exist in data result vector")
|
|
|
+ }
|
|
|
+ podNameStr, ok := podName.(string)
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Pod Name field improperly formatted")
|
|
|
+ }
|
|
|
+ namespace, ok := metricMap["namespace"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Namespace field does not exist in data result vector")
|
|
|
+ }
|
|
|
+ namespaceStr, ok := namespace.(string)
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Namespace field improperly formatted")
|
|
|
+ }
|
|
|
+ dataPoint, ok := val.(map[string]interface{})["value"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Value field does not exist in data result vector")
|
|
|
+ }
|
|
|
+ value, ok := dataPoint.([]interface{})
|
|
|
+ if !ok || len(value) != 2 {
|
|
|
+ return nil, fmt.Errorf("Improperly formatted datapoint from Prometheus")
|
|
|
+ }
|
|
|
+ var vectors []*Vector
|
|
|
+ strVal := value[1].(string)
|
|
|
+ v, _ := strconv.ParseFloat(strVal, 64)
|
|
|
+
|
|
|
+ vectors = append(vectors, &Vector{
|
|
|
+ Timestamp: value[0].(float64),
|
|
|
+ Value: v,
|
|
|
+ })
|
|
|
+
|
|
|
+ key := namespaceStr + "," + podNameStr
|
|
|
+ ncdmap[key] = &NetworkUsageVector{
|
|
|
+ Namespace: namespaceStr,
|
|
|
+ PodName: podNameStr,
|
|
|
+ Values: vectors,
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return ncdmap, nil
|
|
|
+}
|
|
|
+
|
|
|
+func getNetworkUsageVectors(qr interface{}) (map[string]*NetworkUsageVector, error) {
|
|
|
+ ncdmap := make(map[string]*NetworkUsageVector)
|
|
|
+ data, ok := qr.(map[string]interface{})["data"]
|
|
|
+ if !ok {
|
|
|
+ e, err := wrapPrometheusError(qr)
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ return nil, fmt.Errorf(e)
|
|
|
+ }
|
|
|
+ d, ok := data.(map[string]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Data field improperly formatted in prometheus repsonse")
|
|
|
+ }
|
|
|
+ result, ok := d["result"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Result field not present in prometheus response")
|
|
|
+ }
|
|
|
+ results, ok := result.([]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Result field improperly formatted in prometheus response")
|
|
|
+ }
|
|
|
+ for _, val := range results {
|
|
|
+ metricInterface, ok := val.(map[string]interface{})["metric"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Metric field does not exist in data result vector")
|
|
|
+ }
|
|
|
+ metricMap, ok := metricInterface.(map[string]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Metric field is improperly formatted")
|
|
|
+ }
|
|
|
+
|
|
|
+ podName, ok := metricMap["pod_name"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Pod Name does not exist in data result vector")
|
|
|
+ }
|
|
|
+ podNameStr, ok := podName.(string)
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Pod Name field improperly formatted")
|
|
|
+ }
|
|
|
+ namespace, ok := metricMap["namespace"]
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Namespace field does not exist in data result vector")
|
|
|
+ }
|
|
|
+ namespaceStr, ok := namespace.(string)
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Namespace field improperly formatted")
|
|
|
+ }
|
|
|
+ values, ok := val.(map[string]interface{})["values"].([]interface{})
|
|
|
+ if !ok {
|
|
|
+ return nil, fmt.Errorf("Values field is improperly formatted")
|
|
|
+ }
|
|
|
+ var vectors []*Vector
|
|
|
+ for _, value := range values {
|
|
|
+ dataPoint, ok := value.([]interface{})
|
|
|
+ if !ok || len(dataPoint) != 2 {
|
|
|
+ return nil, fmt.Errorf("Improperly formatted datapoint from Prometheus")
|
|
|
+ }
|
|
|
+
|
|
|
+ strVal := dataPoint[1].(string)
|
|
|
+ v, _ := strconv.ParseFloat(strVal, 64)
|
|
|
+ vectors = append(vectors, &Vector{
|
|
|
+ Timestamp: math.Round(dataPoint[0].(float64)/10) * 10,
|
|
|
+ Value: v,
|
|
|
+ })
|
|
|
+ }
|
|
|
+
|
|
|
+ key := namespaceStr + "," + podNameStr
|
|
|
+ ncdmap[key] = &NetworkUsageVector{
|
|
|
+ Namespace: namespaceStr,
|
|
|
+ PodName: podNameStr,
|
|
|
+ Values: vectors,
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return ncdmap, nil
|
|
|
+}
|
|
|
+
|
|
|
+func max(x int, rest ...int) int {
|
|
|
+ curr := x
|
|
|
+ for _, v := range rest {
|
|
|
+ if v > curr {
|
|
|
+ curr = v
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return curr
|
|
|
+}
|
|
|
+
|
|
|
+func getNetworkCost(usage *NetworkUsageData, cloud costAnalyzerCloud.Provider) ([]*Vector, error) {
|
|
|
+ var results []*Vector
|
|
|
+
|
|
|
+ pricing, err := cloud.NetworkPricing()
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ zoneCost := pricing.ZoneNetworkEgressCost
|
|
|
+ regionCost := pricing.RegionNetworkEgressCost
|
|
|
+ internetCost := pricing.InternetNetworkEgressCost
|
|
|
+
|
|
|
+ zlen := len(usage.NetworkZoneEgress)
|
|
|
+ rlen := len(usage.NetworkRegionEgress)
|
|
|
+ ilen := len(usage.NetworkInternetEgress)
|
|
|
+
|
|
|
+ l := max(zlen, rlen, ilen)
|
|
|
+ for i := 0; i < l; i++ {
|
|
|
+ var cost float64 = 0
|
|
|
+ var timestamp float64
|
|
|
+
|
|
|
+ if i < zlen {
|
|
|
+ cost += usage.NetworkZoneEgress[i].Value * zoneCost
|
|
|
+ timestamp = usage.NetworkZoneEgress[i].Timestamp
|
|
|
+ }
|
|
|
+
|
|
|
+ if i < rlen {
|
|
|
+ cost += usage.NetworkRegionEgress[i].Value * regionCost
|
|
|
+ timestamp = usage.NetworkRegionEgress[i].Timestamp
|
|
|
+ }
|
|
|
+
|
|
|
+ if i < ilen {
|
|
|
+ cost += usage.NetworkInternetEgress[i].Value * internetCost
|
|
|
+ timestamp = usage.NetworkInternetEgress[i].Timestamp
|
|
|
+ }
|
|
|
+
|
|
|
+ results = append(results, &Vector{
|
|
|
+ Value: cost,
|
|
|
+ Timestamp: timestamp,
|
|
|
+ })
|
|
|
+ }
|
|
|
+
|
|
|
+ return results, nil
|
|
|
+}
|
|
|
+
|
|
|
func wrapPrometheusError(qr interface{}) (string, error) {
|
|
|
e, ok := qr.(map[string]interface{})["error"]
|
|
|
if !ok {
|
Matt Bolt