|
|
@@ -19,45 +19,47 @@ import (
|
|
|
)
|
|
|
|
|
|
const (
|
|
|
- queryFmtPods = `avg(kube_pod_container_status_running{}) by (pod, namespace, %s)[%s:%s]%s`
|
|
|
- queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s, provider_id)`
|
|
|
- queryFmtRAMRequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
- queryFmtRAMUsageAvg = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
- queryFmtRAMUsageMax = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
- queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
- queryFmtCPURequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
- queryFmtCPUUsageAvg = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
- queryFmtCPUUsageMax = `max(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
- queryFmtGPUsRequested = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
- queryFmtGPUsAllocated = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
- queryFmtNodeCostPerCPUHr = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
|
|
|
- queryFmtNodeCostPerRAMGiBHr = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
|
|
|
- queryFmtNodeCostPerGPUHr = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
|
|
|
- queryFmtNodeIsSpot = `avg_over_time(kubecost_node_is_spot[%s]%s)`
|
|
|
- queryFmtPVCInfo = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]%s`
|
|
|
- queryFmtPVBytes = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, %s)`
|
|
|
- queryFmtPodPVCAllocation = `avg(avg_over_time(pod_pvc_allocation[%s]%s)) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
|
|
|
- queryFmtPVCBytesRequested = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s]%s)) by (persistentvolumeclaim, namespace, %s)`
|
|
|
- queryFmtPVCostPerGiBHour = `avg(avg_over_time(pv_hourly_cost[%s]%s)) by (volumename, %s)`
|
|
|
- queryFmtNetZoneGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
|
|
|
- queryFmtNetZoneCostPerGiB = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s]%s)) by (%s)`
|
|
|
- queryFmtNetRegionGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
|
|
|
- queryFmtNetRegionCostPerGiB = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s]%s)) by (%s)`
|
|
|
- queryFmtNetInternetGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
|
|
|
- queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s]%s)) by (%s)`
|
|
|
- queryFmtNetReceiveBytes = `sum(increase(container_network_receive_bytes_total{pod!="", container="POD"}[%s]%s)) by (pod_name, pod, namespace, %s)`
|
|
|
- queryFmtNetTransferBytes = `sum(increase(container_network_transmit_bytes_total{pod!="", container="POD"}[%s]%s)) by (pod_name, pod, namespace, %s)`
|
|
|
- queryFmtNamespaceLabels = `avg_over_time(kube_namespace_labels[%s]%s)`
|
|
|
- queryFmtNamespaceAnnotations = `avg_over_time(kube_namespace_annotations[%s]%s)`
|
|
|
- queryFmtPodLabels = `avg_over_time(kube_pod_labels[%s]%s)`
|
|
|
- queryFmtPodAnnotations = `avg_over_time(kube_pod_annotations[%s]%s)`
|
|
|
- queryFmtServiceLabels = `avg_over_time(service_selector_labels[%s]%s)`
|
|
|
- queryFmtDeploymentLabels = `avg_over_time(deployment_match_labels[%s]%s)`
|
|
|
- queryFmtStatefulSetLabels = `avg_over_time(statefulSet_match_labels[%s]%s)`
|
|
|
- queryFmtDaemonSetLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s]%s)) by (pod, owner_name, namespace, %s)`
|
|
|
- queryFmtJobLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s]%s)) by (pod, owner_name, namespace ,%s)`
|
|
|
- queryFmtLBCostPerHr = `avg(avg_over_time(kubecost_load_balancer_cost[%s]%s)) by (namespace, service_name, %s)`
|
|
|
- queryFmtLBActiveMins = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]%s`
|
|
|
+ queryFmtPods = `avg(kube_pod_container_status_running{}) by (pod, namespace, %s)[%s:%s]%s`
|
|
|
+ queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s, provider_id)`
|
|
|
+ queryFmtRAMRequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
+ queryFmtRAMUsageAvg = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
+ queryFmtRAMUsageMax = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
+ queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
+ queryFmtCPURequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
+ queryFmtCPUUsageAvg = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
+ queryFmtCPUUsageMax = `max(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
|
|
|
+ queryFmtGPUsRequested = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
+ queryFmtGPUsAllocated = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
|
|
|
+ queryFmtNodeCostPerCPUHr = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
|
|
|
+ queryFmtNodeCostPerRAMGiBHr = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
|
|
|
+ queryFmtNodeCostPerGPUHr = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
|
|
|
+ queryFmtNodeIsSpot = `avg_over_time(kubecost_node_is_spot[%s]%s)`
|
|
|
+ queryFmtPVCInfo = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]%s`
|
|
|
+ queryFmtPVBytes = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, %s)`
|
|
|
+ queryFmtPodPVCAllocation = `avg(avg_over_time(pod_pvc_allocation[%s]%s)) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
|
|
|
+ queryFmtPVCBytesRequested = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s]%s)) by (persistentvolumeclaim, namespace, %s)`
|
|
|
+ queryFmtPVCostPerGiBHour = `avg(avg_over_time(pv_hourly_cost[%s]%s)) by (volumename, %s)`
|
|
|
+ queryFmtNetZoneGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
|
|
|
+ queryFmtNetZoneCostPerGiB = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s]%s)) by (%s)`
|
|
|
+ queryFmtNetRegionGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
|
|
|
+ queryFmtNetRegionCostPerGiB = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s]%s)) by (%s)`
|
|
|
+ queryFmtNetInternetGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
|
|
|
+ queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s]%s)) by (%s)`
|
|
|
+ queryFmtNetReceiveBytes = `sum(increase(container_network_receive_bytes_total{pod!="", container="POD"}[%s]%s)) by (pod_name, pod, namespace, %s)`
|
|
|
+ queryFmtNetTransferBytes = `sum(increase(container_network_transmit_bytes_total{pod!="", container="POD"}[%s]%s)) by (pod_name, pod, namespace, %s)`
|
|
|
+ queryFmtNamespaceLabels = `avg_over_time(kube_namespace_labels[%s]%s)`
|
|
|
+ queryFmtNamespaceAnnotations = `avg_over_time(kube_namespace_annotations[%s]%s)`
|
|
|
+ queryFmtPodLabels = `avg_over_time(kube_pod_labels[%s]%s)`
|
|
|
+ queryFmtPodAnnotations = `avg_over_time(kube_pod_annotations[%s]%s)`
|
|
|
+ queryFmtServiceLabels = `avg_over_time(service_selector_labels[%s]%s)`
|
|
|
+ queryFmtDeploymentLabels = `avg_over_time(deployment_match_labels[%s]%s)`
|
|
|
+ queryFmtStatefulSetLabels = `avg_over_time(statefulSet_match_labels[%s]%s)`
|
|
|
+ queryFmtDaemonSetLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s]%s)) by (pod, owner_name, namespace, %s)`
|
|
|
+ queryFmtJobLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s]%s)) by (pod, owner_name, namespace ,%s)`
|
|
|
+ queryFmtPodsWithReplicaSetOwner = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet"}[%s]%s)) by (pod, owner_name, namespace ,%s)`
|
|
|
+ queryFmtReplicaSetsWithoutOwners = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>"}[%s]%s)) by (replicaset, namespace, %s)`
|
|
|
+ queryFmtLBCostPerHr = `avg(avg_over_time(kubecost_load_balancer_cost[%s]%s)) by (namespace, service_name, %s)`
|
|
|
+ queryFmtLBActiveMins = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]%s`
|
|
|
)
|
|
|
|
|
|
// This is a bit of a hack to work around garbage data from cadvisor
|
|
|
@@ -231,6 +233,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
|
|
|
queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, durStr, offStr, env.GetPromClusterLabel())
|
|
|
resChDaemonSetLabels := ctx.Query(queryDaemonSetLabels)
|
|
|
|
|
|
+ queryPodsWithReplicaSetOwner := fmt.Sprintf(queryFmtPodsWithReplicaSetOwner, durStr, offStr, env.GetPromClusterLabel())
|
|
|
+ resChPodsWithReplicaSetOwner := ctx.Query(queryPodsWithReplicaSetOwner)
|
|
|
+
|
|
|
+ queryReplicaSetsWithoutOwners := fmt.Sprintf(queryFmtReplicaSetsWithoutOwners, durStr, offStr, env.GetPromClusterLabel())
|
|
|
+ resChReplicaSetsWithoutOwners := ctx.Query(queryReplicaSetsWithoutOwners)
|
|
|
+
|
|
|
queryJobLabels := fmt.Sprintf(queryFmtJobLabels, durStr, offStr, env.GetPromClusterLabel())
|
|
|
resChJobLabels := ctx.Query(queryJobLabels)
|
|
|
|
|
|
@@ -280,6 +288,8 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
|
|
|
resDeploymentLabels, _ := resChDeploymentLabels.Await()
|
|
|
resStatefulSetLabels, _ := resChStatefulSetLabels.Await()
|
|
|
resDaemonSetLabels, _ := resChDaemonSetLabels.Await()
|
|
|
+ resPodsWithReplicaSetOwner, _ := resChPodsWithReplicaSetOwner.Await()
|
|
|
+ resReplicaSetsWithoutOwners, _ := resChReplicaSetsWithoutOwners.Await()
|
|
|
resJobLabels, _ := resChJobLabels.Await()
|
|
|
resLBCostPerHr, _ := resChLBCostPerHr.Await()
|
|
|
resLBActiveMins, _ := resChLBActiveMins.Await()
|
|
|
@@ -324,10 +334,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
|
|
|
podStatefulSetMap := labelsToPodControllerMap(podLabels, resToStatefulSetLabels(resStatefulSetLabels))
|
|
|
podDaemonSetMap := resToPodDaemonSetMap(resDaemonSetLabels)
|
|
|
podJobMap := resToPodJobMap(resJobLabels)
|
|
|
+ podReplicaSetMap := resToPodReplicaSetMap(resPodsWithReplicaSetOwner, resReplicaSetsWithoutOwners)
|
|
|
applyControllersToPods(podMap, podDeploymentMap)
|
|
|
applyControllersToPods(podMap, podStatefulSetMap)
|
|
|
applyControllersToPods(podMap, podDaemonSetMap)
|
|
|
applyControllersToPods(podMap, podJobMap)
|
|
|
+ applyControllersToPods(podMap, podReplicaSetMap)
|
|
|
|
|
|
// TODO breakdown network costs?
|
|
|
|
|
|
@@ -1326,6 +1338,48 @@ func resToPodJobMap(resJobLabels []*prom.QueryResult) map[podKey]controllerKey {
|
|
|
return jobLabels
|
|
|
}
|
|
|
|
|
|
+func resToPodReplicaSetMap(resPodsWithReplicaSetOwner []*prom.QueryResult, resReplicaSetsWithoutOwners []*prom.QueryResult) map[podKey]controllerKey {
|
|
|
+ // Build out set of ReplicaSets that have no owners, themselves, such that
|
|
|
+ // the ReplicaSet should be used as the owner of the Pods it controls.
|
|
|
+ // (This should exclude, for example, ReplicaSets that are controlled by
|
|
|
+ // Deployments, in which case the Deployment should be the Pod's owner.)
|
|
|
+ replicaSets := map[controllerKey]struct{}{}
|
|
|
+
|
|
|
+ for _, res := range resReplicaSetsWithoutOwners {
|
|
|
+ controllerKey, err := resultReplicaSetKey(res, env.GetPromClusterLabel(), "namespace", "replicaset")
|
|
|
+ if err != nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ replicaSets[controllerKey] = struct{}{}
|
|
|
+ }
|
|
|
+
|
|
|
+ // Create the mapping of Pods to ReplicaSets, ignoring any ReplicaSets that
|
|
|
+ // to not appear in the set of uncontrolled ReplicaSets above.
|
|
|
+ podToReplicaSet := map[podKey]controllerKey{}
|
|
|
+
|
|
|
+ for _, res := range resPodsWithReplicaSetOwner {
|
|
|
+ controllerKey, err := resultReplicaSetKey(res, env.GetPromClusterLabel(), "namespace", "owner_name")
|
|
|
+ if err != nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if _, ok := replicaSets[controllerKey]; !ok {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ pod, err := res.GetString("pod")
|
|
|
+ if err != nil {
|
|
|
+ log.Warningf("CostModel.ComputeAllocation: ReplicaSet result without pod: %s", controllerKey)
|
|
|
+ }
|
|
|
+
|
|
|
+ podKey := newPodKey(controllerKey.Cluster, controllerKey.Namespace, pod)
|
|
|
+
|
|
|
+ podToReplicaSet[podKey] = controllerKey
|
|
|
+ }
|
|
|
+
|
|
|
+ return podToReplicaSet
|
|
|
+}
|
|
|
+
|
|
|
func applyServicesToPods(podMap map[podKey]*Pod, podLabels map[podKey]map[string]string, allocsByService map[serviceKey][]*kubecost.Allocation, serviceLabels map[serviceKey]map[string]string) {
|
|
|
podServicesMap := map[podKey][]serviceKey{}
|
|
|
|
kirbsauce