|
|
@@ -107,17 +107,22 @@ func NewClusterCostsFromCumulative(cpu, gpu, ram, storage float64, window, offse
|
|
|
}
|
|
|
|
|
|
type Disk struct {
|
|
|
- Cluster string
|
|
|
- Name string
|
|
|
- ProviderID string
|
|
|
- StorageClass string
|
|
|
- Cost float64
|
|
|
- Bytes float64
|
|
|
- Local bool
|
|
|
- Start time.Time
|
|
|
- End time.Time
|
|
|
- Minutes float64
|
|
|
- Breakdown *ClusterCostsBreakdown
|
|
|
+ Cluster string
|
|
|
+ Name string
|
|
|
+ ProviderID string
|
|
|
+ StorageClass string
|
|
|
+ VolumeName string
|
|
|
+ ClaimName string
|
|
|
+ ClaimNamespace string
|
|
|
+ Cost float64
|
|
|
+ Bytes float64
|
|
|
+ BytesUsedAvg float64
|
|
|
+ BytesUsedMax float64
|
|
|
+ Local bool
|
|
|
+ Start time.Time
|
|
|
+ End time.Time
|
|
|
+ Minutes float64
|
|
|
+ Breakdown *ClusterCostsBreakdown
|
|
|
}
|
|
|
|
|
|
type DiskIdentifier struct {
|
|
|
@@ -159,8 +164,13 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, start, end
|
|
|
queryPVSize := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s])) by (%s, persistentvolume)`, durStr, env.GetPromClusterLabel())
|
|
|
queryActiveMins := fmt.Sprintf(`avg(kube_persistentvolume_capacity_bytes) by (%s, persistentvolume)[%s:%dm]`, env.GetPromClusterLabel(), durStr, minsPerResolution)
|
|
|
queryPVStorageClass := fmt.Sprintf(`avg(avg_over_time(kubecost_pv_info[%s])) by (%s, persistentvolume, storageclass)`, durStr, env.GetPromClusterLabel())
|
|
|
+ queryPVUsedAvg := fmt.Sprintf(`avg(avg_over_time(kubelet_volume_stats_used_bytes[%s])) by (%s, persistentvolumeclaim, namespace)`, durStr, env.GetPromClusterLabel())
|
|
|
+ queryPVUsedMax := fmt.Sprintf(`max(max_over_time(kubelet_volume_stats_used_bytes[%s])) by (%s, persistentvolumeclaim, namespace)`, durStr, env.GetPromClusterLabel())
|
|
|
+ queryPVCInfo := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolumeclaim_info[%s])) by (%s, volumename, persistentvolumeclaim, namespace)`, durStr, env.GetPromClusterLabel())
|
|
|
queryLocalStorageCost := fmt.Sprintf(`sum_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
|
|
|
queryLocalStorageUsedCost := fmt.Sprintf(`sum_over_time(sum(container_fs_usage_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
|
|
|
+ queryLocalStorageUsedAvg := fmt.Sprintf(`avg(avg_over_time(container_fs_usage_bytes{device!="tmpfs", id="/"}[%s])) by (instance, %s)`, durStr, env.GetPromClusterLabel())
|
|
|
+ queryLocalStorageUsedMax := fmt.Sprintf(`max(max_over_time(container_fs_usage_bytes{device!="tmpfs", id="/"}[%s])) by (instance, %s)`, durStr, env.GetPromClusterLabel())
|
|
|
queryLocalStorageBytes := fmt.Sprintf(`avg_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm])`, env.GetPromClusterLabel(), durStr, minsPerResolution)
|
|
|
queryLocalActiveMins := fmt.Sprintf(`count(node_total_hourly_cost) by (%s, node)[%s:%dm]`, env.GetPromClusterLabel(), durStr, minsPerResolution)
|
|
|
|
|
|
@@ -168,8 +178,13 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, start, end
|
|
|
resChPVSize := ctx.QueryAtTime(queryPVSize, t)
|
|
|
resChActiveMins := ctx.QueryAtTime(queryActiveMins, t)
|
|
|
resChPVStorageClass := ctx.QueryAtTime(queryPVStorageClass, t)
|
|
|
+ resChPVUsedAvg := ctx.QueryAtTime(queryPVUsedAvg, t)
|
|
|
+ resChPVUsedMax := ctx.QueryAtTime(queryPVUsedMax, t)
|
|
|
+ resChPVCInfo := ctx.QueryAtTime(queryPVCInfo, t)
|
|
|
resChLocalStorageCost := ctx.QueryAtTime(queryLocalStorageCost, t)
|
|
|
resChLocalStorageUsedCost := ctx.QueryAtTime(queryLocalStorageUsedCost, t)
|
|
|
+ resChLocalStoreageUsedAvg := ctx.QueryAtTime(queryLocalStorageUsedAvg, t)
|
|
|
+ resChLocalStoreageUsedMax := ctx.QueryAtTime(queryLocalStorageUsedMax, t)
|
|
|
resChLocalStorageBytes := ctx.QueryAtTime(queryLocalStorageBytes, t)
|
|
|
resChLocalActiveMins := ctx.QueryAtTime(queryLocalActiveMins, t)
|
|
|
|
|
|
@@ -177,8 +192,13 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, start, end
|
|
|
resPVSize, _ := resChPVSize.Await()
|
|
|
resActiveMins, _ := resChActiveMins.Await()
|
|
|
resPVStorageClass, _ := resChPVStorageClass.Await()
|
|
|
+ resPVUsedAvg, _ := resChPVUsedAvg.Await()
|
|
|
+ resPVUsedMax, _ := resChPVUsedMax.Await()
|
|
|
+ resPVCInfo, _ := resChPVCInfo.Await()
|
|
|
resLocalStorageCost, _ := resChLocalStorageCost.Await()
|
|
|
resLocalStorageUsedCost, _ := resChLocalStorageUsedCost.Await()
|
|
|
+ resLocalStorageUsedAvg, _ := resChLocalStoreageUsedAvg.Await()
|
|
|
+ resLocalStorageUsedMax, _ := resChLocalStoreageUsedMax.Await()
|
|
|
resLocalStorageBytes, _ := resChLocalStorageBytes.Await()
|
|
|
resLocalActiveMins, _ := resChLocalActiveMins.Await()
|
|
|
|
|
|
@@ -188,7 +208,43 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, start, end
|
|
|
|
|
|
diskMap := map[DiskIdentifier]*Disk{}
|
|
|
|
|
|
- pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost, provider)
|
|
|
+ for _, result := range resPVCInfo {
|
|
|
+ cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ cluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ volumeName, err := result.GetString("volumename")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing volumename")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ claimName, err := result.GetString("persistentvolumeclaim")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing persistentvolumeclaim")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ claimNamespace, err := result.GetString("namespace")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing namespace")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ key := DiskIdentifier{cluster, volumeName}
|
|
|
+ if _, ok := diskMap[key]; !ok {
|
|
|
+ diskMap[key] = &Disk{
|
|
|
+ Cluster: cluster,
|
|
|
+ Name: volumeName,
|
|
|
+ Breakdown: &ClusterCostsBreakdown{},
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ diskMap[key].VolumeName = volumeName
|
|
|
+ diskMap[key].ClaimName = claimName
|
|
|
+ diskMap[key].ClaimNamespace = claimNamespace
|
|
|
+ }
|
|
|
+
|
|
|
+ pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo, provider)
|
|
|
|
|
|
for _, result := range resLocalStorageCost {
|
|
|
cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
@@ -243,6 +299,56 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, start, end
|
|
|
diskMap[key].Breakdown.System = cost / diskMap[key].Cost
|
|
|
}
|
|
|
|
|
|
+ for _, result := range resLocalStorageUsedAvg {
|
|
|
+ cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ cluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ name, err := result.GetString("instance")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: local storage data missing instance")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ bytesAvg := result.Values[0].Value
|
|
|
+ key := DiskIdentifier{cluster, name}
|
|
|
+ if _, ok := diskMap[key]; !ok {
|
|
|
+ diskMap[key] = &Disk{
|
|
|
+ Cluster: cluster,
|
|
|
+ Name: name,
|
|
|
+ Breakdown: &ClusterCostsBreakdown{},
|
|
|
+ Local: true,
|
|
|
+ }
|
|
|
+ }
|
|
|
+ diskMap[key].BytesUsedAvg = bytesAvg
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, result := range resLocalStorageUsedMax {
|
|
|
+ cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ cluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ name, err := result.GetString("instance")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: local storage data missing instance")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ bytesMax := result.Values[0].Value
|
|
|
+ key := DiskIdentifier{cluster, name}
|
|
|
+ if _, ok := diskMap[key]; !ok {
|
|
|
+ diskMap[key] = &Disk{
|
|
|
+ Cluster: cluster,
|
|
|
+ Name: name,
|
|
|
+ Breakdown: &ClusterCostsBreakdown{},
|
|
|
+ Local: true,
|
|
|
+ }
|
|
|
+ }
|
|
|
+ diskMap[key].BytesUsedMax = bytesMax
|
|
|
+ }
|
|
|
+
|
|
|
for _, result := range resLocalStorageBytes {
|
|
|
cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
if err != nil {
|
|
|
@@ -1181,7 +1287,7 @@ func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startS
|
|
|
}, nil
|
|
|
}
|
|
|
|
|
|
-func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost []*prom.QueryResult, cp cloud.Provider) {
|
|
|
+func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo []*prom.QueryResult, cp cloud.Provider) {
|
|
|
for _, result := range resActiveMins {
|
|
|
cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
if err != nil {
|
|
|
@@ -1291,4 +1397,126 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
|
|
|
diskMap[key].ProviderID = cloud.ParsePVID(providerID)
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+ for _, result := range resPVUsedAvg {
|
|
|
+ cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ cluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ claimName, err := result.GetString("persistentvolumeclaim")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv usage data missing persistentvolumeclaim")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ claimNamespace, err := result.GetString("namespace")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv usage data missing namespace")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ var volumeName string
|
|
|
+
|
|
|
+ for _, thatRes := range resPVCInfo {
|
|
|
+
|
|
|
+ thatCluster, err := thatRes.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ thatCluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ thatVolumeName, err := thatRes.GetString("volumename")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing volumename")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ thatClaimName, err := thatRes.GetString("persistentvolumeclaim")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing persistentvolumeclaim")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ thatClaimNamespace, err := thatRes.GetString("namespace")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing namespace")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ if cluster == thatCluster && claimName == thatClaimName && claimNamespace == thatClaimNamespace {
|
|
|
+ volumeName = thatVolumeName
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ usage := result.Values[0].Value
|
|
|
+
|
|
|
+ key := DiskIdentifier{cluster, volumeName}
|
|
|
+
|
|
|
+ if _, ok := diskMap[key]; !ok {
|
|
|
+ diskMap[key] = &Disk{
|
|
|
+ Cluster: cluster,
|
|
|
+ Name: volumeName,
|
|
|
+ Breakdown: &ClusterCostsBreakdown{},
|
|
|
+ }
|
|
|
+ }
|
|
|
+ diskMap[key].BytesUsedAvg = usage
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, result := range resPVUsedMax {
|
|
|
+ cluster, err := result.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ cluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ claimName, err := result.GetString("persistentvolumeclaim")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv usage data missing persistentvolumeclaim")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ claimNamespace, err := result.GetString("namespace")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv usage data missing namespace")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ var volumeName string
|
|
|
+
|
|
|
+ for _, thatRes := range resPVCInfo {
|
|
|
+
|
|
|
+ thatCluster, err := thatRes.GetString(env.GetPromClusterLabel())
|
|
|
+ if err != nil {
|
|
|
+ thatCluster = env.GetClusterID()
|
|
|
+ }
|
|
|
+
|
|
|
+ thatVolumeName, err := thatRes.GetString("volumename")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing volumename")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ thatClaimName, err := thatRes.GetString("persistentvolumeclaim")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing persistentvolumeclaim")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ thatClaimNamespace, err := thatRes.GetString("namespace")
|
|
|
+ if err != nil {
|
|
|
+ log.Warnf("ClusterDisks: pv claim data missing namespace")
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ if cluster == thatCluster && claimName == thatClaimName && claimNamespace == thatClaimNamespace {
|
|
|
+ volumeName = thatVolumeName
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ usage := result.Values[0].Value
|
|
|
+
|
|
|
+ key := DiskIdentifier{cluster, volumeName}
|
|
|
+
|
|
|
+ if _, ok := diskMap[key]; !ok {
|
|
|
+ diskMap[key] = &Disk{
|
|
|
+ Cluster: cluster,
|
|
|
+ Name: volumeName,
|
|
|
+ Breakdown: &ClusterCostsBreakdown{},
|
|
|
+ }
|
|
|
+ }
|
|
|
+ diskMap[key].BytesUsedMax = usage
|
|
|
+ }
|
|
|
}
|