Clean up

core/pkg/source/datasource.go

@@ -16,8 +16,6 @@ type MetricsQuerier interface {
 
 	// Local Cluster Disks
 	QueryLocalStorageActiveMinutes(start, end time.Time) *Future[LocalStorageActiveMinutesResult]
-	// QueryLocalStorageCost(start, end time.Time) *Future[LocalStorageCostResult]
-	// QueryLocalStorageUsedCost(start, end time.Time) *Future[LocalStorageUsedCostResult]
 	QueryLocalStorageUsedAvg(start, end time.Time) *Future[LocalStorageUsedAvgResult]
 	QueryLocalStorageUsedMax(start, end time.Time) *Future[LocalStorageUsedMaxResult]
 	QueryLocalStorageBytes(start, end time.Time) *Future[LocalStorageBytesResult]

modules/collector-source/pkg/collector/collector.go

@@ -235,8 +235,6 @@ func NewPVActiveMinutesMetricCollector() *metric.MetricCollector {
 //	) by (instance, device, cluster_id)[%s:%dm]
 //
 // ) / 1024 / 1024 / 1024 * %f * %f`
-// NewLocalStorageUsedActiveMinutesMetricCollector does not have an associated query end point but is used in the results
-// of QueryLocalStorageUsedCost
 func NewLocalStorageUsedActiveMinutesMetricCollector() *metric.MetricCollector {
 	return metric.NewMetricCollector(
 		metric.LocalStorageUsedActiveMinutesID,

modules/collector-source/pkg/collector/metricsquerier.go

@@ -74,102 +74,6 @@ func (c *collectorMetricsQuerier) QueryLocalStorageActiveMinutes(start, end time
 	return queryCollector(c, start, end, metric.LocalStorageActiveMinutesID, source.DecodeLocalStorageActiveMinutesResult)
 }
 
-// func (c *collectorMetricsQuerier) QueryLocalStorageCost(start, end time.Time) *source.Future[source.LocalStorageCostResult] {
-// 	queryResults := source.NewQueryResults("LocalStorageCost")
-// 	collector := c.collectorProvider.GetStore(start, end)
-// 	if collector != nil {
-// 		minutesResults, err := collector.Query(metric.LocalStorageActiveMinutesID)
-// 		if err != nil {
-// 			queryResults.Error = err
-// 		}
-// 		minutesByNode := map[string]float64{}
-// 		for _, result := range minutesResults {
-// 			node := result.MetricLabels[source.NodeLabel]
-// 			if node == "" || len(result.Values) == 0 {
-// 				continue
-// 			}
-// 			nodeStart := result.Values[0].Timestamp
-// 			nodeEnd := result.Values[len(result.Values)-1].Timestamp
-// 			if nodeStart == nil || nodeEnd == nil {
-// 				continue
-// 			}
-// 			minutesByNode[node] = nodeEnd.Sub(*nodeStart).Minutes()
-
-// 		}
-// 		bytesResults, err := collector.Query(metric.LocalStorageBytesID)
-// 		if err != nil {
-// 			queryResults.Error = err
-// 		}
-// 		for _, result := range bytesResults {
-// 			instance := result.MetricLabels[source.InstanceLabel]
-// 			if instance == "" || len(result.Values) == 0 {
-// 				continue
-// 			}
-// 			mintues, ok := minutesByNode[instance]
-// 			if !ok {
-// 				continue
-// 			}
-// 			queryResult := result.ToQueryResult()
-// 			bytes := queryResult.Values[0].Value
-// 			GiBs := bytes / GiB
-// 			hours := mintues / 60
-// 			queryResult.Values[0].Value = GiBs * hours * LocalStorageCostPerGiBHr
-// 			queryResults.Results = append(queryResults.Results, queryResult)
-// 		}
-// 	}
-// 	ch := make(source.QueryResultsChan, 1)
-// 	ch <- queryResults
-// 	return source.NewFuture(source.DecodeLocalStorageCostResult, ch)
-// }
-
-// func (c *collectorMetricsQuerier) QueryLocalStorageUsedCost(start, end time.Time) *source.Future[source.LocalStorageUsedCostResult] {
-// 	queryResults := source.NewQueryResults("LocalStorageUsedCost")
-// 	collector := c.collectorProvider.GetStore(start, end)
-// 	if collector != nil {
-// 		minutesResults, err := collector.Query(metric.LocalStorageUsedActiveMinutesID)
-// 		if err != nil {
-// 			queryResults.Error = err
-// 		}
-// 		minutesByNode := map[string]float64{}
-// 		for _, result := range minutesResults {
-// 			node := result.MetricLabels[source.InstanceLabel]
-// 			if node == "" || len(result.Values) == 0 {
-// 				continue
-// 			}
-// 			nodeStart := result.Values[0].Timestamp
-// 			nodeEnd := result.Values[len(result.Values)-1].Timestamp
-// 			if nodeStart == nil || nodeEnd == nil {
-// 				continue
-// 			}
-// 			minutesByNode[node] = nodeEnd.Sub(*nodeStart).Minutes()
-
-// 		}
-// 		bytesResults, err := collector.Query(metric.LocalStorageUsedAverageID)
-// 		if err != nil {
-// 			queryResults.Error = err
-// 		}
-// 		for _, result := range bytesResults {
-// 			instance := result.MetricLabels[source.InstanceLabel]
-// 			if instance == "" || len(result.Values) == 0 {
-// 				continue
-// 			}
-// 			mintues, ok := minutesByNode[instance]
-// 			if !ok {
-// 				continue
-// 			}
-// 			queryResult := result.ToQueryResult()
-// 			bytes := queryResult.Values[0].Value
-// 			GiBs := bytes / GiB
-// 			hours := mintues / 60
-// 			queryResult.Values[0].Value = GiBs * hours * LocalStorageCostPerGiBHr
-// 			queryResults.Results = append(queryResults.Results, queryResult)
-// 		}
-// 	}
-// 	ch := make(source.QueryResultsChan, 1)
-// 	ch <- queryResults
-// 	return source.NewFuture(source.DecodeLocalStorageUsedCostResult, ch)
-// }
-
 func (c *collectorMetricsQuerier) QueryLocalStorageUsedAvg(start, end time.Time) *source.Future[source.LocalStorageUsedAvgResult] {
 	return queryCollector(c, start, end, metric.LocalStorageUsedAverageID, source.DecodeLocalStorageUsedAvgResult)
 }

modules/collector-source/pkg/collector/metricsquerier_test.go

@@ -182,74 +182,6 @@ func GetMockCollectorProvider() StoreProvider {
 	}
 }
 
-// func TestCollectorMetricsQuerier_QueryLocalStorageCost(t *testing.T) {
-// 	start1, _ := time.Parse(time.RFC3339, Start1Str)
-// 	end1, _ := time.Parse(time.RFC3339, End1Str)
-
-// 	c := collectorMetricsQuerier{
-// 		collectorProvider: GetMockCollectorProvider(),
-// 	}
-// 	resCh := c.QueryLocalStorageCost(start1, end1)
-// 	res, err := resCh.Await()
-// 	if err != nil {
-// 		t.Errorf("unexpected error: %v", err.Error())
-// 	}
-// 	expected := []*source.LocalStorageCostResult{
-// 		{
-// 			Cluster:  "",
-// 			Instance: "node1",
-// 			Device:   "local",
-// 			Data: []*util.Vector{
-// 				{
-// 					Value: LocalStorageCostPerGiBHr * 2,
-// 				},
-// 			},
-// 		},
-// 	}
-// 	if len(res) != len(expected) {
-// 		t.Errorf("length of result was not as expected: got = %d, want %d", len(res), len(expected))
-// 	}
-// 	for i, got := range res {
-// 		if !reflect.DeepEqual(got, expected[i]) {
-// 			t.Errorf("result at index %d did not match: got = %v, want %v", i, got, expected[i])
-// 		}
-// 	}
-// }
-
-// func TestCollectorMetricsQuerier_QueryLocalStorageUsedCost(t *testing.T) {
-// 	start1, _ := time.Parse(time.RFC3339, Start1Str)
-// 	end1, _ := time.Parse(time.RFC3339, End1Str)
-
-// 	c := collectorMetricsQuerier{
-// 		collectorProvider: GetMockCollectorProvider(),
-// 	}
-// 	resCh := c.QueryLocalStorageUsedCost(start1, end1)
-// 	res, err := resCh.Await()
-// 	if err != nil {
-// 		t.Errorf("unexpected error: %v", err.Error())
-// 	}
-// 	expected := []*source.LocalStorageUsedCostResult{
-// 		{
-// 			Cluster:  "",
-// 			Instance: "node1",
-// 			Device:   "local",
-// 			Data: []*util.Vector{
-// 				{
-// 					Value: LocalStorageCostPerGiBHr,
-// 				},
-// 			},
-// 		},
-// 	}
-// 	if len(res) != len(expected) {
-// 		t.Errorf("length of result was not as expected: got = %d, want %d", len(res), len(expected))
-// 	}
-// 	for i, got := range res {
-// 		if !reflect.DeepEqual(got, expected[i]) {
-// 			t.Errorf("result at index %d did not match: got = %v, want %v", i, got, expected[i])
-// 		}
-// 	}
-// }
-
 func TestCollectorMetricsQuerier_QueryNodeActiveMinutes(t *testing.T) {
 	start1, _ := time.Parse(time.RFC3339, Start1Str)
 	end1, _ := time.Parse(time.RFC3339, End1Str)

modules/prometheus-source/pkg/prom/metricsquerier.go

@@ -129,56 +129,6 @@ func (pds *PrometheusMetricsQuerier) QueryPVActiveMinutes(start, end time.Time)
 	return source.NewFuture(source.DecodePVActiveMinutesResult, ctx.QueryAtTime(queryPVActiveMins, end))
 }
 
-// func (pds *PrometheusMetricsQuerier) QueryLocalStorageCost(start, end time.Time) *source.Future[source.LocalStorageCostResult] {
-// 	const queryName = "QueryLocalStorageCost"
-// 	const localStorageCostQuery = `sum_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, uid, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
-
-// 	cfg := pds.promConfig
-// 	minsPerResolution := cfg.DataResolutionMinutes
-
-// 	durStr := pds.durationStringFor(start, end, minsPerResolution, false)
-// 	if durStr == "" {
-// 		panic(fmt.Sprintf("failed to parse duration string passed to %s", queryName))
-// 	}
-
-// 	// hourlyToCumulative is a scaling factor that, when multiplied by an
-// 	// hourly value, converts it to a cumulative value; i.e. [$/hr] *
-// 	// [min/res]*[hr/min] = [$/res]
-// 	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-// 	costPerGBHr := 0.04 / 730.0
-
-// 	queryLocalStorageCost := fmt.Sprintf(localStorageCostQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-// 	log.Debugf(PrometheusMetricsQueryLogFormat, queryName, end.Unix(), queryLocalStorageCost)
-
-// 	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-// 	return source.NewFuture(source.DecodeLocalStorageCostResult, ctx.QueryAtTime(queryLocalStorageCost, end))
-// }
-
-// func (pds *PrometheusMetricsQuerier) QueryLocalStorageUsedCost(start, end time.Time) *source.Future[source.LocalStorageUsedCostResult] {
-// 	const queryName = "QueryLocalStorageUsedCost"
-// 	const localStorageUsedCostQuery = `sum_over_time(sum(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, uid, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
-
-// 	cfg := pds.promConfig
-// 	minsPerResolution := cfg.DataResolutionMinutes
-
-// 	durStr := pds.durationStringFor(start, end, minsPerResolution, false)
-// 	if durStr == "" {
-// 		panic(fmt.Sprintf("failed to parse duration string passed to %s", queryName))
-// 	}
-
-// 	// hourlyToCumulative is a scaling factor that, when multiplied by an
-// 	// hourly value, converts it to a cumulative value; i.e. [$/hr] *
-// 	// [min/res]*[hr/min] = [$/res]
-// 	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-// 	costPerGBHr := 0.04 / 730.0
-
-// 	queryLocalStorageUsedCost := fmt.Sprintf(localStorageUsedCostQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-// 	log.Debugf(PrometheusMetricsQueryLogFormat, queryName, end.Unix(), queryLocalStorageUsedCost)
-
-// 	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-// 	return source.NewFuture(source.DecodeLocalStorageUsedCostResult, ctx.QueryAtTime(queryLocalStorageUsedCost, end))
-// }
-
 func (pds *PrometheusMetricsQuerier) QueryLocalStorageUsedAvg(start, end time.Time) *source.Future[source.LocalStorageUsedAvgResult] {
 	const queryName = "QueryLocalStorageUsedAvg"
 	const localStorageUsedAvgQuery = `avg(sum(avg_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, uid, %s, job)) by (instance, device, uid, %s)`

modules/prometheus-source/pkg/prom/metricsquerier_test.go

@@ -96,12 +96,10 @@ func TestQueryLogs(t *testing.T) {
 	queryStart := queryEnd.Add(-24 * time.Hour)
 
 	tests := map[string]func(time.Time, time.Time){
-		"QueryPVActiveMinutes":           func(s, e time.Time) { querier.QueryPVActiveMinutes(s, e) },
-		"QueryPVUsedAverage":             func(s, e time.Time) { querier.QueryPVUsedAverage(s, e) },
-		"QueryPVUsedMax":                 func(s, e time.Time) { querier.QueryPVUsedMax(s, e) },
-		"QueryLocalStorageActiveMinutes": func(s, e time.Time) { querier.QueryLocalStorageActiveMinutes(s, e) },
-		// "QueryLocalStorageCost":                         func(s, e time.Time) { querier.QueryLocalStorageCost(s, e) },
-		// "QueryLocalStorageUsedCost":                     func(s, e time.Time) { querier.QueryLocalStorageUsedCost(s, e) },
+		"QueryPVActiveMinutes":                          func(s, e time.Time) { querier.QueryPVActiveMinutes(s, e) },
+		"QueryPVUsedAverage":                            func(s, e time.Time) { querier.QueryPVUsedAverage(s, e) },
+		"QueryPVUsedMax":                                func(s, e time.Time) { querier.QueryPVUsedMax(s, e) },
+		"QueryLocalStorageActiveMinutes":                func(s, e time.Time) { querier.QueryLocalStorageActiveMinutes(s, e) },
 		"QueryLocalStorageUsedAvg":                      func(s, e time.Time) { querier.QueryLocalStorageUsedAvg(s, e) },
 		"QueryLocalStorageUsedMax":                      func(s, e time.Time) { querier.QueryLocalStorageUsedMax(s, e) },
 		"QueryLocalStorageBytes":                        func(s, e time.Time) { querier.QueryLocalStorageBytes(s, e) },

pkg/costmodel/cluster.go

@@ -126,23 +126,17 @@ func ClusterDisks(dataSource source.OpenCostDataSource, cp models.Provider, star
 	// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/RootDeviceStorage.html
 	// https://learn.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview#temporary-disk
 	// https://cloud.google.com/compute/docs/disks/local-ssd
-	// resLocalStorageCost := []*source.LocalStorageCostResult{}
-	// resLocalStorageUsedCost := []*source.LocalStorageUsedCostResult{}
 	resLocalStorageUsedAvg := []*source.LocalStorageUsedAvgResult{}
 	resLocalStorageUsedMax := []*source.LocalStorageUsedMaxResult{}
 	resLocalStorageBytes := []*source.LocalStorageBytesResult{}
 	resLocalActiveMins := []*source.LocalStorageActiveMinutesResult{}
 
 	if env.IsAssetIncludeLocalDiskCost() {
-		// resChLocalStorageCost := source.WithGroup(grp, mq.QueryLocalStorageCost(start, end))
-		// resChLocalStorageUsedCost := source.WithGroup(grp, mq.QueryLocalStorageUsedCost(start, end))
 		resChLocalStoreageUsedAvg := source.WithGroup(grp, mq.QueryLocalStorageUsedAvg(start, end))
 		resChLocalStoreageUsedMax := source.WithGroup(grp, mq.QueryLocalStorageUsedMax(start, end))
 		resChLocalStorageBytes := source.WithGroup(grp, mq.QueryLocalStorageBytes(start, end))
 		resChLocalActiveMins := source.WithGroup(grp, mq.QueryLocalStorageActiveMinutes(start, end))
 
-		// resLocalStorageCost, _ = resChLocalStorageCost.Await()
-		// resLocalStorageUsedCost, _ = resChLocalStorageUsedCost.Await()
 		resLocalStorageUsedAvg, _ = resChLocalStoreageUsedAvg.Await()
 		resLocalStorageUsedMax, _ = resChLocalStoreageUsedMax.Await()
 		resLocalStorageBytes, _ = resChLocalStorageBytes.Await()
@@ -208,61 +202,6 @@ func ClusterDisks(dataSource source.OpenCostDataSource, cp models.Provider, star
 		}
 	}
 
-	// for _, result := range resLocalStorageCost {
-	// 	cluster := result.Cluster
-	// 	if cluster == "" {
-	// 		cluster = coreenv.GetClusterID()
-	// 	}
-
-	// 	name := result.Instance
-	// 	if name == "" {
-	// 		log.Warnf("ClusterDisks: local storage data missing instance")
-	// 		continue
-	// 	}
-
-	// 	device := result.Device
-	// 	if device == "" {
-	// 		log.Warnf("ClusterDisks: local storage data missing device")
-	// 		continue
-	// 	}
-
-	// 	cost := result.Data[0].Value
-	// 	key := DiskIdentifier{cluster, name}
-	// 	ls, ok := localStorageDisks[key]
-	// 	if !ok || ls.device != device {
-	// 		continue
-	// 	}
-	// 	ls.disk.Cost = cost
-
-	// }
-
-	// for _, result := range resLocalStorageUsedCost {
-	// 	cluster := result.Cluster
-	// 	if cluster == "" {
-	// 		cluster = coreenv.GetClusterID()
-	// 	}
-
-	// 	name := result.Instance
-	// 	if name == "" {
-	// 		log.Warnf("ClusterDisks: local storage data missing instance")
-	// 		continue
-	// 	}
-
-	// 	device := result.Device
-	// 	if device == "" {
-	// 		log.Warnf("ClusterDisks: local storage data missing device")
-	// 		continue
-	// 	}
-
-	// 	cost := result.Data[0].Value
-	// 	key := DiskIdentifier{cluster, name}
-	// 	ls, ok := localStorageDisks[key]
-	// 	if !ok || ls.device != device {
-	// 		continue
-	// 	}
-	// 	ls.disk.Breakdown.System = cost / ls.disk.Cost
-	// }
-
 	for _, result := range resLocalStorageUsedAvg {
 		cluster := result.Cluster
 		if cluster == "" {