Merge pull request #104 from kubecost/AjayTripathy-support-projections

support projections

Dockerfile

@@ -27,7 +27,7 @@ RUN set -e ;\
         -ldflags "-X main.gitCommit=${GIT_COMMIT}${GIT_DIRTY}" \
         -o /go/bin/app
 
-FROM alpine:3.4
+FROM alpine:3.9.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build-env /go/bin/app /go/bin/app
 ADD ./cloud/default.json /models/default.json

cloud/aws.json

@@ -4,6 +4,7 @@
     "CPU": "0.031611",
     "spotCPU": "0.006655",
     "RAM": "0.004237",
+    "GPU": "0.95",
     "spotRAM": "0.000892",
     "spotLabel": "kops.k8s.io/instancegroup",
     "spotLabelValue": "spotinstance-nodes",

cloud/awsprovider.go

@@ -46,6 +46,8 @@ type AWS struct {
 	ValidPricingKeys        map[string]bool
 	Clientset               *kubernetes.Clientset
 	BaseCPUPrice            string
+	BaseRAMPrice            string
+	BaseGPUPrice            string
 	BaseSpotCPUPrice        string
 	BaseSpotRAMPrice        string
 	SpotLabelName           string
@@ -356,6 +358,8 @@ func (aws *AWS) DownloadPricingData() error {
 		klog.V(1).Infof("Error downloading default pricing data: %s", err.Error())
 	}
 	aws.BaseCPUPrice = c.CPU
+	aws.BaseRAMPrice = c.RAM
+	aws.BaseGPUPrice = c.GPU
 	aws.BaseSpotCPUPrice = c.SpotCPU
 	aws.BaseSpotRAMPrice = c.SpotRAM
 	aws.SpotLabelName = c.SpotLabel
@@ -528,6 +532,8 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 				GPU:          terms.GPU,
 				Storage:      terms.Storage,
 				BaseCPUPrice: aws.BaseCPUPrice,
+				BaseRAMPrice: aws.BaseRAMPrice,
+				BaseGPUPrice: aws.BaseGPUPrice,
 				UsageType:    usageType,
 			}, nil
 		}
@@ -539,6 +545,8 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 			RAMCost:      aws.BaseSpotRAMPrice,
 			Storage:      terms.Storage,
 			BaseCPUPrice: aws.BaseCPUPrice,
+			BaseRAMPrice: aws.BaseRAMPrice,
+			BaseGPUPrice: aws.BaseGPUPrice,
 			UsageType:    usageType,
 		}, nil
 	}
@@ -554,6 +562,8 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 		GPU:          terms.GPU,
 		Storage:      terms.Storage,
 		BaseCPUPrice: aws.BaseCPUPrice,
+		BaseRAMPrice: aws.BaseRAMPrice,
+		BaseGPUPrice: aws.BaseGPUPrice,
 		UsageType:    usageType,
 	}, nil
 }
@@ -589,6 +599,8 @@ func (aws *AWS) NodePricing(k Key) (*Node, error) {
 		return &Node{
 			Cost:             aws.BaseCPUPrice,
 			BaseCPUPrice:     aws.BaseCPUPrice,
+			BaseRAMPrice:     aws.BaseRAMPrice,
+			BaseGPUPrice:     aws.BaseGPUPrice,
 			UsageType:        usageType,
 			UsesBaseCPUPrice: true,
 		}, nil

cloud/provider.go

@@ -34,6 +34,8 @@ type Node struct {
 	StorageCost      string `json:"storageHourlyCost"`
 	UsesBaseCPUPrice bool   `json:"usesDefaultPrice"`
 	BaseCPUPrice     string `json:"baseCPUPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
+	BaseRAMPrice     string `json:"baseRAMPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
+	BaseGPUPrice     string `json:"baseGPUPrice"`
 	UsageType        string `json:"usageType"`
 	GPU              string `json:"gpu"` // GPU represents the number of GPU on the instance
 	GPUName          string `json:"gpuName"`
@@ -164,6 +166,7 @@ type CustomPricing struct {
 	AzureClientID       string `json:"azureClientID"`
 	AzureClientSecret   string `json:"azureClientSecret"`
 	AzureTenantID       string `json:"azureTenantID"`
+	CurrencyCode        string `json:"currencyCode"`
 }
 
 func SetCustomPricingField(obj *CustomPricing, name string, value string) error {

costmodel/cluster.go

@@ -0,0 +1,158 @@
+package costmodel
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	prometheusClient "github.com/prometheus/client_golang/api"
+	"k8s.io/klog"
+)
+
+const (
+	queryClusterCores = `sum(
+		avg(kube_node_status_capacity_cpu_cores %s) by (node) * avg(node_cpu_hourly_cost %s) by (node) * 730 +
+		avg(node_gpu_hourly_cost %s) by (node) * 730
+	  )`
+
+	queryClusterRAM = `sum(
+		avg(kube_node_status_capacity_memory_bytes %s) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost %s) by (node) * 730
+	  )`
+
+	queryStorage = `sum(
+		avg(avg_over_time(pv_hourly_cost[%s] %s)) by (persistentvolume) * 730 
+		* avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s] %s)) by (persistentvolume) / 1024 / 1024 / 1024
+	  ) +
+	  sum(avg(container_fs_limit_bytes{device!="tmpfs", id="/"} %s) by (instance) / 1024 / 1024 / 1024) * 0.04`
+
+	queryTotal = `sum(
+		avg(kube_node_status_capacity_cpu_cores %s) by (node) * avg(node_cpu_hourly_cost %s) by (node) * 730 +
+		avg(node_gpu_hourly_cost %s) by (node) * 730 
+	  ) +
+
+	  sum(
+		avg(kube_node_status_capacity_memory_bytes %s) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost %s) by (node) * 730
+	  ) +
+
+	  sum(
+		avg(avg_over_time(pv_hourly_cost[%s] %s)) by (persistentvolume) * 730 
+		* avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s] %s)) by (persistentvolume) / 1024 / 1024 / 1024
+	  ) +
+	  sum(avg(container_fs_limit_bytes{device!="tmpfs", id="/"} %s) by (instance) / 1024 / 1024 / 1024) * 0.04`
+)
+
+type Totals struct {
+	TotalCost   [][]string `json:"totalcost"`
+	CPUCost     [][]string `json:"cpucost"`
+	MemCost     [][]string `json:"memcost"`
+	StorageCost [][]string `json:"storageCost"`
+}
+
+func resultToTotal(qr interface{}) ([][]string, error) {
+	data, ok := qr.(map[string]interface{})["data"]
+	if !ok {
+		return nil, fmt.Errorf("Improperly formatted response from prometheus, response %+v has no data field", data)
+	}
+	r, ok := data.(map[string]interface{})["result"]
+	if !ok {
+		return nil, fmt.Errorf("Improperly formatted data from prometheus, data has no result field")
+	}
+	results, ok := r.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("Improperly formatted results from prometheus, result field is not a slice")
+	}
+	res, ok := results[0].(map[string]interface{})["values"]
+	totals := [][]string{}
+	for _, val := range res.([]interface{}) {
+		if !ok {
+			return nil, fmt.Errorf("Improperly formatted results from prometheus, value is not a field in the vector")
+		}
+		dataPoint, ok := val.([]interface{})
+		//log.Printf("%+v", dataPoint)
+		if !ok || len(dataPoint) != 2 {
+			return nil, fmt.Errorf("Improperly formatted datapoint from Prometheus")
+		}
+		d0 := fmt.Sprintf("%f", dataPoint[0].(float64))
+		toAppend := []string{
+			d0,
+			dataPoint[1].(string),
+		}
+		totals = append(totals, toAppend)
+	}
+	return totals, nil
+}
+
+// ClusterCostsOverTime gives the full cluster costs over time
+func ClusterCostsOverTime(cli prometheusClient.Client, startString, endString, windowString, offset string) (*Totals, error) {
+
+	layout := "2006-01-02T15:04:05.000Z"
+
+	start, err := time.Parse(layout, startString)
+	if err != nil {
+		klog.V(1).Infof("Error parsing time " + startString + ". Error: " + err.Error())
+		return nil, err
+	}
+	end, err := time.Parse(layout, endString)
+	if err != nil {
+		klog.V(1).Infof("Error parsing time " + endString + ". Error: " + err.Error())
+		return nil, err
+	}
+	window, err := time.ParseDuration(windowString)
+	if err != nil {
+		klog.V(1).Infof("Error parsing time " + windowString + ". Error: " + err.Error())
+		return nil, err
+	}
+
+	qCores := fmt.Sprintf(queryClusterCores, offset, offset, offset)
+	qRAM := fmt.Sprintf(queryClusterRAM, offset, offset)
+	qStorage := fmt.Sprintf(queryStorage, windowString, offset, windowString, offset, offset)
+	qTotal := fmt.Sprintf(queryTotal, offset, offset, offset, offset, offset, windowString, offset, windowString, offset, offset)
+	log.Printf("%s", qTotal)
+
+	resultClusterCores, err := queryRange(cli, qCores, start, end, window)
+	if err != nil {
+		return nil, err
+	}
+	resultClusterRAM, err := queryRange(cli, qRAM, start, end, window)
+	if err != nil {
+		return nil, err
+	}
+
+	resultStorage, err := queryRange(cli, qStorage, start, end, window)
+	if err != nil {
+		return nil, err
+	}
+
+	resultTotal, err := queryRange(cli, qTotal, start, end, window)
+	if err != nil {
+		return nil, err
+	}
+
+	coreTotal, err := resultToTotal(resultClusterCores)
+	if err != nil {
+		return nil, err
+	}
+
+	ramTotal, err := resultToTotal(resultClusterRAM)
+	if err != nil {
+		return nil, err
+	}
+
+	storageTotal, err := resultToTotal(resultStorage)
+	if err != nil {
+		return nil, err
+	}
+
+	clusterTotal, err := resultToTotal(resultTotal)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Totals{
+		TotalCost:   clusterTotal,
+		CPUCost:     coreTotal,
+		MemCost:     ramTotal,
+		StorageCost: storageTotal,
+	}, nil
+
+}

costmodel/costmodel.go

@@ -80,9 +80,9 @@ const (
 	) by (namespace,container_name,pod_name,node)`
 	queryRAMUsageStr = `sort_desc(
 		avg(
-			label_replace(count_over_time(container_memory_usage_bytes{container_name!="",container_name!="POD", instance!=""}[%s] %s), "node", "$1", "instance","(.+)") 
+			label_replace(count_over_time(container_memory_working_set_bytes{container_name!="",container_name!="POD", instance!=""}[%s] %s), "node", "$1", "instance","(.+)") 
 			* 
-			label_replace(avg_over_time(container_memory_usage_bytes{container_name!="",container_name!="POD", instance!=""}[%s] %s), "node", "$1", "instance","(.+)") 
+			label_replace(avg_over_time(container_memory_working_set_bytes{container_name!="",container_name!="POD", instance!=""}[%s] %s), "node", "$1", "instance","(.+)") 
 		) by (namespace,container_name,pod_name,node)
 	)`
 	queryCPURequestsStr = `avg(
@@ -247,7 +247,13 @@ func ComputeCostData(cli prometheusClient.Client, clientset kubernetes.Interface
 
 	pvClaimMapping, err := getPVInfoVector(resultPVRequests)
 	if err != nil {
-		return nil, err
+		klog.Infof("Unable to get PV Data: %s", err.Error())
+	}
+	if pvClaimMapping != nil {
+		err = addPVData(clientset, pvClaimMapping, cloud)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	err = addPVData(clientset, pvClaimMapping, cloud)
@@ -659,6 +665,10 @@ func GetPVCost(pv *costAnalyzerCloud.PV, kpv *v1.PersistentVolume, cloud costAna
 }
 
 func getNodeCost(clientset kubernetes.Interface, cloud costAnalyzerCloud.Provider) (map[string]*costAnalyzerCloud.Node, error) {
+	cfg, err := cloud.GetConfig()
+	if err != nil {
+		return nil, err
+	}
 	nodeList, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
 	if err != nil {
 		return nil, err
@@ -689,34 +699,68 @@ func getNodeCost(clientset kubernetes.Interface, cloud costAnalyzerCloud.Provide
 
 		if cnode.GPU != "" && cnode.GPUCost == "" { // We couldn't find a gpu cost, so fix cpu and ram, then accordingly
 			klog.V(3).Infof("GPU without cost found for %s, calculating...", cloud.GetKey(nodeLabels).Features())
-			basePrice, err := strconv.ParseFloat(cnode.BaseCPUPrice, 64)
+			defaultCPU, err := strconv.ParseFloat(cfg.CPU, 64)
+			if err != nil {
+				klog.V(3).Infof("Could not parse default cpu price")
+				return nil, err
+			}
+			defaultRAM, err := strconv.ParseFloat(cfg.RAM, 64)
 			if err != nil {
-				klog.V(3).Infof("Error parsing node base price. Error: " + err.Error())
+				klog.V(3).Infof("Could not parse default ram price")
 				return nil, err
 			}
-			nodePrice, err := strconv.ParseFloat(cnode.Cost, 64)
+			defaultGPU, err := strconv.ParseFloat(cfg.RAM, 64)
 			if err != nil {
-				klog.V(3).Infof("Error parsing node cost. Error: " + err.Error())
+				klog.V(3).Infof("Could not parse default gpu price")
 				return nil, err
 			}
-			totalCPUPrice := basePrice * cpu
-			totalRAMPrice := 0.1 * totalCPUPrice
-			ramPrice := totalRAMPrice / (ram / 1024 / 1024 / 1024)
-			gpuPrice := nodePrice - totalCPUPrice - totalRAMPrice
-			cnode.VCPUCost = fmt.Sprintf("%f", basePrice)
+			cpuToRAMRatio := defaultCPU / defaultRAM
+			gpuToRAMRatio := defaultGPU / defaultRAM
+
+			ramGB := ram / 1024 / 1024 / 1024
+			ramMultiple := gpuToRAMRatio + cpu*cpuToRAMRatio + ramGB
+			var nodePrice float64
+			if cnode.Cost != "" {
+				nodePrice, err = strconv.ParseFloat(cnode.Cost, 64)
+				if err != nil {
+					klog.V(3).Infof("Could not parse total node price")
+					return nil, err
+				}
+			} else {
+				nodePrice, err = strconv.ParseFloat(cnode.VCPUCost, 64) // all the price was allocated the the CPU
+				if err != nil {
+					klog.V(3).Infof("Could not parse node vcpu price")
+					return nil, err
+				}
+			}
+
+			ramPrice := (nodePrice / ramMultiple)
+			cpuPrice := ramPrice * cpuToRAMRatio
+			gpuPrice := ramPrice * gpuToRAMRatio
+			cnode.VCPUCost = fmt.Sprintf("%f", cpuPrice)
 			cnode.RAMCost = fmt.Sprintf("%f", ramPrice)
 			cnode.RAMBytes = fmt.Sprintf("%f", ram)
 			cnode.GPUCost = fmt.Sprintf("%f", gpuPrice)
-			klog.V(2).Infof("Computed \"%s\" GPU Cost := %v", name, cnode.GPUCost)
+
 		} else {
 			if cnode.RAMCost == "" { // We couldn't find a ramcost, so fix cpu and allocate ram accordingly
 				klog.V(3).Infof("No RAM cost found for %s, calculating...", cloud.GetKey(nodeLabels).Features())
-				basePrice, err := strconv.ParseFloat(cnode.BaseCPUPrice, 64)
+				defaultCPU, err := strconv.ParseFloat(cfg.CPU, 64)
+				if err != nil {
+					klog.V(3).Infof("Could not parse default cpu price")
+					return nil, err
+				}
+				defaultRAM, err := strconv.ParseFloat(cfg.RAM, 64)
 				if err != nil {
-					klog.V(3).Infof("Could not find base total node price")
+					klog.V(3).Infof("Could not parse default ram price")
 					return nil, err
 				}
-				totalCPUPrice := basePrice * cpu
+				cpuToRAMRatio := defaultCPU / defaultRAM
+
+				ramGB := ram / 1024 / 1024 / 1024
+
+				ramMultiple := cpu*cpuToRAMRatio + ramGB
+
 				var nodePrice float64
 				if cnode.Cost != "" {
 					nodePrice, err = strconv.ParseFloat(cnode.Cost, 64)
@@ -731,11 +775,9 @@ func getNodeCost(clientset kubernetes.Interface, cloud costAnalyzerCloud.Provide
 						return nil, err
 					}
 				}
-				if totalCPUPrice >= nodePrice {
-					totalCPUPrice = 0.9 * nodePrice // just allocate RAM costs to 10% of the node price here to avoid 0 or negative in the numerator
-				}
-				ramPrice := (nodePrice - totalCPUPrice) / (ram / 1024 / 1024 / 1024)
-				cpuPrice := totalCPUPrice / cpu
+
+				ramPrice := (nodePrice / ramMultiple)
+				cpuPrice := ramPrice * cpuToRAMRatio
 
 				cnode.VCPUCost = fmt.Sprintf("%f", cpuPrice)
 				cnode.RAMCost = fmt.Sprintf("%f", ramPrice)
@@ -894,12 +936,14 @@ func ComputeCostDataRange(cli prometheusClient.Client, clientset kubernetes.Inte
 	}
 	pvClaimMapping, err := getPVInfoVectors(resultPVRequests)
 	if err != nil {
-		return nil, err
+		// Just log for compatibility with KSM less than 1.6
+		klog.Infof("Unable to get PV Data: %s", err.Error())
 	}
-
-	err = addPVData(clientset, pvClaimMapping, cloud)
-	if err != nil {
-		return nil, err
+	if pvClaimMapping != nil {
+		err = addPVData(clientset, pvClaimMapping, cloud)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	containerNameCost := make(map[string]*CostData)
@@ -1277,7 +1321,7 @@ func getPVInfoVectors(qr interface{}) (map[string]*PersistentVolumeClaimData, er
 			strVal := dataPoint[1].(string)
 			v, _ := strconv.ParseFloat(strVal, 64)
 			vectors = append(vectors, &Vector{
-				Timestamp: dataPoint[0].(float64),
+				Timestamp: math.Round(dataPoint[0].(float64)/10) * 10,
 				Value:     v,
 			})
 		}
@@ -1620,7 +1664,7 @@ func getContainerMetricVectors(qr interface{}, normalize bool, normalizationValu
 				v = v / normalizationValue
 			}
 			vectors = append(vectors, &Vector{
-				Timestamp: dataPoint[0].(float64),
+				Timestamp: math.Round(dataPoint[0].(float64)/10) * 10,
 				Value:     v,
 			})
 		}

main.go

@@ -6,7 +6,9 @@ import (
 	"flag"
 	"net/http"
 	"os"
+	"reflect"
 	"strconv"
+	"strings"
 	"time"
 
 	"k8s.io/klog"
@@ -48,6 +50,7 @@ type Accesses struct {
 	NodeTotalPriceRecorder        *prometheus.GaugeVec
 	RAMAllocationRecorder         *prometheus.GaugeVec
 	CPUAllocationRecorder         *prometheus.GaugeVec
+	GPUAllocationRecorder         *prometheus.GaugeVec
 }
 
 type DataEnvelope struct {
@@ -60,6 +63,7 @@ type DataEnvelope struct {
 func wrapData(data interface{}, err error) []byte {
 	var resp []byte
 	if err != nil {
+		klog.V(1).Infof("Error returned to client: %s", err.Error())
 		resp, _ = json.Marshal(&DataEnvelope{
 			Code:    500,
 			Status:  "error",
@@ -87,18 +91,69 @@ func (a *Accesses) RefreshPricingData(w http.ResponseWriter, r *http.Request, ps
 	w.Write(wrapData(nil, err))
 }
 
+func filterFields(fields string, data map[string]*costModel.CostData) map[string]costModel.CostData {
+	fs := strings.Split(fields, ",")
+	fmap := make(map[string]bool)
+	for _, f := range fs {
+		fieldNameLower := strings.ToLower(f) // convert to go struct name by uppercasing first letter
+		klog.V(1).Infof("to delete: %s", fieldNameLower)
+		fmap[fieldNameLower] = true
+	}
+	filteredData := make(map[string]costModel.CostData)
+	for cname, costdata := range data {
+		s := reflect.TypeOf(*costdata)
+		val := reflect.ValueOf(*costdata)
+		costdata2 := costModel.CostData{}
+		cd2 := reflect.New(reflect.Indirect(reflect.ValueOf(costdata2)).Type()).Elem()
+		n := s.NumField()
+		for i := 0; i < n; i++ {
+			field := s.Field(i)
+			value := val.Field(i)
+			value2 := cd2.Field(i)
+			if _, ok := fmap[strings.ToLower(field.Name)]; !ok {
+				value2.Set(reflect.Value(value))
+			}
+		}
+		filteredData[cname] = cd2.Interface().(costModel.CostData)
+	}
+	return filteredData
+}
+
 func (a *Accesses) CostDataModel(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	window := r.URL.Query().Get("timeWindow")
 	offset := r.URL.Query().Get("offset")
+	fields := r.URL.Query().Get("filterFields")
 
 	if offset != "" {
 		offset = "offset " + offset
 	}
 
 	data, err := costModel.ComputeCostData(a.PrometheusClient, a.KubeClientSet, a.Cloud, window, offset)
+	if fields != "" {
+		filteredData := filterFields(fields, data)
+		w.Write(wrapData(filteredData, err))
+	} else {
+		w.Write(wrapData(data, err))
+	}
+}
+
+func (a *Accesses) ClusterCostsOverTime(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	start := r.URL.Query().Get("start")
+	end := r.URL.Query().Get("end")
+	window := r.URL.Query().Get("timeWindow")
+	offset := r.URL.Query().Get("offset")
+
+	if offset != "" {
+		offset = "offset " + offset
+	}
+
+	data, err := costModel.ClusterCostsOverTime(a.PrometheusClient, start, end, window, offset)
 	w.Write(wrapData(data, err))
 }
 
@@ -109,9 +164,15 @@ func (a *Accesses) CostDataModelRange(w http.ResponseWriter, r *http.Request, ps
 	start := r.URL.Query().Get("start")
 	end := r.URL.Query().Get("end")
 	window := r.URL.Query().Get("window")
+	fields := r.URL.Query().Get("filterFields")
 
 	data, err := costModel.ComputeCostDataRange(a.PrometheusClient, a.KubeClientSet, a.Cloud, start, end, window)
-	w.Write(wrapData(data, err))
+	if fields != "" {
+		filteredData := filterFields(fields, data)
+		w.Write(wrapData(filteredData, err))
+	} else {
+		w.Write(wrapData(data, err))
+	}
 }
 
 func (a *Accesses) OutofClusterCosts(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
@@ -203,7 +264,7 @@ func (a *Accesses) recordPrices() {
 	go func() {
 		for {
 			klog.V(3).Info("Recording prices...")
-			data, err := costModel.ComputeCostData(a.PrometheusClient, a.KubeClientSet, a.Cloud, "1m", "")
+			data, err := costModel.ComputeCostData(a.PrometheusClient, a.KubeClientSet, a.Cloud, "2m", "")
 			if err != nil {
 				klog.V(1).Info("Error in price recording: " + err.Error())
 				// zero the for loop so the time.Sleep will still work
@@ -247,6 +308,10 @@ func (a *Accesses) recordPrices() {
 				if len(costs.CPUAllocation) > 0 {
 					a.CPUAllocationRecorder.WithLabelValues(namespace, podName, containerName, nodeName, nodeName).Set(costs.CPUAllocation[0].Value)
 				}
+				if len(costs.GPUReq) > 0 {
+					// allocation here is set to the request because shared GPU usage not yet supported.
+					a.GPUAllocationRecorder.WithLabelValues(namespace, podName, containerName, nodeName, nodeName).Set(costs.GPUReq[0].Value)
+				}
 
 				storageClasses, _ := a.KubeClientSet.StorageV1().StorageClasses().List(metav1.ListOptions{})
 
@@ -358,6 +423,11 @@ func main() {
 		Help: "container_cpu_allocation Percent of a single CPU used in a minute",
 	}, []string{"namespace", "pod", "container", "instance", "node"})
 
+	GPUAllocation := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "container_gpu_allocation",
+		Help: "container_gpu_allocation GPU used",
+	}, []string{"namespace", "pod", "container", "instance", "node"})
+
 	prometheus.MustRegister(cpuGv)
 	prometheus.MustRegister(ramGv)
 	prometheus.MustRegister(gpuGv)
@@ -376,6 +446,7 @@ func main() {
 		NodeTotalPriceRecorder:        totalGv,
 		RAMAllocationRecorder:         RAMAllocation,
 		CPUAllocationRecorder:         CPUAllocation,
+		GPUAllocationRecorder:         GPUAllocation,
 		PersistentVolumePriceRecorder: pvGv,
 	}
 
@@ -398,6 +469,7 @@ func main() {
 	router.POST("/updateAthenaInfoConfigs", a.UpdateAthenaInfoConfigs)
 	router.POST("/updateBigQueryInfoConfigs", a.UpdateBigQueryInfoConfigs)
 	router.POST("/updateConfigByKey", a.UpdateConfigByKey)
+	router.GET("/clusterCostsOverTime", a.ClusterCostsOverTime)
 
 	rootMux := http.NewServeMux()
 	rootMux.Handle("/", router)