Merge pull request #545 from kubecost/cw-407-load-balancer-costs

add external load balancer costs to prometheus metric recording

go.sum

@@ -559,6 +559,8 @@ k8s.io/apimachinery v0.0.0-20190913075812-e119e5e154b6 h1:tGU1C/vMoUV2ZakSH6wQq2
 k8s.io/apimachinery v0.0.0-20190913075812-e119e5e154b6/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
 k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag=
 k8s.io/apimachinery v0.19.0 h1:gjKnAda/HZp5k4xQYjL0K/Yb66IvNqjthCb03QlKpaQ=
+k8s.io/apimachinery v0.19.1 h1:cwsxZazM/LA9aUsBaL4bRS5ygoM6bYp8dFk22DSYQa4=
+k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc=
 k8s.io/client-go v0.0.0-20190404172613-2e1a3ed22ac5 h1:BwY2C//EoWktJi74O6R2REBonrhsfhRI0qfVwOjOPp8=
 k8s.io/client-go v0.0.0-20190404172613-2e1a3ed22ac5/go.mod h1:bIEHXHbykaOlj+pgLllzLJ2RPGdzkjtqdk0Il07KPEM=
 k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4Oi9QTp5cVpjTE+nqg6g=

pkg/cloud/awsprovider.go

@@ -848,6 +848,25 @@ func (aws *AWS) NetworkPricing() (*Network, error) {
 	}, nil
 }
 
+func (aws *AWS) LoadBalancerPricing() (*LoadBalancer, error) {
+	fffrc := 0.025
+	afrc := 0.010
+	lbidc := 0.008
+
+	numForwardingRules := 1.0
+	dataIngressGB := 0.0
+
+	var totalCost float64
+	if numForwardingRules < 5 {
+		totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB
+	} else {
+		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
+	}
+	return &LoadBalancer{
+		Cost: totalCost,
+	}, nil
+}
+
 // AllNodePricing returns all the billing data fetched.
 func (aws *AWS) AllNodePricing() (interface{}, error) {
 	aws.DownloadPricingDataLock.RLock()

pkg/cloud/azureprovider.go

@@ -650,6 +650,25 @@ func (az *Azure) NetworkPricing() (*Network, error) {
 	}, nil
 }
 
+func (azr *Azure) LoadBalancerPricing() (*LoadBalancer, error) {
+	fffrc := 0.025
+	afrc := 0.010
+	lbidc := 0.008
+
+	numForwardingRules := 1.0
+	dataIngressGB := 0.0
+
+	var totalCost float64
+	if numForwardingRules < 5 {
+		totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB
+	} else {
+		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
+	}
+	return &LoadBalancer{
+		Cost: totalCost,
+	}, nil
+}
+
 type azurePvKey struct {
 	Labels                 map[string]string
 	StorageClass           string

pkg/cloud/customprovider.go

@@ -240,6 +240,37 @@ func (cp *CustomProvider) NetworkPricing() (*Network, error) {
 	}, nil
 }
 
+func (cp *CustomProvider) LoadBalancerPricing() (*LoadBalancer, error) {
+	cpricing, err := cp.Config.GetCustomPricingData()
+	if err != nil {
+		return nil, err
+	}
+	fffrc, err := strconv.ParseFloat(cpricing.FirstFiveForwardingRulesCost, 64)
+	if err != nil {
+		return nil, err
+	}
+	afrc, err := strconv.ParseFloat(cpricing.AdditionalForwardingRuleCost, 64)
+	if err != nil {
+		return nil, err
+	}
+	lbidc, err := strconv.ParseFloat(cpricing.LBIngressDataCost, 64)
+	if err != nil {
+		return nil, err
+	}
+	var totalCost float64
+	numForwardingRules := 1.0 // hard-code at 1 for now
+	dataIngressGB := 0.0      // hard-code at 0 for now
+
+	if numForwardingRules < 5 {
+		totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB
+	} else {
+		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
+	}
+	return &LoadBalancer{
+		Cost: totalCost,
+	}, nil
+}
+
 func (*CustomProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
 	return &awsPVKey{
 		Labels:                 pv.Labels,

pkg/cloud/gcpprovider.go

@@ -1048,6 +1048,25 @@ func (gcp *GCP) NetworkPricing() (*Network, error) {
 	}, nil
 }
 
+func (gcp *GCP) LoadBalancerPricing() (*LoadBalancer, error) {
+	fffrc := 0.025
+	afrc := 0.010
+	lbidc := 0.008
+
+	numForwardingRules := 1.0
+	dataIngressGB := 0.0
+
+	var totalCost float64
+	if numForwardingRules < 5 {
+		totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB
+	} else {
+		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
+	}
+	return &LoadBalancer{
+		Cost: totalCost,
+	}, nil
+}
+
 const (
 	GCPReservedInstanceResourceTypeRAM string = "MEMORY"
 	GCPReservedInstanceResourceTypeCPU string = "VCPU"

pkg/cloud/provider.go

@@ -70,6 +70,18 @@ func (n *Node) IsSpot() bool {
 	}
 }
 
+// LoadBalancer is the interface by which the provider and cost model communicate LoadBalancer prices.
+// The provider will best-effort try to fill out this struct.
+type LoadBalancer struct {
+	IngressIPAddresses []string `json:"IngressIPAddresses"`
+	Cost               float64  `json:"hourlyCost"`
+}
+
+// TODO: used for dynamic cloud provider price fetching.
+// determine what identifies a load balancer in the json returned from the cloud provider pricing API call
+// type LBKey interface {
+// }
+
 // Network is the interface by which the provider and cost model communicate network egress prices.
 // The provider will best-effort try to fill out this struct.
 type Network struct {
@@ -111,51 +123,54 @@ type OutOfClusterAllocation struct {
 }
 
 type CustomPricing struct {
-	Provider              string            `json:"provider"`
-	Description           string            `json:"description"`
-	CPU                   string            `json:"CPU"`
-	SpotCPU               string            `json:"spotCPU"`
-	RAM                   string            `json:"RAM"`
-	SpotRAM               string            `json:"spotRAM"`
-	GPU                   string            `json:"GPU"`
-	SpotGPU               string            `json:"spotGPU"`
-	Storage               string            `json:"storage"`
-	ZoneNetworkEgress     string            `json:"zoneNetworkEgress"`
-	RegionNetworkEgress   string            `json:"regionNetworkEgress"`
-	InternetNetworkEgress string            `json:"internetNetworkEgress"`
-	SpotLabel             string            `json:"spotLabel,omitempty"`
-	SpotLabelValue        string            `json:"spotLabelValue,omitempty"`
-	GpuLabel              string            `json:"gpuLabel,omitempty"`
-	GpuLabelValue         string            `json:"gpuLabelValue,omitempty"`
-	ServiceKeyName        string            `json:"awsServiceKeyName,omitempty"`
-	ServiceKeySecret      string            `json:"awsServiceKeySecret,omitempty"`
-	SpotDataRegion        string            `json:"awsSpotDataRegion,omitempty"`
-	SpotDataBucket        string            `json:"awsSpotDataBucket,omitempty"`
-	SpotDataPrefix        string            `json:"awsSpotDataPrefix,omitempty"`
-	ProjectID             string            `json:"projectID,omitempty"`
-	AthenaProjectID       string            `json:"athenaProjectID,omitempty"`
-	AthenaBucketName      string            `json:"athenaBucketName"`
-	AthenaRegion          string            `json:"athenaRegion"`
-	AthenaDatabase        string            `json:"athenaDatabase"`
-	AthenaTable           string            `json:"athenaTable"`
-	MasterPayerARN        string            `json:"masterPayerARN"`
-	BillingDataDataset    string            `json:"billingDataDataset,omitempty"`
-	CustomPricesEnabled   string            `json:"customPricesEnabled"`
-	DefaultIdle           string            `json:"defaultIdle"`
-	AzureSubscriptionID   string            `json:"azureSubscriptionID"`
-	AzureClientID         string            `json:"azureClientID"`
-	AzureClientSecret     string            `json:"azureClientSecret"`
-	AzureTenantID         string            `json:"azureTenantID"`
-	AzureBillingRegion    string            `json:"azureBillingRegion"`
-	CurrencyCode          string            `json:"currencyCode"`
-	Discount              string            `json:"discount"`
-	NegotiatedDiscount    string            `json:"negotiatedDiscount"`
-	SharedCosts           map[string]string `json:"sharedCost"`
-	ClusterName           string            `json:"clusterName"`
-	SharedNamespaces      string            `json:"sharedNamespaces"`
-	SharedLabelNames      string            `json:"sharedLabelNames"`
-	SharedLabelValues     string            `json:"sharedLabelValues"`
-	ReadOnly              string            `json:"readOnly"`
+	Provider                     string            `json:"provider"`
+	Description                  string            `json:"description"`
+	CPU                          string            `json:"CPU"`
+	SpotCPU                      string            `json:"spotCPU"`
+	RAM                          string            `json:"RAM"`
+	SpotRAM                      string            `json:"spotRAM"`
+	GPU                          string            `json:"GPU"`
+	SpotGPU                      string            `json:"spotGPU"`
+	Storage                      string            `json:"storage"`
+	ZoneNetworkEgress            string            `json:"zoneNetworkEgress"`
+	RegionNetworkEgress          string            `json:"regionNetworkEgress"`
+	InternetNetworkEgress        string            `json:"internetNetworkEgress"`
+	FirstFiveForwardingRulesCost string            `json:"firstFiveForwardingRulesCost"`
+	AdditionalForwardingRuleCost string            `json:"additionalForwardingRuleCost"`
+	LBIngressDataCost            string            `json:"LBIngressDataCost"`
+	SpotLabel                    string            `json:"spotLabel,omitempty"`
+	SpotLabelValue               string            `json:"spotLabelValue,omitempty"`
+	GpuLabel                     string            `json:"gpuLabel,omitempty"`
+	GpuLabelValue                string            `json:"gpuLabelValue,omitempty"`
+	ServiceKeyName               string            `json:"awsServiceKeyName,omitempty"`
+	ServiceKeySecret             string            `json:"awsServiceKeySecret,omitempty"`
+	SpotDataRegion               string            `json:"awsSpotDataRegion,omitempty"`
+	SpotDataBucket               string            `json:"awsSpotDataBucket,omitempty"`
+	SpotDataPrefix               string            `json:"awsSpotDataPrefix,omitempty"`
+	ProjectID                    string            `json:"projectID,omitempty"`
+	AthenaProjectID              string            `json:"athenaProjectID,omitempty"`
+	AthenaBucketName             string            `json:"athenaBucketName"`
+	AthenaRegion                 string            `json:"athenaRegion"`
+	AthenaDatabase               string            `json:"athenaDatabase"`
+	AthenaTable                  string            `json:"athenaTable"`
+	MasterPayerARN               string            `json:"masterPayerARN"`
+	BillingDataDataset           string            `json:"billingDataDataset,omitempty"`
+	CustomPricesEnabled          string            `json:"customPricesEnabled"`
+	DefaultIdle                  string            `json:"defaultIdle"`
+	AzureSubscriptionID          string            `json:"azureSubscriptionID"`
+	AzureClientID                string            `json:"azureClientID"`
+	AzureClientSecret            string            `json:"azureClientSecret"`
+	AzureTenantID                string            `json:"azureTenantID"`
+	AzureBillingRegion           string            `json:"azureBillingRegion"`
+	CurrencyCode                 string            `json:"currencyCode"`
+	Discount                     string            `json:"discount"`
+	NegotiatedDiscount           string            `json:"negotiatedDiscount"`
+	SharedCosts                  map[string]string `json:"sharedCost"`
+	ClusterName                  string            `json:"clusterName"`
+	SharedNamespaces             string            `json:"sharedNamespaces"`
+	SharedLabelNames             string            `json:"sharedLabelNames"`
+	SharedLabelValues            string            `json:"sharedLabelValues"`
+	ReadOnly                     string            `json:"readOnly"`
 }
 
 type ServiceAccountStatus struct {
@@ -175,7 +190,8 @@ type Provider interface {
 	GetDisks() ([]byte, error)
 	NodePricing(Key) (*Node, error)
 	PVPricing(PVKey) (*PV, error)
-	NetworkPricing() (*Network, error)
+	NetworkPricing() (*Network, error)           // TODO: add key interface arg for dynamic price fetching
+	LoadBalancerPricing() (*LoadBalancer, error) // TODO: add key interface arg for dynamic price fetching
 	AllNodePricing() (interface{}, error)
 	DownloadPricingData() error
 	GetKey(map[string]string, *v1.Node) Key

pkg/costmodel/cluster.go

@@ -783,6 +783,110 @@ func ClusterNodes(cp cloud.Provider, client prometheus.Client, duration, offset
 	return nodeMap, nil
 }
 
+type LoadBalancer struct {
+	Cluster    string
+	Name       string
+	ProviderID string
+	Cost       float64
+	Start      time.Time
+	Minutes    float64
+}
+
+func ClusterLoadBalancers(cp cloud.Provider, client prometheus.Client, duration, offset time.Duration) (map[string]*LoadBalancer, []error) {
+	durationStr := fmt.Sprintf("%dm", int64(duration.Minutes()))
+	offsetStr := fmt.Sprintf(" offset %dm", int64(offset.Minutes()))
+	if offset < time.Minute {
+		offsetStr = ""
+	}
+
+	// minsPerResolution determines accuracy and resource use for the following
+	// queries. Smaller values (higher resolution) result in better accuracy,
+	// but more expensive queries, and vice-a-versa.
+	minsPerResolution := 5
+
+	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
+	// value, converts it to a cumulative value; i.e.
+	// [$/hr] * [min/res]*[hr/min] = [$/res]
+	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
+
+	ctx := prom.NewContext(client)
+	queryLBCost := fmt.Sprintf(`sum_over_time((avg(kubecost_load_balancer_cost) by (namespace, service_name))[%s:%dm]%s) * %f`, durationStr, minsPerResolution, offsetStr, hourlyToCumulative)
+	queryActiveMins := fmt.Sprintf(`count(kubecost_load_balancer_cost) by (namespace, service_name)[%s:%dm]%s`, durationStr, minsPerResolution, offsetStr)
+
+	resChLBCost := ctx.Query(queryLBCost)
+	resChActiveMins := ctx.Query(queryActiveMins)
+
+	resLBCost, _ := resChLBCost.Await()
+	resActiveMins, _ := resChActiveMins.Await()
+
+	if ctx.ErrorCollector.IsError() {
+		return nil, ctx.Errors()
+	}
+
+	loadBalancerMap := map[string]*LoadBalancer{}
+
+	for _, result := range resLBCost {
+		cluster, err := result.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+		namespace, err := result.GetString("namespace")
+		if err != nil {
+			log.Warningf("ClusterLoadBalancers: LB cost data missing namespace")
+			continue
+		}
+		serviceName, err := result.GetString("service_name")
+		if err != nil {
+			log.Warningf("ClusterLoadBalancers: LB cost data missing service_name")
+			continue
+		}
+		providerID := ""
+		lbCost := result.Values[0].Value
+
+		key := fmt.Sprintf("%s/%s/%s", cluster, namespace, serviceName)
+		if _, ok := loadBalancerMap[key]; !ok {
+			loadBalancerMap[key] = &LoadBalancer{
+				Cluster:    cluster,
+				Name:       namespace + "/" + serviceName,
+				ProviderID: providerID, // cp.ParseID(providerID) if providerID does get recorded later
+			}
+		}
+		loadBalancerMap[key].Cost += lbCost
+	}
+
+	for _, result := range resActiveMins {
+		cluster, err := result.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+		namespace, err := result.GetString("namespace")
+		if err != nil {
+			log.Warningf("ClusterLoadBalancers: LB cost data missing namespace")
+			continue
+		}
+		serviceName, err := result.GetString("service_name")
+		if err != nil {
+			log.Warningf("ClusterLoadBalancers: LB cost data missing service_name")
+			continue
+		}
+		key := fmt.Sprintf("%s/%s/%s", cluster, namespace, serviceName)
+
+		if len(result.Values) == 0 {
+			continue
+		}
+
+		s := time.Unix(int64(result.Values[0].Timestamp), 0)
+		e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0)
+		mins := e.Sub(s).Minutes()
+
+		// TODO niko/assets if mins >= threshold, interpolate for missing data?
+
+		loadBalancerMap[key].Start = s
+		loadBalancerMap[key].Minutes = mins
+	}
+	return loadBalancerMap, nil
+}
+
 // ComputeClusterCosts gives the cumulative and monthly-rate cluster costs over a window of time for all clusters.
 func ComputeClusterCosts(client prometheus.Client, provider cloud.Provider, window, offset string, withBreakdown bool) (map[string]*ClusterCosts, error) {
 	// Compute number of minutes in the full interval, for use interpolating missed scrapes or scaling missing data

pkg/costmodel/costmodel.go

@@ -1183,6 +1183,37 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 	return nodes, nil
 }
 
+// TODO: drop some logs
+func (cm *CostModel) GetLBCost(cp costAnalyzerCloud.Provider) (map[string]*costAnalyzerCloud.LoadBalancer, error) {
+	// for fetching prices from cloud provider
+	// cfg, err := cp.GetConfig()
+	// if err != nil {
+	// 	return nil, err
+	// }
+
+	servicesList := cm.Cache.GetAllServices()
+	loadBalancerMap := make(map[string]*costAnalyzerCloud.LoadBalancer)
+
+	for _, service := range servicesList {
+		namespace := service.GetObjectMeta().GetNamespace()
+		name := service.GetObjectMeta().GetName()
+		key := namespace + "," + name // + "," + clusterID?
+
+		if service.Spec.Type == "LoadBalancer" {
+			loadBalancer, err := cp.LoadBalancerPricing()
+			if err != nil {
+				return nil, err
+			}
+			newLoadBalancer := *loadBalancer
+			for _, loadBalancerIngress := range service.Status.LoadBalancer.Ingress {
+				newLoadBalancer.IngressIPAddresses = append(newLoadBalancer.IngressIPAddresses, loadBalancerIngress.IP)
+			}
+			loadBalancerMap[key] = &newLoadBalancer
+		}
+	}
+	return loadBalancerMap, nil
+}
+
 func getPodServices(cache clustercache.ClusterCache, podList []*v1.Pod, clusterID string) (map[string]map[string][]string, error) {
 	servicesList := cache.GetAllServices()
 	podServicesMapping := make(map[string]map[string][]string)

pkg/costmodel/metrics.go

@@ -408,6 +408,7 @@ func StartCostModelMetricRecording(a *Accesses) bool {
 
 		containerSeen := make(map[string]bool)
 		nodeSeen := make(map[string]bool)
+		loadBalancerSeen := make(map[string]bool)
 		pvSeen := make(map[string]bool)
 		pvcSeen := make(map[string]bool)
 
@@ -511,6 +512,19 @@ func StartCostModelMetricRecording(a *Accesses) bool {
 				nodeSeen[labelKey] = true
 			}
 
+			loadBalancers, err := a.Model.GetLBCost(a.Cloud)
+			for lbKey, lb := range loadBalancers {
+				// TODO: parse (if necessary) and calculate cost associated with loadBalancer based on dynamic cloud prices fetched into each lb struct on GetLBCost() call
+				keyParts := getLabelStringsFromKey(lbKey)
+				namespace := keyParts[0]
+				serviceName := keyParts[1]
+				ingressIP := lb.IngressIPAddresses[0] // assumes one ingress IP per load balancer
+				a.LBCostRecorder.WithLabelValues(ingressIP, namespace, serviceName).Set(lb.Cost)
+
+				labelKey := getKeyFromLabelStrings(namespace, serviceName)
+				loadBalancerSeen[labelKey] = true
+			}
+
 			for _, costs := range data {
 				nodeName := costs.NodeName
 
@@ -623,6 +637,14 @@ func StartCostModelMetricRecording(a *Accesses) bool {
 					nodeSeen[labelString] = false
 				}
 			}
+			for labelString, seen := range loadBalancerSeen {
+				if !seen {
+					labels := getLabelStringsFromKey(labelString)
+					a.LBCostRecorder.DeleteLabelValues(labels...)
+				} else {
+					loadBalancerSeen[labelString] = false
+				}
+			}
 			for labelString, seen := range containerSeen {
 				if !seen {
 					labels := getLabelStringsFromKey(labelString)

pkg/costmodel/router.go

@@ -75,6 +75,7 @@ type Accesses struct {
 	GPUAllocationRecorder         *prometheus.GaugeVec
 	PVAllocationRecorder          *prometheus.GaugeVec
 	ClusterManagementCostRecorder *prometheus.GaugeVec
+	LBCostRecorder                *prometheus.GaugeVec
 	NetworkZoneEgressRecorder     prometheus.Gauge
 	NetworkRegionEgressRecorder   prometheus.Gauge
 	NetworkInternetEgressRecorder prometheus.Gauge
@@ -829,7 +830,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) {
 
 	// TODO: General Architecture Note: Several passes have been made to modularize a lot of
 	// TODO: our code, but the router still continues to be the obvious entry point for new \
-	// TODO: features. We should look to spliting out the actual "router" functionality and
+	// TODO: features. We should look to split out the actual "router" functionality and
 	// TODO: implement a builder -> controller for stitching new features and other dependencies.
 	clusterManager := newClusterManager()
 
@@ -898,6 +899,10 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) {
 		Name: "kubecost_cluster_management_cost",
 		Help: "kubecost_cluster_management_cost Hourly cost paid as a cluster management fee.",
 	}, []string{"provisioner_name"})
+	LBCostRecorder := prometheus.NewGaugeVec(prometheus.GaugeOpts{ // no differentiation between ELB and ALB right now
+		Name: "kubecost_load_balancer_cost",
+		Help: "kubecost_load_balancer_cost Hourly cost of load balancer",
+	}, []string{"ingress_ip", "namespace", "service_name"}) // assumes one ingress IP per load balancer
 
 	prometheus.MustRegister(cpuGv)
 	prometheus.MustRegister(ramGv)
@@ -911,6 +916,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) {
 	prometheus.MustRegister(GPUAllocation)
 	prometheus.MustRegister(NetworkZoneEgressRecorder, NetworkRegionEgressRecorder, NetworkInternetEgressRecorder)
 	prometheus.MustRegister(ClusterManagementCostRecorder)
+	prometheus.MustRegister(LBCostRecorder)
 	prometheus.MustRegister(ServiceCollector{
 		KubeClientSet: kubeClientset,
 	})
@@ -947,6 +953,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) {
 		NetworkInternetEgressRecorder: NetworkInternetEgressRecorder,
 		PersistentVolumePriceRecorder: pvGv,
 		ClusterManagementCostRecorder: ClusterManagementCostRecorder,
+		LBCostRecorder:                LBCostRecorder,
 		Model:                         NewCostModel(k8sCache),
 		OutOfClusterCache:             outOfClusterCache,
 	}