package cloud import ( "context" "fmt" "io" "io/ioutil" "math" "net/http" "regexp" "strconv" "strings" "sync" "time" "github.com/kubecost/cost-model/pkg/clustercache" "github.com/kubecost/cost-model/pkg/env" "github.com/kubecost/cost-model/pkg/log" "github.com/kubecost/cost-model/pkg/util" "github.com/kubecost/cost-model/pkg/util/fileutil" "github.com/kubecost/cost-model/pkg/util/json" "github.com/kubecost/cost-model/pkg/util/timeutil" "cloud.google.com/go/bigquery" "cloud.google.com/go/compute/metadata" "golang.org/x/oauth2" "golang.org/x/oauth2/google" compute "google.golang.org/api/compute/v1" v1 "k8s.io/api/core/v1" "k8s.io/klog" ) const GKE_GPU_TAG = "cloud.google.com/gke-accelerator" const BigqueryUpdateType = "bigqueryupdate" // List obtained by installing the `gcloud` CLI tool, // logging into gcp account, and running command // `gcloud compute regions list` var gcpRegions = []string{ "asia-east1", "asia-east2", "asia-northeast1", "asia-northeast2", "asia-northeast3", "asia-south1", "asia-south2", "asia-southeast1", "asia-southeast2", "australia-southeast1", "australia-southeast2", "europe-central2", "europe-north1", "europe-west1", "europe-west2", "europe-west3", "europe-west4", "europe-west6", "northamerica-northeast1", "northamerica-northeast2", "southamerica-east1", "us-central1", "us-east1", "us-east4", "us-west1", "us-west2", "us-west3", "us-west4", } type userAgentTransport struct { userAgent string base http.RoundTripper } func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { req.Header.Set("User-Agent", t.userAgent) return t.base.RoundTrip(req) } // GCP implements a provider interface for GCP type GCP struct { Pricing map[string]*GCPPricing Clientset clustercache.ClusterCache APIKey string BaseCPUPrice string ProjectID string BillingDataDataset string DownloadPricingDataLock sync.RWMutex ReservedInstances []*GCPReservedInstance Config *ProviderConfig ServiceKeyProvided bool ValidPricingKeys map[string]bool clusterManagementPrice float64 clusterProjectId string clusterRegion string clusterProvisioner string *CustomProvider } type gcpAllocation struct { Aggregator bigquery.NullString Environment bigquery.NullString Service string Cost float64 } type multiKeyGCPAllocation struct { Keys bigquery.NullString Service string Cost float64 } // GetLocalStorageQuery returns the cost of local storage for the given window. Setting rate=true // returns hourly spend. Setting used=true only tracks used storage, not total. func (gcp *GCP) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string { // TODO Set to the price for the appropriate storage class. It's not trivial to determine the local storage disk type // See https://cloud.google.com/compute/disks-image-pricing#persistentdisk localStorageCost := 0.04 baseMetric := "container_fs_limit_bytes" if used { baseMetric = "container_fs_usage_bytes" } fmtOffset := timeutil.DurationToPromOffsetString(offset) fmtCumulativeQuery := `sum( sum_over_time(%s{device!="tmpfs", id="/"}[%s:1m]%s) ) by (%s) / 60 / 730 / 1024 / 1024 / 1024 * %f` fmtMonthlyQuery := `sum( avg_over_time(%s{device!="tmpfs", id="/"}[%s:1m]%s) ) by (%s) / 1024 / 1024 / 1024 * %f` fmtQuery := fmtCumulativeQuery if rate { fmtQuery = fmtMonthlyQuery } fmtWindow := timeutil.DurationString(window) return fmt.Sprintf(fmtQuery, baseMetric, fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost) } func (gcp *GCP) GetConfig() (*CustomPricing, error) { c, err := gcp.Config.GetCustomPricingData() if err != nil { return nil, err } if c.Discount == "" { c.Discount = "30%" } if c.NegotiatedDiscount == "" { c.NegotiatedDiscount = "0%" } if c.CurrencyCode == "" { c.CurrencyCode = "USD" } if c.ShareTenancyCosts == "" { c.ShareTenancyCosts = defaultShareTenancyCost } return c, nil } // BigQueryConfig contain the required config and credentials to access OOC resources for GCP type BigQueryConfig struct { ProjectID string `json:"projectID"` BillingDataDataset string `json:"billingDataDataset"` Key map[string]string `json:"key"` } // IsEmpty returns true if all fields in config are empty, false if not. func (bqc *BigQueryConfig) IsEmpty() bool { return bqc.ProjectID == "" && bqc.BillingDataDataset == "" && (bqc.Key == nil || len(bqc.Key) == 0) } func (gcp *GCP) GetManagementPlatform() (string, error) { nodes := gcp.Clientset.GetAllNodes() if len(nodes) > 0 { n := nodes[0] version := n.Status.NodeInfo.KubeletVersion if strings.Contains(version, "gke") { return "gke", nil } } return "", nil } // Attempts to load a GCP auth secret and copy the contents to the key file. func (*GCP) loadGCPAuthSecret() { path := env.GetConfigPathWithDefault("/models/") keyPath := path + "key.json" keyExists, _ := fileutil.FileExists(keyPath) if keyExists { klog.V(1).Infof("GCP Auth Key already exists, no need to load from secret") return } exists, err := fileutil.FileExists(authSecretPath) if !exists || err != nil { errMessage := "Secret does not exist" if err != nil { errMessage = err.Error() } klog.V(4).Infof("[Warning] Failed to load auth secret, or was not mounted: %s", errMessage) return } result, err := ioutil.ReadFile(authSecretPath) if err != nil { klog.V(4).Infof("[Warning] Failed to load auth secret, or was not mounted: %s", err.Error()) return } err = ioutil.WriteFile(keyPath, result, 0644) if err != nil { klog.V(4).Infof("[Warning] Failed to copy auth secret to %s: %s", keyPath, err.Error()) } } func (gcp *GCP) UpdateConfigFromConfigMap(a map[string]string) (*CustomPricing, error) { return gcp.Config.UpdateFromMap(a) } func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) { return gcp.Config.Update(func(c *CustomPricing) error { if updateType == BigqueryUpdateType { a := BigQueryConfig{} err := json.NewDecoder(r).Decode(&a) if err != nil { return err } c.ProjectID = a.ProjectID c.BillingDataDataset = a.BillingDataDataset if len(a.Key) > 0 { j, err := json.Marshal(a.Key) if err != nil { return err } path := env.GetConfigPathWithDefault("/models/") keyPath := path + "key.json" err = ioutil.WriteFile(keyPath, j, 0644) if err != nil { return err } gcp.ServiceKeyProvided = true } } else if updateType == AthenaInfoUpdateType { a := AwsAthenaInfo{} err := json.NewDecoder(r).Decode(&a) if err != nil { return err } c.AthenaBucketName = a.AthenaBucketName c.AthenaRegion = a.AthenaRegion c.AthenaDatabase = a.AthenaDatabase c.AthenaTable = a.AthenaTable c.ServiceKeyName = a.ServiceKeyName c.ServiceKeySecret = a.ServiceKeySecret c.AthenaProjectID = a.AccountID } else { a := make(map[string]interface{}) err := json.NewDecoder(r).Decode(&a) if err != nil { return err } for k, v := range a { kUpper := strings.Title(k) // Just so we consistently supply / receive the same values, uppercase the first letter. vstr, ok := v.(string) if ok { err := SetCustomPricingField(c, kUpper, vstr) if err != nil { return err } } else { return fmt.Errorf("type error while updating config for %s", kUpper) } } } if env.IsRemoteEnabled() { err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName) if err != nil { return err } } return nil }) } // ClusterName returns the name of a GKE cluster, as provided by metadata. func (gcp *GCP) ClusterInfo() (map[string]string, error) { remoteEnabled := env.IsRemoteEnabled() metadataClient := metadata.NewClient(&http.Client{Transport: userAgentTransport{ userAgent: "kubecost", base: http.DefaultTransport, }}) attribute, err := metadataClient.InstanceAttributeValue("cluster-name") if err != nil { klog.Infof("Error loading metadata cluster-name: %s", err.Error()) } c, err := gcp.GetConfig() if err != nil { klog.V(1).Infof("Error opening config: %s", err.Error()) } if c.ClusterName != "" { attribute = c.ClusterName } m := make(map[string]string) m["name"] = attribute m["provider"] = "GCP" m["project"] = gcp.clusterProjectId m["region"] = gcp.clusterRegion m["provisioner"] = gcp.clusterProvisioner m["id"] = env.GetClusterID() m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled) return m, nil } func (gcp *GCP) ClusterManagementPricing() (string, float64, error) { return gcp.clusterProvisioner, gcp.clusterManagementPrice, nil } func (*GCP) GetAddresses() ([]byte, error) { // metadata API setup metadataClient := metadata.NewClient(&http.Client{Transport: userAgentTransport{ userAgent: "kubecost", base: http.DefaultTransport, }}) projID, err := metadataClient.ProjectID() if err != nil { return nil, err } client, err := google.DefaultClient(oauth2.NoContext, "https://www.googleapis.com/auth/compute.readonly") if err != nil { return nil, err } svc, err := compute.New(client) if err != nil { return nil, err } res, err := svc.Addresses.AggregatedList(projID).Do() if err != nil { return nil, err } return json.Marshal(res) } // GetDisks returns the GCP disks backing PVs. Useful because sometimes k8s will not clean up PVs correctly. Requires a json config in /var/configs with key region. func (*GCP) GetDisks() ([]byte, error) { // metadata API setup metadataClient := metadata.NewClient(&http.Client{Transport: userAgentTransport{ userAgent: "kubecost", base: http.DefaultTransport, }}) projID, err := metadataClient.ProjectID() if err != nil { return nil, err } client, err := google.DefaultClient(oauth2.NoContext, "https://www.googleapis.com/auth/compute.readonly") if err != nil { return nil, err } svc, err := compute.New(client) if err != nil { return nil, err } res, err := svc.Disks.AggregatedList(projID).Do() if err != nil { return nil, err } return json.Marshal(res) } // GCPPricing represents GCP pricing data for a SKU type GCPPricing struct { Name string `json:"name"` SKUID string `json:"skuId"` Description string `json:"description"` Category *GCPResourceInfo `json:"category"` ServiceRegions []string `json:"serviceRegions"` PricingInfo []*PricingInfo `json:"pricingInfo"` ServiceProviderName string `json:"serviceProviderName"` Node *Node `json:"node"` PV *PV `json:"pv"` } // PricingInfo contains metadata about a cost. type PricingInfo struct { Summary string `json:"summary"` PricingExpression *PricingExpression `json:"pricingExpression"` CurrencyConversionRate float64 `json:"currencyConversionRate"` EffectiveTime string `json:""` } // PricingExpression contains metadata about a cost. type PricingExpression struct { UsageUnit string `json:"usageUnit"` UsageUnitDescription string `json:"usageUnitDescription"` BaseUnit string `json:"baseUnit"` BaseUnitConversionFactor int64 `json:"-"` DisplayQuantity int `json:"displayQuantity"` TieredRates []*TieredRates `json:"tieredRates"` } // TieredRates contain data about variable pricing. type TieredRates struct { StartUsageAmount int `json:"startUsageAmount"` UnitPrice *UnitPriceInfo `json:"unitPrice"` } // UnitPriceInfo contains data about the actual price being charged. type UnitPriceInfo struct { CurrencyCode string `json:"currencyCode"` Units string `json:"units"` Nanos float64 `json:"nanos"` } // GCPResourceInfo contains metadata about the node. type GCPResourceInfo struct { ServiceDisplayName string `json:"serviceDisplayName"` ResourceFamily string `json:"resourceFamily"` ResourceGroup string `json:"resourceGroup"` UsageType string `json:"usageType"` } func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[string]PVKey) (map[string]*GCPPricing, string, error) { gcpPricingList := make(map[string]*GCPPricing) var nextPageToken string dec := json.NewDecoder(r) for { t, err := dec.Token() if err == io.EOF { break } if t == "skus" { _, err := dec.Token() // consumes [ if err != nil { return nil, "", err } for dec.More() { product := &GCPPricing{} err := dec.Decode(&product) if err != nil { return nil, "", err } usageType := strings.ToLower(product.Category.UsageType) instanceType := strings.ToLower(product.Category.ResourceGroup) if instanceType == "ssd" && strings.Contains(product.Description, "SSD backed") && !strings.Contains(product.Description, "Regional") { // TODO: support regional lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1 var nanos float64 if lastRateIndex > -1 && len(product.PricingInfo) > 0 { nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos } else { continue } hourlyPrice := (nanos * math.Pow10(-9)) / 730 for _, sr := range product.ServiceRegions { region := sr candidateKey := region + "," + "ssd" if _, ok := pvKeys[candidateKey]; ok { product.PV = &PV{ Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } gcpPricingList[candidateKey] = product continue } } continue } else if instanceType == "ssd" && strings.Contains(product.Description, "SSD backed") && strings.Contains(product.Description, "Regional") { // TODO: support regional lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1 var nanos float64 if lastRateIndex > -1 && len(product.PricingInfo) > 0 { nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos } else { continue } hourlyPrice := (nanos * math.Pow10(-9)) / 730 for _, sr := range product.ServiceRegions { region := sr candidateKey := region + "," + "ssd" + "," + "regional" if _, ok := pvKeys[candidateKey]; ok { product.PV = &PV{ Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } gcpPricingList[candidateKey] = product continue } } continue } else if instanceType == "pdstandard" && !strings.Contains(product.Description, "Regional") { // TODO: support regional lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1 var nanos float64 if lastRateIndex > -1 && len(product.PricingInfo) > 0 { nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos } else { continue } hourlyPrice := (nanos * math.Pow10(-9)) / 730 for _, sr := range product.ServiceRegions { region := sr candidateKey := region + "," + "pdstandard" if _, ok := pvKeys[candidateKey]; ok { product.PV = &PV{ Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } gcpPricingList[candidateKey] = product continue } } continue } else if instanceType == "pdstandard" && strings.Contains(product.Description, "Regional") { // TODO: support regional lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1 var nanos float64 if lastRateIndex > -1 && len(product.PricingInfo) > 0 { nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos } else { continue } hourlyPrice := (nanos * math.Pow10(-9)) / 730 for _, sr := range product.ServiceRegions { region := sr candidateKey := region + "," + "pdstandard" + "," + "regional" if _, ok := pvKeys[candidateKey]; ok { product.PV = &PV{ Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } gcpPricingList[candidateKey] = product continue } } continue } if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "CUSTOM") { instanceType = "custom" } if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "N2") && !strings.Contains(strings.ToUpper(product.Description), "PREMIUM") { if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "N2D AMD") { instanceType = "n2dstandard" } else { instanceType = "n2standard" } } if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "COMPUTE OPTIMIZED") { instanceType = "c2standard" } if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "E2 INSTANCE") { instanceType = "e2" } partialCPUMap := make(map[string]float64) partialCPUMap["e2micro"] = 0.25 partialCPUMap["e2small"] = 0.5 partialCPUMap["e2medium"] = 1 /* var partialCPU float64 if strings.ToLower(instanceType) == "f1micro" { partialCPU = 0.2 } else if strings.ToLower(instanceType) == "g1small" { partialCPU = 0.5 } */ var gpuType string provIdRx := regexp.MustCompile("(Nvidia Tesla [^ ]+) ") for matchnum, group := range provIdRx.FindStringSubmatch(product.Description) { if matchnum == 1 { gpuType = strings.ToLower(strings.Join(strings.Split(group, " "), "-")) klog.V(4).Info("GPU type found: " + gpuType) } } candidateKeys := []string{} if gcp.ValidPricingKeys == nil { gcp.ValidPricingKeys = make(map[string]bool) } for _, region := range product.ServiceRegions { if instanceType == "e2" { // this needs to be done to handle a partial cpu mapping candidateKeys = append(candidateKeys, region+","+"e2micro"+","+usageType) candidateKeys = append(candidateKeys, region+","+"e2small"+","+usageType) candidateKeys = append(candidateKeys, region+","+"e2medium"+","+usageType) candidateKeys = append(candidateKeys, region+","+"e2standard"+","+usageType) candidateKeys = append(candidateKeys, region+","+"e2custom"+","+usageType) } else { candidateKey := region + "," + instanceType + "," + usageType candidateKeys = append(candidateKeys, candidateKey) } } for _, candidateKey := range candidateKeys { instanceType = strings.Split(candidateKey, ",")[1] // we may have overriden this while generating candidate keys region := strings.Split(candidateKey, ",")[0] candidateKeyGPU := candidateKey + ",gpu" gcp.ValidPricingKeys[candidateKey] = true gcp.ValidPricingKeys[candidateKeyGPU] = true if gpuType != "" { lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1 var nanos float64 if lastRateIndex > -1 && len(product.PricingInfo) > 0 { nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos } else { continue } hourlyPrice := nanos * math.Pow10(-9) for k, key := range inputKeys { if key.GPUType() == gpuType+","+usageType { if region == strings.Split(k, ",")[0] { klog.V(3).Infof("Matched GPU to node in region \"%s\"", region) klog.V(4).Infof("PRODUCT DESCRIPTION: %s", product.Description) matchedKey := key.Features() if pl, ok := gcpPricingList[matchedKey]; ok { pl.Node.GPUName = gpuType pl.Node.GPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64) pl.Node.GPU = "1" } else { product.Node = &Node{ GPUName: gpuType, GPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), GPU: "1", } gcpPricingList[matchedKey] = product } klog.V(3).Infof("Added data for " + matchedKey) } } } } else { _, ok := inputKeys[candidateKey] _, ok2 := inputKeys[candidateKeyGPU] if ok || ok2 { lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1 var nanos float64 if lastRateIndex > -1 && len(product.PricingInfo) > 0 { nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos } else { continue } hourlyPrice := nanos * math.Pow10(-9) if hourlyPrice == 0 { continue } else if strings.Contains(strings.ToUpper(product.Description), "RAM") { if instanceType == "custom" { klog.V(4).Infof("RAM custom sku is: " + product.Name) } if _, ok := gcpPricingList[candidateKey]; ok { gcpPricingList[candidateKey].Node.RAMCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64) } else { product = &GCPPricing{} product.Node = &Node{ RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } partialCPU, pcok := partialCPUMap[instanceType] if pcok { product.Node.VCPU = fmt.Sprintf("%f", partialCPU) } product.Node.UsageType = usageType gcpPricingList[candidateKey] = product } if _, ok := gcpPricingList[candidateKeyGPU]; ok { klog.V(1).Infof("Adding RAM %f for %s", hourlyPrice, candidateKeyGPU) gcpPricingList[candidateKeyGPU].Node.RAMCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64) } else { klog.V(1).Infof("Adding RAM %f for %s", hourlyPrice, candidateKeyGPU) product = &GCPPricing{} product.Node = &Node{ RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } partialCPU, pcok := partialCPUMap[instanceType] if pcok { product.Node.VCPU = fmt.Sprintf("%f", partialCPU) } product.Node.UsageType = usageType gcpPricingList[candidateKeyGPU] = product } break } else { if _, ok := gcpPricingList[candidateKey]; ok { gcpPricingList[candidateKey].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64) } else { product = &GCPPricing{} product.Node = &Node{ VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } partialCPU, pcok := partialCPUMap[instanceType] if pcok { product.Node.VCPU = fmt.Sprintf("%f", partialCPU) } product.Node.UsageType = usageType gcpPricingList[candidateKey] = product } if _, ok := gcpPricingList[candidateKeyGPU]; ok { gcpPricingList[candidateKeyGPU].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64) } else { product = &GCPPricing{} product.Node = &Node{ VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64), } partialCPU, pcok := partialCPUMap[instanceType] if pcok { product.Node.VCPU = fmt.Sprintf("%f", partialCPU) } product.Node.UsageType = usageType gcpPricingList[candidateKeyGPU] = product } break } } } } } } if t == "nextPageToken" { pageToken, err := dec.Token() if err != nil { klog.V(2).Infof("Error parsing nextpage token: " + err.Error()) return nil, "", err } if pageToken.(string) != "" { nextPageToken = pageToken.(string) } else { nextPageToken = "done" } } } return gcpPricingList, nextPageToken, nil } func (gcp *GCP) parsePages(inputKeys map[string]Key, pvKeys map[string]PVKey) (map[string]*GCPPricing, error) { var pages []map[string]*GCPPricing c, err := gcp.GetConfig() if err != nil { return nil, err } url := "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus?key=" + gcp.APIKey + "¤cyCode=" + c.CurrencyCode klog.V(2).Infof("Fetch GCP Billing Data from URL: %s", url) var parsePagesHelper func(string) error parsePagesHelper = func(pageToken string) error { if pageToken == "done" { return nil } else if pageToken != "" { url = url + "&pageToken=" + pageToken } resp, err := http.Get(url) if err != nil { return err } page, token, err := gcp.parsePage(resp.Body, inputKeys, pvKeys) if err != nil { return err } pages = append(pages, page) return parsePagesHelper(token) } err = parsePagesHelper("") if err != nil { return nil, err } returnPages := make(map[string]*GCPPricing) for _, page := range pages { for k, v := range page { if val, ok := returnPages[k]; ok { //keys may need to be merged if val.Node != nil { if val.Node.VCPUCost == "" { val.Node.VCPUCost = v.Node.VCPUCost } if val.Node.RAMCost == "" { val.Node.RAMCost = v.Node.RAMCost } if val.Node.GPUCost == "" { val.Node.GPUCost = v.Node.GPUCost val.Node.GPU = v.Node.GPU val.Node.GPUName = v.Node.GPUName } } if val.PV != nil { if val.PV.Cost == "" { val.PV.Cost = v.PV.Cost } } } else { returnPages[k] = v } } } klog.V(1).Infof("ALL PAGES: %+v", returnPages) for k, v := range returnPages { if v.Node != nil { klog.V(1).Infof("Returned Page: %s : %+v", k, v.Node) } if v.PV != nil { klog.V(1).Infof("Returned Page: %s : %+v", k, v.PV) } } return returnPages, err } // DownloadPricingData fetches data from the GCP Pricing API. Requires a key-- a kubecost key is provided for quickstart, but should be replaced by a users. func (gcp *GCP) DownloadPricingData() error { gcp.DownloadPricingDataLock.Lock() defer gcp.DownloadPricingDataLock.Unlock() c, err := gcp.Config.GetCustomPricingData() if err != nil { klog.V(2).Infof("Error downloading default pricing data: %s", err.Error()) return err } gcp.loadGCPAuthSecret() gcp.BaseCPUPrice = c.CPU gcp.ProjectID = c.ProjectID gcp.BillingDataDataset = c.BillingDataDataset nodeList := gcp.Clientset.GetAllNodes() inputkeys := make(map[string]Key) defaultRegion := "" // Sometimes, PVs may be missing the region label. In that case assume that they are in the same region as the nodes for _, n := range nodeList { labels := n.GetObjectMeta().GetLabels() if _, ok := labels["cloud.google.com/gke-nodepool"]; ok { // The node is part of a GKE nodepool, so you're paying a cluster management cost gcp.clusterManagementPrice = 0.10 gcp.clusterProvisioner = "GKE" } r, _ := util.GetRegion(labels) if r != "" { defaultRegion = r } key := gcp.GetKey(labels, n) inputkeys[key.Features()] = key } pvList := gcp.Clientset.GetAllPersistentVolumes() storageClasses := gcp.Clientset.GetAllStorageClasses() storageClassMap := make(map[string]map[string]string) for _, storageClass := range storageClasses { params := storageClass.Parameters storageClassMap[storageClass.ObjectMeta.Name] = params if storageClass.GetAnnotations()["storageclass.kubernetes.io/is-default-class"] == "true" || storageClass.GetAnnotations()["storageclass.beta.kubernetes.io/is-default-class"] == "true" { storageClassMap["default"] = params storageClassMap[""] = params } } pvkeys := make(map[string]PVKey) for _, pv := range pvList { params, ok := storageClassMap[pv.Spec.StorageClassName] if !ok { log.DedupedWarningf(5, "Unable to find params for storageClassName %s", pv.Name) continue } key := gcp.GetPVKey(pv, params, defaultRegion) pvkeys[key.Features()] = key } reserved, err := gcp.getReservedInstances() if err != nil { klog.V(1).Infof("Failed to lookup reserved instance data: %s", err.Error()) } else { klog.V(1).Infof("Found %d reserved instances", len(reserved)) gcp.ReservedInstances = reserved for _, r := range reserved { klog.V(1).Infof("%s", r) } } pages, err := gcp.parsePages(inputkeys, pvkeys) if err != nil { return err } gcp.Pricing = pages return nil } func (gcp *GCP) PVPricing(pvk PVKey) (*PV, error) { gcp.DownloadPricingDataLock.RLock() defer gcp.DownloadPricingDataLock.RUnlock() pricing, ok := gcp.Pricing[pvk.Features()] if !ok { klog.V(3).Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features()) return &PV{}, nil } return pricing.PV, nil } // Stubbed NetworkPricing for GCP. Pull directly from gcp.json for now func (gcp *GCP) NetworkPricing() (*Network, error) { cpricing, err := gcp.Config.GetCustomPricingData() if err != nil { return nil, err } znec, err := strconv.ParseFloat(cpricing.ZoneNetworkEgress, 64) if err != nil { return nil, err } rnec, err := strconv.ParseFloat(cpricing.RegionNetworkEgress, 64) if err != nil { return nil, err } inec, err := strconv.ParseFloat(cpricing.InternetNetworkEgress, 64) if err != nil { return nil, err } return &Network{ ZoneNetworkEgressCost: znec, RegionNetworkEgressCost: rnec, InternetNetworkEgressCost: inec, }, nil } func (gcp *GCP) LoadBalancerPricing() (*LoadBalancer, error) { fffrc := 0.025 afrc := 0.010 lbidc := 0.008 numForwardingRules := 1.0 dataIngressGB := 0.0 var totalCost float64 if numForwardingRules < 5 { totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB } else { totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB } return &LoadBalancer{ Cost: totalCost, }, nil } const ( GCPReservedInstanceResourceTypeRAM string = "MEMORY" GCPReservedInstanceResourceTypeCPU string = "VCPU" GCPReservedInstanceStatusActive string = "ACTIVE" GCPReservedInstancePlanOneYear string = "TWELVE_MONTH" GCPReservedInstancePlanThreeYear string = "THIRTY_SIX_MONTH" ) type GCPReservedInstancePlan struct { Name string CPUCost float64 RAMCost float64 } type GCPReservedInstance struct { ReservedRAM int64 ReservedCPU int64 Plan *GCPReservedInstancePlan StartDate time.Time EndDate time.Time Region string } func (r *GCPReservedInstance) String() string { return fmt.Sprintf("[CPU: %d, RAM: %d, Region: %s, Start: %s, End: %s]", r.ReservedCPU, r.ReservedRAM, r.Region, r.StartDate.String(), r.EndDate.String()) } type GCPReservedCounter struct { RemainingCPU int64 RemainingRAM int64 Instance *GCPReservedInstance } func newReservedCounter(instance *GCPReservedInstance) *GCPReservedCounter { return &GCPReservedCounter{ RemainingCPU: instance.ReservedCPU, RemainingRAM: instance.ReservedRAM, Instance: instance, } } // Two available Reservation plans for GCP, 1-year and 3-year var gcpReservedInstancePlans map[string]*GCPReservedInstancePlan = map[string]*GCPReservedInstancePlan{ GCPReservedInstancePlanOneYear: &GCPReservedInstancePlan{ Name: GCPReservedInstancePlanOneYear, CPUCost: 0.019915, RAMCost: 0.002669, }, GCPReservedInstancePlanThreeYear: &GCPReservedInstancePlan{ Name: GCPReservedInstancePlanThreeYear, CPUCost: 0.014225, RAMCost: 0.001907, }, } func (gcp *GCP) ApplyReservedInstancePricing(nodes map[string]*Node) { numReserved := len(gcp.ReservedInstances) // Early return if no reserved instance data loaded if numReserved == 0 { klog.V(4).Infof("[Reserved] No Reserved Instances") return } now := time.Now() counters := make(map[string][]*GCPReservedCounter) for _, r := range gcp.ReservedInstances { if now.Before(r.StartDate) || now.After(r.EndDate) { klog.V(1).Infof("[Reserved] Skipped Reserved Instance due to dates") continue } _, ok := counters[r.Region] counter := newReservedCounter(r) if !ok { counters[r.Region] = []*GCPReservedCounter{counter} } else { counters[r.Region] = append(counters[r.Region], counter) } } gcpNodes := make(map[string]*v1.Node) currentNodes := gcp.Clientset.GetAllNodes() // Create a node name -> node map for _, gcpNode := range currentNodes { gcpNodes[gcpNode.GetName()] = gcpNode } // go through all provider nodes using k8s nodes for region for nodeName, node := range nodes { // Reset reserved allocation to prevent double allocation node.Reserved = nil kNode, ok := gcpNodes[nodeName] if !ok { klog.V(4).Infof("[Reserved] Could not find K8s Node with name: %s", nodeName) continue } nodeRegion, ok := util.GetRegion(kNode.Labels) if !ok { klog.V(4).Infof("[Reserved] Could not find node region") continue } reservedCounters, ok := counters[nodeRegion] if !ok { klog.V(4).Infof("[Reserved] Could not find counters for region: %s", nodeRegion) continue } node.Reserved = &ReservedInstanceData{ ReservedCPU: 0, ReservedRAM: 0, } for _, reservedCounter := range reservedCounters { if reservedCounter.RemainingCPU != 0 { nodeCPU, _ := strconv.ParseInt(node.VCPU, 10, 64) nodeCPU -= node.Reserved.ReservedCPU node.Reserved.CPUCost = reservedCounter.Instance.Plan.CPUCost if reservedCounter.RemainingCPU >= nodeCPU { reservedCounter.RemainingCPU -= nodeCPU node.Reserved.ReservedCPU += nodeCPU } else { node.Reserved.ReservedCPU += reservedCounter.RemainingCPU reservedCounter.RemainingCPU = 0 } } if reservedCounter.RemainingRAM != 0 { nodeRAMF, _ := strconv.ParseFloat(node.RAMBytes, 64) nodeRAM := int64(nodeRAMF) nodeRAM -= node.Reserved.ReservedRAM node.Reserved.RAMCost = reservedCounter.Instance.Plan.RAMCost if reservedCounter.RemainingRAM >= nodeRAM { reservedCounter.RemainingRAM -= nodeRAM node.Reserved.ReservedRAM += nodeRAM } else { node.Reserved.ReservedRAM += reservedCounter.RemainingRAM reservedCounter.RemainingRAM = 0 } } } } } func (gcp *GCP) getReservedInstances() ([]*GCPReservedInstance, error) { var results []*GCPReservedInstance ctx := context.Background() computeService, err := compute.NewService(ctx) if err != nil { return nil, err } commitments, err := computeService.RegionCommitments.AggregatedList(gcp.ProjectID).Do() if err != nil { return nil, err } for regionKey, commitList := range commitments.Items { for _, commit := range commitList.Commitments { if commit.Status != GCPReservedInstanceStatusActive { continue } var vcpu int64 = 0 var ram int64 = 0 for _, resource := range commit.Resources { switch resource.Type { case GCPReservedInstanceResourceTypeRAM: ram = resource.Amount * 1024 * 1024 case GCPReservedInstanceResourceTypeCPU: vcpu = resource.Amount default: klog.V(4).Infof("Failed to handle resource type: %s", resource.Type) } } var region string regionStr := strings.Split(regionKey, "/") if len(regionStr) == 2 { region = regionStr[1] } timeLayout := "2006-01-02T15:04:05Z07:00" startTime, err := time.Parse(timeLayout, commit.StartTimestamp) if err != nil { klog.V(1).Infof("Failed to parse start date: %s", commit.StartTimestamp) continue } endTime, err := time.Parse(timeLayout, commit.EndTimestamp) if err != nil { klog.V(1).Infof("Failed to parse end date: %s", commit.EndTimestamp) continue } // Look for a plan based on the name. Default to One Year if it fails plan, ok := gcpReservedInstancePlans[commit.Plan] if !ok { plan = gcpReservedInstancePlans[GCPReservedInstancePlanOneYear] } results = append(results, &GCPReservedInstance{ Region: region, ReservedRAM: ram, ReservedCPU: vcpu, Plan: plan, StartDate: startTime, EndDate: endTime, }) } } return results, nil } type pvKey struct { Labels map[string]string StorageClass string StorageClassParameters map[string]string DefaultRegion string } func (key *pvKey) ID() string { return "" } func (key *pvKey) GetStorageClass() string { return key.StorageClass } func (gcp *GCP) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey { return &pvKey{ Labels: pv.Labels, StorageClass: pv.Spec.StorageClassName, StorageClassParameters: parameters, DefaultRegion: defaultRegion, } } func (key *pvKey) Features() string { // TODO: regional cluster pricing. storageClass := key.StorageClassParameters["type"] if storageClass == "pd-ssd" { storageClass = "ssd" } else if storageClass == "pd-standard" { storageClass = "pdstandard" } replicationType := "" if rt, ok := key.StorageClassParameters["replication-type"]; ok { if rt == "regional-pd" { replicationType = ",regional" } } region, _ := util.GetRegion(key.Labels) if region == "" { region = key.DefaultRegion } return region + "," + storageClass + replicationType } type gcpKey struct { Labels map[string]string } func (gcp *GCP) GetKey(labels map[string]string, n *v1.Node) Key { return &gcpKey{ Labels: labels, } } func (gcp *gcpKey) ID() string { return "" } func (gcp *gcpKey) GPUType() string { if t, ok := gcp.Labels[GKE_GPU_TAG]; ok { var usageType string if t, ok := gcp.Labels["cloud.google.com/gke-preemptible"]; ok && t == "true" { usageType = "preemptible" } else { usageType = "ondemand" } klog.V(4).Infof("GPU of type: \"%s\" found", t) return t + "," + usageType } return "" } func parseGCPInstanceTypeLabel(it string) string { var instanceType string splitByDash := strings.Split(it, "-") // GKE nodes are labeled with the GCP instance type, but users can deploy on GCP // with tools like K3s, whose instance type labels will be "k3s". This logic // avoids a panic in the slice operation then there are no dashes (-) in the // instance type label value. if len(splitByDash) < 2 { instanceType = "unknown" } else { instanceType = strings.ToLower(strings.Join(splitByDash[:2], "")) if instanceType == "n1highmem" || instanceType == "n1highcpu" { instanceType = "n1standard" // These are priced the same. TODO: support n1ultrahighmem } else if instanceType == "n2highmem" || instanceType == "n2highcpu" { instanceType = "n2standard" } else if instanceType == "e2highmem" || instanceType == "e2highcpu" { instanceType = "e2standard" } else if strings.HasPrefix(instanceType, "custom") { instanceType = "custom" // The suffix of custom does not matter } } return instanceType } // GetKey maps node labels to information needed to retrieve pricing data func (gcp *gcpKey) Features() string { var instanceType string it, _ := util.GetInstanceType(gcp.Labels) if it == "" { log.DedupedErrorf(1, "Missing or Unknown 'node.kubernetes.io/instance-type' node label") instanceType = "unknown" } else { instanceType = parseGCPInstanceTypeLabel(it) } r, _ := util.GetRegion(gcp.Labels) region := strings.ToLower(r) var usageType string if t, ok := gcp.Labels["cloud.google.com/gke-preemptible"]; ok && t == "true" { usageType = "preemptible" } else { usageType = "ondemand" } if _, ok := gcp.Labels[GKE_GPU_TAG]; ok { return region + "," + instanceType + "," + usageType + "," + "gpu" } return region + "," + instanceType + "," + usageType } // AllNodePricing returns the GCP pricing objects stored func (gcp *GCP) AllNodePricing() (interface{}, error) { gcp.DownloadPricingDataLock.RLock() defer gcp.DownloadPricingDataLock.RUnlock() return gcp.Pricing, nil } func (gcp *GCP) getPricing(key Key) (*GCPPricing, bool) { gcp.DownloadPricingDataLock.RLock() defer gcp.DownloadPricingDataLock.RUnlock() n, ok := gcp.Pricing[key.Features()] return n, ok } func (gcp *GCP) isValidPricingKey(key Key) bool { gcp.DownloadPricingDataLock.RLock() defer gcp.DownloadPricingDataLock.RUnlock() _, ok := gcp.ValidPricingKeys[key.Features()] return ok } // NodePricing returns GCP pricing data for a single node func (gcp *GCP) NodePricing(key Key) (*Node, error) { if n, ok := gcp.getPricing(key); ok { klog.V(4).Infof("Returning pricing for node %s: %+v from SKU %s", key, n.Node, n.Name) n.Node.BaseCPUPrice = gcp.BaseCPUPrice return n.Node, nil } else if ok := gcp.isValidPricingKey(key); ok { err := gcp.DownloadPricingData() if err != nil { return nil, fmt.Errorf("Download pricing data failed: %s", err.Error()) } if n, ok := gcp.getPricing(key); ok { klog.V(4).Infof("Returning pricing for node %s: %+v from SKU %s", key, n.Node, n.Name) n.Node.BaseCPUPrice = gcp.BaseCPUPrice return n.Node, nil } klog.V(1).Infof("[Warning] no pricing data found for %s: %s", key.Features(), key) return nil, fmt.Errorf("Warning: no pricing data found for %s", key) } return nil, fmt.Errorf("Warning: no pricing data found for %s", key) } func (gcp *GCP) ServiceAccountStatus() *ServiceAccountStatus { return &ServiceAccountStatus{ Checks: []*ServiceAccountCheck{}, } } func (gcp *GCP) PricingSourceStatus() map[string]*PricingSource { return make(map[string]*PricingSource) } func (gcp *GCP) CombinedDiscountForNode(instanceType string, isPreemptible bool, defaultDiscount, negotiatedDiscount float64) float64 { class := strings.Split(instanceType, "-")[0] return 1.0 - ((1.0 - sustainedUseDiscount(class, defaultDiscount, isPreemptible)) * (1.0 - negotiatedDiscount)) } func (gcp *GCP) Regions() []string { return gcpRegions } func sustainedUseDiscount(class string, defaultDiscount float64, isPreemptible bool) float64 { if isPreemptible { return 0.0 } discount := defaultDiscount switch class { case "e2", "f1", "g1": discount = 0.0 case "n2", "n2d": discount = 0.2 } return discount } func parseGCPProjectID(id string) string { // gce://guestbook-12345/... // => guestbook-12345 rx := regexp.MustCompile("gce://([^/]*)/*") match := rx.FindStringSubmatch(id) if len(match) >= 2 { return match[1] } // Return empty string if an account could not be parsed from provided string return "" }