add files

Dockerfile

@@ -12,6 +12,7 @@ ENV PATH=$GOPATH/bin:$PATH
 RUN mkdir -p $GOPATH/src/app 
 RUN mkdir -p /models
 ADD ./cloud/default.json /models/default.json
+ADD ./cloud/azure.json /models/azure.json
 ADD . $GOPATH/src/app
 ADD ./cloud/ /go/src/app/vendor/github.com/kubecost/cost-model/cloud/
 ADD ./costmodel/ /go/src/app/vendor/github.com/kubecost/cost-model/costmodel/

cloud/provider.go

@@ -2,35 +2,35 @@ package cloud
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"log"
-	"math"
-	"net/http"
 	"net/url"
 	"os"
-	"strconv"
 	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws/awserr"
 
 	"cloud.google.com/go/compute/metadata"
-	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/google"
-	compute "google.golang.org/api/compute/v1"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/athena"
-	"github.com/aws/aws-sdk-go/service/ec2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
+// Node is the interface by which the provider and cost model communicate.
+// The provider will best-effort try to fill out this struct.
+type Node struct {
+	Cost             string `json:"hourlyCost"`
+	VCPU             string `json:"CPU"`
+	VCPUCost         string `json:"CPUHourlyCost"`
+	RAM              string `json:"RAM"`
+	RAMCost          string `json:"RAMGBHourlyCost"`
+	Storage          string `json:"storage"`
+	StorageCost      string `json:"storageHourlyCost"`
+	UsesBaseCPUPrice bool   `json:"usesDefaultPrice"`
+	BaseCPUPrice     string `json:"baseCPUPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
+	UsageType        string `json:"usageType"`
+}
+
+// Provider represents a k8s provider.
 type Provider interface {
 	ClusterName() ([]byte, error)
 	AddServiceKey(url.Values) error
@@ -43,8 +43,9 @@ type Provider interface {
 	QuerySQL(string) ([]byte, error)
 }
 
-func GetDefaultPricingData() (*CustomPricing, error) {
-	jsonFile, err := os.Open("/models/default.json")
+// GetDefaultPricingData will search for a json file representing pricing data in /models/ and use it for base pricing info.
+func GetDefaultPricingData(fname string) (*CustomPricing, error) {
+	jsonFile, err := os.Open("/models/" + fname)
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +54,7 @@ func GetDefaultPricingData() (*CustomPricing, error) {
 	if err != nil {
 		return nil, err
 	}
-	var customPricing *CustomPricing = &CustomPricing{}
+	var customPricing = &CustomPricing{}
 	err = json.Unmarshal([]byte(byteValue), customPricing)
 	if err != nil {
 		return nil, err
@@ -69,7 +70,7 @@ type CustomPricing struct {
 	RAM            string `json:"RAM"`
 	SpotRAM        string `json:"spotRAM"`
 	SpotLabel      string `json:"spotLabel,omitempty"`
-	SpotLabelValue string `json:"spotLabel,omitempty"`
+	SpotLabelValue string `json:"spotLabelValue,omitempty"`
 }
 
 type NodePrice struct {
@@ -116,7 +117,7 @@ func (c *CustomProvider) DownloadPricingData() error {
 		m := make(map[string]*NodePrice)
 		c.Pricing = m
 	}
-	p, err := GetDefaultPricingData()
+	p, err := GetDefaultPricingData("default.json")
 	if err != nil {
 		return err
 	}
@@ -142,18 +143,7 @@ func (*CustomProvider) QuerySQL(query string) ([]byte, error) {
 	return nil, nil
 }
 
-type Node struct {
-	Cost             string `json:"hourlyCost"`
-	VCPU             string `json:"CPU"`
-	VCPUCost         string `json:"CPUHourlyCost"`
-	RAM              string `json:"RAM"`
-	RAMCost          string `json:"RAMGBHourlyCost"`
-	Storage          string `json:"storage"`
-	StorageCost      string `json:"storageHourlyCost"`
-	UsesBaseCPUPrice bool   `json:"usesDefaultPrice"`
-	BaseCPUPrice     string `json:"baseCPUPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
-}
-
+// NewProvider looks at the nodespec or provider metadata server to decide which provider to instantiate.
 func NewProvider(clientset *kubernetes.Clientset, apiKey string) (Provider, error) {
 	if metadata.OnGCE() {
 		if apiKey == "" {
@@ -161,722 +151,29 @@ func NewProvider(clientset *kubernetes.Clientset, apiKey string) (Provider, erro
 		}
 		return &GCP{
 			Clientset: clientset,
-			ApiKey:    apiKey,
+			APIKey:    apiKey,
 		}, nil
-	} else {
-		nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
-		if err != nil {
-			return nil, err
-		}
-		provider := strings.ToLower(nodes.Items[0].Spec.ProviderID)
-		if strings.HasPrefix(provider, "aws") {
-			return &AWS{
-				Clientset: clientset,
-			}, nil
-		} else {
-			log.Printf("Unsupported provider, falling back to default")
-			return &CustomProvider{
-				Clientset: clientset,
-			}, nil
-		}
 	}
-}
-
-type userAgentTransport struct {
-	userAgent string
-	base      http.RoundTripper
-}
-
-func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.Header.Set("User-Agent", t.userAgent)
-	return t.base.RoundTrip(req)
-}
-
-type GCP struct {
-	Pricing      map[string]*GCPPricing
-	Clientset    *kubernetes.Clientset
-	ApiKey       string
-	BaseCPUPrice string
-}
-
-func (*GCP) QuerySQL(query string) ([]byte, error) {
-	return nil, nil
-}
-
-func (*GCP) ClusterName() ([]byte, error) {
-	metadataClient := metadata.NewClient(&http.Client{Transport: userAgentTransport{
-		userAgent: "kubecost",
-		base:      http.DefaultTransport,
-	}})
-
-	attribute, err := metadataClient.InstanceAttributeValue("cluster-name")
+	nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
 	if err != nil {
 		return nil, err
 	}
-
-	m := make(map[string]string)
-	m["name"] = attribute
-	m["provider"] = "GCP"
-	return json.Marshal(m)
-}
-
-func (*GCP) AddServiceKey(formValues url.Values) error {
-	key := formValues.Get("key")
-	k := []byte(key)
-	return ioutil.WriteFile("/var/configs/key.json", k, 0644)
-}
-
-func (*GCP) GetDisks() ([]byte, error) {
-	// metadata API setup
-	metadataClient := metadata.NewClient(&http.Client{Transport: userAgentTransport{
-		userAgent: "kubecost",
-		base:      http.DefaultTransport,
-	}})
-	projID, err := metadataClient.ProjectID()
-	if err != nil {
-		return nil, err
-	}
-
-	client, err := google.DefaultClient(oauth2.NoContext,
-		"https://www.googleapis.com/auth/compute.readonly")
-	if err != nil {
-		return nil, err
-	}
-	svc, err := compute.New(client)
-	if err != nil {
-		return nil, err
-	}
-	res, err := svc.Disks.AggregatedList(projID).Do()
-
-	if err != nil {
-		return nil, err
-	}
-	return json.Marshal(res)
-
-}
-
-type GCPPricing struct {
-	Name                string           `json:"name"`
-	SKUID               string           `json:"skuId"`
-	Description         string           `json:"description"`
-	Category            *GCPResourceInfo `json:"category"`
-	ServiceRegions      []string         `json:"serviceRegions"`
-	PricingInfo         []*PricingInfo   `json:"pricingInfo"`
-	ServiceProviderName string           `json:"serviceProviderName"`
-	Node                *Node            `json:"node"`
-}
-
-type PricingInfo struct {
-	Summary                string             `json:"summary"`
-	PricingExpression      *PricingExpression `json:"pricingExpression"`
-	CurrencyConversionRate int                `json:"currencyConversionRate"`
-	EffectiveTime          string             `json:""`
-}
-
-type PricingExpression struct {
-	UsageUnit                string         `json:"usageUnit"`
-	UsageUnitDescription     string         `json:"usageUnitDescription"`
-	BaseUnit                 string         `json:"baseUnit"`
-	BaseUnitConversionFactor int64          `json:"-"`
-	DisplayQuantity          int            `json:"displayQuantity"`
-	TieredRates              []*TieredRates `json:"tieredRates"`
-}
-
-type TieredRates struct {
-	StartUsageAmount int            `json:"startUsageAmount"`
-	UnitPrice        *UnitPriceInfo `json:"unitPrice"`
-}
-
-type UnitPriceInfo struct {
-	CurrencyCode string  `json:"currencyCode"`
-	Units        string  `json:"units"`
-	Nanos        float64 `json:"nanos"`
-}
-
-type GCPResourceInfo struct {
-	ServiceDisplayName string `json:"serviceDisplayName"`
-	ResourceFamily     string `json:"resourceFamily"`
-	ResourceGroup      string `json:"resourceGroup"`
-	UsageType          string `json:"usageType"`
-}
-
-func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]bool) (map[string]*GCPPricing, string) {
-	gcpPricingList := make(map[string]*GCPPricing)
-	var nextPageToken string
-	dec := json.NewDecoder(r)
-	for {
-		t, err := dec.Token()
-		if err == io.EOF {
-			break
-		}
-		//fmt.Printf("%v  \n", t)
-		if t == "skus" {
-			dec.Token() // [
-			for dec.More() {
-
-				product := &GCPPricing{}
-				err := dec.Decode(&product)
-				if err != nil {
-					fmt.Printf("Error: " + err.Error())
-					break
-				}
-				usageType := strings.ToLower(product.Category.UsageType)
-				instanceType := strings.ToLower(product.Category.ResourceGroup)
-
-				if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "CUSTOM") {
-					instanceType = "custom"
-				}
-				// instance.toLowerCase() === “f1micro”
-				var partialCPU float64
-				if strings.ToLower(instanceType) == "f1micro" {
-					partialCPU = 0.2
-				} else if strings.ToLower(instanceType) == "g1small" {
-					partialCPU = 0.5
-				}
-
-				for _, sr := range product.ServiceRegions {
-					region := sr
-
-					candidateKey := region + "," + instanceType + "," + usageType
-					if _, ok := inputKeys[candidateKey]; ok {
-						lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
-						var nanos float64
-						if len(product.PricingInfo) > 0 {
-							nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
-						} else {
-							continue
-						}
-
-						hourlyPrice := nanos * math.Pow10(-9)
-						if hourlyPrice == 0 {
-							continue
-						} else if strings.Contains(strings.ToUpper(product.Description), "RAM") {
-							if instanceType == "custom" {
-								log.Printf("RAM custom sku is: " + product.Name)
-							}
-							if _, ok := gcpPricingList[candidateKey]; ok {
-								gcpPricingList[candidateKey].Node.RAMCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
-							} else {
-								product.Node = &Node{
-									RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
-								}
-								if partialCPU != 0 {
-									product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
-								}
-								gcpPricingList[candidateKey] = product
-							}
-							break
-						} else {
-							if _, ok := gcpPricingList[candidateKey]; ok {
-								gcpPricingList[candidateKey].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
-							} else {
-								product.Node = &Node{
-									VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
-								}
-								if partialCPU != 0 {
-									product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
-								}
-								gcpPricingList[candidateKey] = product
-							}
-							break
-						}
-					}
-				}
-			}
-		}
-		if t == "nextPageToken" {
-			pageToken, err := dec.Token()
-			if err != nil {
-				log.Printf("Error parsing nextpage token: " + err.Error())
-				break
-			}
-			if pageToken.(string) != "" {
-				nextPageToken = pageToken.(string)
-			} else {
-				nextPageToken = "done"
-			}
-		}
-	}
-	return gcpPricingList, nextPageToken
-}
-
-func (gcp *GCP) parsePages(inputKeys map[string]bool) (map[string]*GCPPricing, error) {
-	var pages []map[string]*GCPPricing
-	url := "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus?key=" + gcp.ApiKey //AIzaSyDXQPG_MHUEy9neR7stolq6l0ujXmjJlvk
-	log.Printf("URL: %s", url)
-	var parsePagesHelper func(string) error
-	parsePagesHelper = func(pageToken string) error {
-		if pageToken == "done" {
-			return nil
-		} else if pageToken != "" {
-			url = url + "&pageToken=" + pageToken
-		}
-		resp, err := http.Get(url)
-		if err != nil {
-			return err
-		}
-		page, token := gcp.parsePage(resp.Body, inputKeys)
-		pages = append(pages, page)
-		return parsePagesHelper(token)
-	}
-	err := parsePagesHelper("")
-	returnPages := make(map[string]*GCPPricing)
-	for _, page := range pages {
-		for k, v := range page {
-			if val, ok := returnPages[k]; ok { //keys may need to be merged
-				if val.Node.RAMCost != "" && val.Node.VCPUCost == "" {
-					val.Node.VCPUCost = v.Node.VCPUCost
-				} else if val.Node.VCPUCost != "" && val.Node.RAMCost == "" {
-					val.Node.RAMCost = v.Node.RAMCost
-				} else {
-					returnPages[k] = v
-				}
-			} else {
-				returnPages[k] = v
-			}
-		}
-	}
-	return returnPages, err
-}
-
-func (gcp *GCP) DownloadPricingData() error {
-
-	nodeList, err := gcp.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
-	if err != nil {
-		return err
-	}
-	inputkeys := make(map[string]bool)
-
-	for _, n := range nodeList.Items {
-		labels := n.GetObjectMeta().GetLabels()
-		key := gcp.GetKey(labels)
-		inputkeys[key] = true
-	}
-
-	pages, err := gcp.parsePages(inputkeys)
-
-	if err != nil {
-		return err
-	}
-	gcp.Pricing = pages
-	c, err := GetDefaultPricingData()
-	if err != nil {
-		log.Printf("Error downloading default pricing data: %s", err.Error())
-	}
-	gcp.BaseCPUPrice = c.CPU
-
-	return nil
-}
-
-func (gcp *GCP) GetKey(labels map[string]string) string {
-
-	instanceType := strings.ToLower(strings.Join(strings.Split(labels["beta.kubernetes.io/instance-type"], "-")[:2], ""))
-	if instanceType == "n1highmem" || instanceType == "n1highcpu" {
-		instanceType = "n1standard" // These are priced the same. TODO: support n1ultrahighmem
-	} else if strings.HasPrefix(instanceType, "custom") {
-		instanceType = "custom" // The suffix of custom does not matter
-	}
-	region := strings.ToLower(labels["failure-domain.beta.kubernetes.io/region"])
-	var usageType string
-	if t, ok := labels["cloud.google.com/gke-preemptible"]; ok && t == "true" {
-		usageType = "preemptible"
-	} else {
-		usageType = "ondemand"
-	}
-	return region + "," + instanceType + "," + usageType
-}
-
-func (gcp *GCP) AllNodePricing() (interface{}, error) {
-	return gcp.Pricing, nil
-}
-
-func (gcp *GCP) NodePricing(key string) (*Node, error) {
-	if n, ok := gcp.Pricing[key]; ok {
-		log.Printf("Returning pricing for node %s: %+v from SKU %s", key, n.Node, n.Name)
-		n.Node.BaseCPUPrice = gcp.BaseCPUPrice
-		return n.Node, nil
-	} else {
-		log.Printf("Warning: no pricing data found for %s", key)
-		return nil, fmt.Errorf("Warning: no pricing data found for %s", key)
-	}
-}
-
-type AWS struct {
-	Pricing          map[string]*AWSProductTerms
-	ValidPricingKeys map[string]bool
-	Clientset        *kubernetes.Clientset
-	BaseCPUPrice     string
-}
-
-type AWSPricing struct {
-	Products map[string]*AWSProduct `json:"products"`
-	Terms    AWSPricingTerms        `json:"terms"`
-}
-
-type AWSProduct struct {
-	Sku        string               `json:"sku"`
-	Attributes AWSProductAttributes `json:"attributes"`
-}
-
-type AWSProductAttributes struct {
-	Location        string `json:"location"`
-	InstanceType    string `json:"instanceType"`
-	Memory          string `json:"memory"`
-	Storage         string `json:"storage"`
-	VCpu            string `json:"vcpu"`
-	UsageType       string `json:"usagetype"`
-	OperatingSystem string `json:"operatingSystem"`
-	PreInstalledSw  string `json:"preInstalledSw"`
-}
-
-type AWSPricingTerms struct {
-	OnDemand map[string]map[string]*AWSOfferTerm `json:"OnDemand"`
-	Reserved map[string]map[string]*AWSOfferTerm `json:"Reserved"`
-}
-
-type AWSOfferTerm struct {
-	Sku             string                  `json:"sku"`
-	PriceDimensions map[string]*AWSRateCode `json:"priceDimensions"`
-}
-
-type AWSRateCode struct {
-	Unit         string          `json:"unit"`
-	PricePerUnit AWSCurrencyCode `json:"pricePerUnit"`
-}
-
-type AWSCurrencyCode struct {
-	USD string `json:"USD"`
-}
-
-type AWSProductTerms struct {
-	Sku      string        `json:"sku"`
-	OnDemand *AWSOfferTerm `json:"OnDemand"`
-	Reserved *AWSOfferTerm `json:"Reserved"`
-	Memory   string        `json:"memory"`
-	Storage  string        `json:"storage"`
-	VCpu     string        `json:"vcpu"`
-}
-
-const OnDemandRateCode = ".JRTCKXETXF"
-const ReservedRateCode = ".38NPMPTW36"
-const HourlyRateCode = ".6YS6EN2CT7"
-
-func (aws *AWS) KubeAttrConversion(location, instanceType, operatingSystem string) string {
-	locationToRegion := map[string]string{
-		"US East (Ohio)":             "us-east-2",
-		"US East (N. Virginia)":      "us-east-1",
-		"US West (N. California)":    "us-west-1",
-		"US West (Oregon)":           "us-west-2",
-		"Asia Pacific (Mumbai)":      "ap-south-1",
-		"Asia Pacific (Osaka-Local)": "ap-northeast-3",
-		"Asia Pacific (Seoul)":       "ap-northeast-2",
-		"Asia Pacific (Singapore)":   "ap-southeast-1",
-		"Asia Pacific (Sydney)":      "ap-southeast-2",
-		"Asia Pacific (Tokyo)":       "ap-northeast-1",
-		"Canada (Central)":           "ca-central-1",
-		"China (Beijing)":            "cn-north-1",
-		"China (Ningxia)":            "cn-northwest-1",
-		"EU (Frankfurt)":             "eu-central-1",
-		"EU (Ireland)":               "eu-west-1",
-		"EU (London)":                "eu-west-2",
-		"EU (Paris)":                 "eu-west-3",
-		"EU (Stockholm)":             "eu-north-1",
-		"South America (São Paulo)":  "sa-east-1",
-		"AWS GovCloud (US-East)":     "us-gov-east-1",
-		"AWS GovCloud (US)":          "us-gov-west-1",
-	}
-
-	operatingSystem = strings.ToLower(operatingSystem)
-
-	region := locationToRegion[location]
-	return region + "," + instanceType + "," + operatingSystem
-}
-
-func (aws *AWS) GetKey(labels map[string]string) string {
-	instanceType := labels["beta.kubernetes.io/instance-type"]
-	operatingSystem := labels["beta.kubernetes.io/os"]
-	region := labels["failure-domain.beta.kubernetes.io/region"]
-	return region + "," + instanceType + "," + operatingSystem
-}
-
-func (aws *AWS) DownloadPricingData() error {
-
-	nodeList, err := aws.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
-	if err != nil {
-		return err
-	}
-	inputkeys := make(map[string]bool)
-
-	for _, n := range nodeList.Items {
-		labels := n.GetObjectMeta().GetLabels()
-
-		key := aws.GetKey(labels)
-		inputkeys[key] = true
-	}
-
-	aws.Pricing = make(map[string]*AWSProductTerms)
-	aws.ValidPricingKeys = make(map[string]bool)
-	skusToKeys := make(map[string]string)
-
-	resp, err := http.Get("https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json")
-	if err != nil {
-		return err
-	}
-
-	dec := json.NewDecoder(resp.Body)
-	for {
-		t, err := dec.Token()
-		if err == io.EOF {
-			fmt.Printf("done \n")
-			break
-		}
-		if t == "products" {
-			dec.Token() //{
-			for dec.More() {
-				dec.Token() // the sku token
-				product := &AWSProduct{}
-				err := dec.Decode(&product)
-
-				if err != nil {
-					fmt.Printf("Error: " + err.Error())
-					break
-				}
-				if product.Attributes.PreInstalledSw == "NA" &&
-					(strings.HasPrefix(product.Attributes.UsageType, "BoxUsage") || strings.Contains(product.Attributes.UsageType, "-BoxUsage")) {
-					key := aws.KubeAttrConversion(product.Attributes.Location, product.Attributes.InstanceType, product.Attributes.OperatingSystem)
-					if inputkeys[key] {
-						aws.Pricing[key] = &AWSProductTerms{
-							Sku:     product.Sku,
-							Memory:  product.Attributes.Memory,
-							Storage: product.Attributes.Storage,
-							VCpu:    product.Attributes.VCpu,
-						}
-						skusToKeys[product.Sku] = key
-					}
-					aws.ValidPricingKeys[key] = true
-				}
-			}
-		}
-		if t == "terms" {
-			dec.Token()
-			termType, _ := dec.Token()
-			if termType == "OnDemand" {
-				dec.Token() // {
-				for dec.More() {
-					sku, _ := dec.Token()
-					dec.Token()
-					skuOnDemand, _ := dec.Token()
-					offerTerm := &AWSOfferTerm{}
-					err := dec.Decode(&offerTerm)
-					if err != nil {
-						fmt.Printf("Error: " + err.Error())
-					}
-					if sku.(string)+OnDemandRateCode == skuOnDemand {
-						key, ok := skusToKeys[sku.(string)]
-						if ok {
-							aws.Pricing[key].OnDemand = offerTerm
-						}
-					}
-					dec.Token()
-				}
-				dec.Token()
-			}
-		}
-	}
-
-	if err != nil {
-		return err
-	}
-	c, err := GetDefaultPricingData()
-	if err != nil {
-		log.Printf("Error downloading default pricing data: %s", err.Error())
-	}
-	aws.BaseCPUPrice = c.CPU
-	return nil
-}
-
-func (aws *AWS) AllNodePricing() (interface{}, error) {
-	return aws.Pricing, nil
-}
-
-func (aws *AWS) NodePricing(key string) (*Node, error) {
-	//return json.Marshal(aws.Pricing[key])
-	terms, ok := aws.Pricing[key]
-	if ok {
-		cost := terms.OnDemand.PriceDimensions[terms.Sku+OnDemandRateCode+HourlyRateCode].PricePerUnit.USD
-		return &Node{
-			Cost:    cost,
-			VCPU:    terms.VCpu,
-			RAM:     terms.Memory,
-			Storage: terms.Storage,
+	provider := strings.ToLower(nodes.Items[0].Spec.ProviderID)
+	if strings.HasPrefix(provider, "aws") {
+		return &AWS{
+			Clientset: clientset,
 		}, nil
-	} else if _, ok := aws.ValidPricingKeys[key]; ok {
-		err := aws.DownloadPricingData()
-		if err != nil {
-			return nil, err
-		}
-		terms := aws.Pricing[key]
-		cost := terms.OnDemand.PriceDimensions[terms.Sku+OnDemandRateCode+HourlyRateCode].PricePerUnit.USD
-		return &Node{
-			Cost:         cost,
-			VCPU:         terms.VCpu,
-			RAM:          terms.Memory,
-			Storage:      terms.Storage,
-			BaseCPUPrice: aws.BaseCPUPrice,
+	} else if strings.HasPrefix(provider, "azure") {
+		return &Azure{
+			CustomProvider: &CustomProvider{
+				Clientset: clientset,
+			},
 		}, nil
 	} else {
-		return nil, errors.New("Invalid Pricing Key: " + key + "\n")
-	}
-}
-
-func (*AWS) ClusterName() ([]byte, error) {
-
-	attribute := "AWS Cluster #1"
-
-	m := make(map[string]string)
-	m["name"] = attribute
-	m["provider"] = "AWS"
-	return json.Marshal(m)
-}
-
-func (*AWS) AddServiceKey(formValues url.Values) error {
-	keyID := formValues.Get("access_key_ID")
-	key := formValues.Get("secret_access_key")
-	m := make(map[string]string)
-	m["access_key_ID"] = keyID
-	m["secret_access_key"] = key
-	json, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile("/var/configs/key.json", json, 0644)
-}
-
-func (*AWS) GetDisks() ([]byte, error) {
-	jsonFile, err := os.Open("/var/configs/key.json")
-	if err == nil {
-		byteValue, _ := ioutil.ReadAll(jsonFile)
-		var result map[string]string
-		json.Unmarshal([]byte(byteValue), &result)
-		os.Setenv("AWS_ACCESS_KEY_ID", result["access_key_ID"])
-		os.Setenv("AWS_SECRET_ACCESS_KEY", result["secret_access_key"])
-	} else if os.IsNotExist(err) {
-		log.Printf("Using Default Credentials")
-	} else {
-		return nil, err
-	}
-	defer jsonFile.Close()
-	byteValue, _ := ioutil.ReadAll(jsonFile)
-	var result map[string]string
-	json.Unmarshal([]byte(byteValue), &result)
-	os.Setenv("AWS_ACCESS_KEY_ID", result["access_key_ID"])
-	os.Setenv("AWS_SECRET_ACCESS_KEY", result["secret_access_key"])
-	c := &aws.Config{
-		Region:      aws.String("us-east-1"),
-		Credentials: credentials.NewEnvCredentials(),
-	}
-	s := session.Must(session.NewSession(c))
-
-	ec2Svc := ec2.New(s)
-	input := &ec2.DescribeVolumesInput{}
-	volumeResult, err := ec2Svc.DescribeVolumes(input)
-	if err != nil {
-		if aerr, ok := err.(awserr.Error); ok {
-			switch aerr.Code() {
-			default:
-				return nil, aerr
-			}
-		} else {
-			return nil, err
-		}
-	}
-	return json.Marshal(volumeResult)
-}
-
-func (*AWS) QuerySQL(query string) ([]byte, error) {
-	jsonFile, err := os.Open("/var/configs/key.json")
-	if err == nil {
-		byteValue, _ := ioutil.ReadAll(jsonFile)
-		var result map[string]string
-		json.Unmarshal([]byte(byteValue), &result)
-		os.Setenv("AWS_ACCESS_KEY_ID", result["access_key_ID"])
-		os.Setenv("AWS_SECRET_ACCESS_KEY", result["secret_access_key"])
-	} else if os.IsNotExist(err) {
-		log.Printf("Using Default Credentials")
-	} else {
-		return nil, err
-	}
-	defer jsonFile.Close()
-	athenaConfigs, err := os.Open("/var/configs/athena.json")
-	if err != nil {
-		return nil, err
-	}
-	defer athenaConfigs.Close()
-	bytes, _ := ioutil.ReadAll(athenaConfigs)
-	var athenaConf map[string]string
-	json.Unmarshal([]byte(bytes), &athenaConf)
-	region := aws.String(athenaConf["region"])
-	resultsBucket := athenaConf["output"]
-	database := athenaConf["database"]
-
-	c := &aws.Config{
-		Region:      region,
-		Credentials: credentials.NewEnvCredentials(),
-	}
-	s := session.Must(session.NewSession(c))
-	svc := athena.New(s)
-
-	var e athena.StartQueryExecutionInput
-
-	var r athena.ResultConfiguration
-	r.SetOutputLocation(resultsBucket)
-	e.SetResultConfiguration(&r)
-
-	e.SetQueryString(query)
-	var q athena.QueryExecutionContext
-	q.SetDatabase(database)
-	e.SetQueryExecutionContext(&q)
-
-	res, err := svc.StartQueryExecution(&e)
-	if err != nil {
-		return nil, err
-	}
-
-	fmt.Println("StartQueryExecution result:")
-	fmt.Println(res.GoString())
-
-	var qri athena.GetQueryExecutionInput
-	qri.SetQueryExecutionId(*res.QueryExecutionId)
-
-	var qrop *athena.GetQueryExecutionOutput
-	duration := time.Duration(2) * time.Second // Pause for 2 seconds
-
-	for {
-		qrop, err = svc.GetQueryExecution(&qri)
-		if err != nil {
-			return nil, err
-		}
-		if *qrop.QueryExecution.Status.State != "RUNNING" {
-			break
-		}
-		time.Sleep(duration)
+		log.Printf("Unsupported provider, falling back to default")
+		return &CustomProvider{
+			Clientset: clientset,
+		}, nil
 	}
-	if *qrop.QueryExecution.Status.State == "SUCCEEDED" {
-
-		var ip athena.GetQueryResultsInput
-		ip.SetQueryExecutionId(*res.QueryExecutionId)
 
-		op, err := svc.GetQueryResults(&ip)
-		if err != nil {
-			return nil, err
-		}
-		bytes, err := json.Marshal(op.ResultSet)
-		if err != nil {
-			return nil, err
-		}
-
-		return bytes, nil
-	} else {
-		return nil, fmt.Errorf("Error getting query results : %s", *qrop.QueryExecution.Status.State)
-	}
 }

costmodel/costmodel.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"math"
 	"net/http"
+	"sort"
 	"strconv"
 	"time"
 
@@ -197,13 +198,41 @@ func getContainerAllocation(req []*Vector, used []*Vector) []*Vector {
 		return req
 	}
 	var allocation []*Vector
-	for i, reqV := range req {
-		usedV := used[i]
-		allocation = append(allocation, &Vector{
-			Timestamp: usedV.Timestamp,
-			Value:     math.Max(usedV.Value, reqV.Value),
-		})
+
+	var timestamps []float64
+	reqMap := make(map[float64]float64)
+	for _, reqV := range req {
+		if reqV.Timestamp == 0 {
+			continue
+		}
+		reqV.Timestamp = math.Round(reqV.Timestamp/10) * 10
+		reqMap[reqV.Timestamp] = reqV.Value
+		timestamps = append(timestamps, reqV.Timestamp)
+	}
+	usedMap := make(map[float64]float64)
+	for _, usedV := range used {
+		usedV.Timestamp = math.Round(usedV.Timestamp/10) * 10
+		usedMap[usedV.Timestamp] = usedV.Value
+		timestamps = append(timestamps, usedV.Timestamp)
+	}
+
+	sort.Float64s(timestamps)
+	for _, t := range timestamps {
+		rv, okR := reqMap[t]
+		uv, okU := usedMap[t]
+		allocationVector := &Vector{
+			Timestamp: t,
+		}
+		if okR && okU {
+			allocationVector.Value = math.Max(rv, uv)
+		} else if okR {
+			allocationVector.Value = rv
+		} else if okU {
+			allocationVector.Value = uv
+		}
+		allocation = append(allocation, allocationVector)
 	}
+
 	return allocation
 }
 
@@ -229,20 +258,13 @@ func getNodeCost(clientset *kubernetes.Clientset, cloud costAnalyzerCloud.Provid
 			cpu, _ = strconv.ParseFloat(cnode.VCPU, 64)
 		}
 		var ram float64
-		log.Printf("CNODE RAM : %s", cnode.RAM)
 		if cnode.RAM == "" {
-			log.Printf("RAMSTRING: %s", n.Status.Capacity.Memory().String())
 			cnode.RAM = n.Status.Capacity.Memory().String()
-			ram = float64(n.Status.Capacity.Memory().Value())
-		} else {
-			ram, _ = strconv.ParseFloat(cnode.RAM, 64)
 		}
-		log.Printf("RAM USAGE: %f", ram)
+		ram = float64(n.Status.Capacity.Memory().Value())
 		if cnode.RAMCost == "" { // We couldn't find a ramcost, so fix cpu and allocate ram accordingly
 			basePrice, _ := strconv.ParseFloat(cnode.BaseCPUPrice, 64)
-			log.Printf("BASEPRICE: %f", basePrice)
 			totalCPUPrice := basePrice * cpu
-			log.Printf("TOTALCPUPRICE: %f", basePrice)
 			var nodePrice float64
 			if cnode.Cost != "" {
 				log.Printf("Use given nodeprice as whole node price")
@@ -251,11 +273,13 @@ func getNodeCost(clientset *kubernetes.Clientset, cloud costAnalyzerCloud.Provid
 				log.Printf("Use cpuprice as whole node price")
 				nodePrice, _ = strconv.ParseFloat(cnode.VCPUCost, 64) // all the price was allocated the the CPU
 			}
-			log.Printf("NODEPRICE: %f", basePrice)
-			ramPrice := (nodePrice - totalCPUPrice) / (ram / 1024 / 1024 / 1024)
-			if ramPrice < 0 {
-				ramPrice = 0
+			if totalCPUPrice >= nodePrice {
+				totalCPUPrice = 0.9 * nodePrice // just allocate RAM costs to 10% of the node price here to avoid 0 or negative in the numerator
 			}
+			ramPrice := (nodePrice - totalCPUPrice) / (ram / 1024 / 1024 / 1024)
+			cpuPrice := totalCPUPrice / cpu
+
+			cnode.VCPUCost = fmt.Sprintf("%f", cpuPrice)
 			cnode.RAMCost = fmt.Sprintf("%f", ramPrice)
 			log.Printf(cnode.RAMCost)
 		}

kubernetes/deployment.yaml

@@ -29,5 +29,5 @@ spec:
             - name: PROMETHEUS_SERVER_ENDPOINT
               value: <add a prometheus server endpoint> # kube-state-metrics and prometheus must be installed.
             - name: CLOUD_PROVIDER_API_KEY
-              value: "AIzaSyDXQPG_MHUEy9neR7stolq6l0ujXmjJlvk" # The GCP Pricing API requires a key.
+              value: "AIzaSyD29bGxmHAVEOBYtgd8sYM2gM2ekfxQX4U" # The GCP Pricing API requires a key.
           imagePullPolicy: Always

main.go

@@ -48,6 +48,7 @@ func wrapData(data interface{}, err error) []byte {
 	return resp
 }
 
+// RefreshPricingData needs to be called when a new node joins the fleet, since we cache the relevant subsets of pricing data to avoid storing the whole thing.
 func (a *Accesses) RefreshPricingData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")