Merge pull request #21 from mdaniel/issue-12-support

Issue 12 support

cloud/awsprovider.go

@@ -2,7 +2,6 @@ package cloud
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -169,8 +168,11 @@ func (aws *AWS) DownloadPricingData() error {
 	aws.ValidPricingKeys = make(map[string]bool)
 	skusToKeys := make(map[string]string)
 
-	resp, err := http.Get("https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json")
+	pricingURL := "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json"
+	log.Printf("starting download of \"%s\", which is quite large ...", pricingURL)
+	resp, err := http.Get(pricingURL)
 	if err != nil {
+		log.Printf("Bogus fetch of \"%s\": %v", pricingURL, err)
 		return err
 	}
 
@@ -178,7 +180,7 @@ func (aws *AWS) DownloadPricingData() error {
 	for {
 		t, err := dec.Token()
 		if err == io.EOF {
-			fmt.Printf("done \n")
+			log.Printf("done loading \"%s\"\n", pricingURL)
 			break
 		}
 		if t == "products" {
@@ -189,7 +191,7 @@ func (aws *AWS) DownloadPricingData() error {
 				err := dec.Decode(&product)
 
 				if err != nil {
-					fmt.Printf("Error: " + err.Error())
+					log.Printf("Error parsing response from \"%s\": %v", pricingURL, err.Error())
 					break
 				}
 				if product.Attributes.PreInstalledSw == "NA" &&
@@ -222,7 +224,7 @@ func (aws *AWS) DownloadPricingData() error {
 					offerTerm := &AWSOfferTerm{}
 					err := dec.Decode(&offerTerm)
 					if err != nil {
-						fmt.Printf("Error: " + err.Error())
+						log.Printf("Error decoding AWS Offer Term: " + err.Error())
 					}
 					if sku.(string)+OnDemandRateCode == skuOnDemand {
 						key, ok := skusToKeys[sku.(string)]
@@ -315,7 +317,7 @@ func (aws *AWS) NodePricing(key string) (*Node, error) {
 			UsageType:    usageType,
 		}, nil
 	} else {
-		return nil, errors.New("Invalid Pricing Key: " + key + "\n")
+		return nil, fmt.Errorf("Invalid Pricing Key \"%s\"", key)
 	}
 }
 
@@ -337,11 +339,11 @@ func (*AWS) AddServiceKey(formValues url.Values) error {
 	m := make(map[string]string)
 	m["access_key_ID"] = keyID
 	m["secret_access_key"] = key
-	json, err := json.Marshal(m)
+	result, err := json.Marshal(m)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile("/var/configs/key.json", json, 0644)
+	return ioutil.WriteFile("/var/configs/key.json", result, 0644)
 }
 
 // GetDisks returns the AWS disks backing PVs. Useful because sometimes k8s will not clean up PVs correctly. Requires a json config in /var/configs with key region.
@@ -354,7 +356,7 @@ func (*AWS) GetDisks() ([]byte, error) {
 		os.Setenv("AWS_ACCESS_KEY_ID", result["access_key_ID"])
 		os.Setenv("AWS_SECRET_ACCESS_KEY", result["secret_access_key"])
 	} else if os.IsNotExist(err) {
-		log.Printf("Using Default Credentials")
+		log.Print("Using Default Credentials")
 	} else {
 		return nil, err
 	}
@@ -402,7 +404,7 @@ func (*AWS) QuerySQL(query string) ([]byte, error) {
 		os.Setenv("AWS_ACCESS_KEY_ID", result["access_key_ID"])
 		os.Setenv("AWS_SECRET_ACCESS_KEY", result["secret_access_key"])
 	} else if os.IsNotExist(err) {
-		log.Printf("Using Default Credentials")
+		log.Print("Using Default Credentials")
 	} else {
 		return nil, err
 	}
@@ -442,8 +444,8 @@ func (*AWS) QuerySQL(query string) ([]byte, error) {
 		return nil, err
 	}
 
-	fmt.Println("StartQueryExecution result:")
-	fmt.Println(res.GoString())
+	log.Println("StartQueryExecution result:")
+	log.Println(res.GoString())
 
 	var qri athena.GetQueryExecutionInput
 	qri.SetQueryExecutionId(*res.QueryExecutionId)

cloud/provider.go

@@ -2,7 +2,7 @@ package cloud
 
 import (
 	"encoding/json"
-	"fmt"
+	"errors"
 	"io/ioutil"
 	"log"
 	"net/url"
@@ -147,8 +147,9 @@ func (*CustomProvider) QuerySQL(query string) ([]byte, error) {
 // NewProvider looks at the nodespec or provider metadata server to decide which provider to instantiate.
 func NewProvider(clientset *kubernetes.Clientset, apiKey string) (Provider, error) {
 	if metadata.OnGCE() {
+		log.Print("metadata reports we are in GCE")
 		if apiKey == "" {
-			return nil, fmt.Errorf("Supply a GCP Key to start getting data")
+			return nil, errors.New("Supply a GCP Key to start getting data")
 		}
 		return &GCP{
 			Clientset: clientset,
@@ -161,17 +162,19 @@ func NewProvider(clientset *kubernetes.Clientset, apiKey string) (Provider, erro
 	}
 	provider := strings.ToLower(nodes.Items[0].Spec.ProviderID)
 	if strings.HasPrefix(provider, "aws") {
+		log.Print("Found ProviderID starting with \"aws\", using AWS Provider")
 		return &AWS{
 			Clientset: clientset,
 		}, nil
 	} else if strings.HasPrefix(provider, "azure") {
+		log.Print("Found ProviderID starting with \"azure\", using Azure Provider")
 		return &Azure{
 			CustomProvider: &CustomProvider{
 				Clientset: clientset,
 			},
 		}, nil
 	} else {
-		log.Printf("Unsupported provider, falling back to default")
+		log.Print("Unsupported provider, falling back to default")
 		return &CustomProvider{
 			Clientset: clientset,
 		}, nil

costmodel/costmodel.go

@@ -13,7 +13,7 @@ import (
 
 	costAnalyzerCloud "github.com/kubecost/cost-model/cloud"
 	prometheusClient "github.com/prometheus/client_golang/api"
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
@@ -112,7 +112,7 @@ func ComputeCostData(cli prometheusClient.Client, clientset *kubernetes.Clientse
 	for _, pod := range podlist.Items {
 		podName := pod.GetObjectMeta().GetName()
 		ns := pod.GetObjectMeta().GetNamespace()
-		labels := pod.GetObjectMeta().GetLabels()
+		podLabels := pod.GetObjectMeta().GetLabels()
 		nodeName := pod.Spec.NodeName
 		var nodeData *costAnalyzerCloud.Node
 		if _, ok := nodes[nodeName]; ok {
@@ -181,7 +181,7 @@ func ComputeCostData(cli prometheusClient.Client, clientset *kubernetes.Clientse
 				CPUUsed:      []*Vector{findContainerMetric(resultCPUUsage, containerName, podName, ns)},
 				GPUReq:       []*Vector{GPUReqV},
 				PVData:       pvReq,
-				Labels:       labels,
+				Labels:       podLabels,
 			}
 			costs.CPUAllocation = getContainerAllocation(costs.CPUReq, costs.CPUUsed)
 			costs.RAMAllocation = getContainerAllocation(costs.RAMReq, costs.RAMUsed)
@@ -245,8 +245,8 @@ func getNodeCost(clientset *kubernetes.Clientset, cloud costAnalyzerCloud.Provid
 	nodes := make(map[string]*costAnalyzerCloud.Node)
 	for _, n := range nodeList.Items {
 		name := n.GetObjectMeta().GetName()
-		labels := n.GetObjectMeta().GetLabels()
-		cnode, err := cloud.NodePricing(cloud.GetKey(labels))
+		nodeLabels := n.GetObjectMeta().GetLabels()
+		cnode, err := cloud.NodePricing(cloud.GetKey(nodeLabels))
 		if err != nil {
 			log.Printf("Error getting node. Error: " + err.Error())
 			continue
@@ -269,10 +269,10 @@ func getNodeCost(clientset *kubernetes.Clientset, cloud costAnalyzerCloud.Provid
 			totalCPUPrice := basePrice * cpu
 			var nodePrice float64
 			if cnode.Cost != "" {
-				log.Printf("Use given nodeprice as whole node price")
+				log.Print("Use given nodeprice as whole node price")
 				nodePrice, _ = strconv.ParseFloat(cnode.Cost, 64)
 			} else {
-				log.Printf("Use cpuprice as whole node price")
+				log.Print("Use cpuprice as whole node price")
 				nodePrice, _ = strconv.ParseFloat(cnode.VCPUCost, 64) // all the price was allocated the the CPU
 			}
 			if totalCPUPrice >= nodePrice {
@@ -284,7 +284,7 @@ func getNodeCost(clientset *kubernetes.Clientset, cloud costAnalyzerCloud.Provid
 			cnode.VCPUCost = fmt.Sprintf("%f", cpuPrice)
 			cnode.RAMCost = fmt.Sprintf("%f", ramPrice)
 			cnode.RAMBytes = fmt.Sprintf("%f", ram)
-			log.Printf(cnode.RAMCost)
+			log.Printf("Node \"%s\" RAM Cost := %v", name, cnode.RAMCost)
 		}
 		nodes[name] = cnode
 	}
@@ -292,7 +292,6 @@ func getNodeCost(clientset *kubernetes.Clientset, cloud costAnalyzerCloud.Provid
 }
 
 func getPodServices(clientset *kubernetes.Clientset, podList *v1.PodList) (map[string]map[string][]string, error) {
-	//servicesList, err := clientset.Core().Services("").List(metav1.ListOptions{})
 	servicesList, err := clientset.CoreV1().Services("").List(metav1.ListOptions{})
 	if err != nil {
 		return nil, err
@@ -423,7 +422,7 @@ func ComputeCostDataRange(cli prometheusClient.Client, clientset *kubernetes.Cli
 	for _, pod := range podlist.Items {
 		podName := pod.GetObjectMeta().GetName()
 		ns := pod.GetObjectMeta().GetNamespace()
-		labels := pod.GetObjectMeta().GetLabels()
+		podLabels := pod.GetObjectMeta().GetLabels()
 		nodeName := pod.Spec.NodeName
 		var nodeData *costAnalyzerCloud.Node
 		if _, ok := nodes[nodeName]; ok {
@@ -502,7 +501,7 @@ func ComputeCostDataRange(cli prometheusClient.Client, clientset *kubernetes.Cli
 				CPUUsed:      findContainerMetricVectors(resultCPUUsage, containerName, podName, ns),
 				GPUReq:       GPUReqV,
 				PVData:       pvReq,
-				Labels:       labels,
+				Labels:       podLabels,
 			}
 			costs.RAMAllocation = getContainerAllocation(costs.RAMReq, costs.RAMUsed)
 			costs.CPUAllocation = getContainerAllocation(costs.CPUReq, costs.CPUUsed)