first pass of GCP external cost data

Dockerfile

@@ -33,4 +33,5 @@ COPY --from=build-env /go/bin/app /go/bin/app
 ADD ./cloud/default.json /models/default.json
 ADD ./cloud/azure.json /models/azure.json
 ADD ./cloud/aws.json /models/aws.json
+ADD ./cloud/gcp.json /models/gcp.json
 ENTRYPOINT ["/go/bin/app"]

cloud/aws.json

@@ -12,5 +12,5 @@
     "awsSpotDataRegion":"us-east-2",
     "awsSpotDataBucket": "kc-test-spot",
     "awsSpotDataPrefix": "spotdata",
-    "awsProjectID": "530337586275"
+    "projectID": "530337586275"
 }

cloud/awsprovider.go

@@ -169,7 +169,7 @@ func (k *awsKey) ID() string {
 			return group
 		}
 	}
-	klog.V(3).Info("Could not find instance ID in \"%s\"", k.ProviderID)
+	klog.V(3).Infof("Could not find instance ID in \"%s\"", k.ProviderID)
 	return ""
 }
 
@@ -560,6 +560,10 @@ func (*AWS) GetDisks() ([]byte, error) {
 	return json.Marshal(volumeResult)
 }
 
+func (*AWS) ExternalAllocations(start string, end string) ([]*OutOfClusterAllocation, error) {
+	return nil, nil // TODO: transform the QuerySQL lines into the new OutOfClusterAllocation Struct
+}
+
 // QuerySQL can query a properly configured Athena database.
 // Used to fetch billing data.
 // Requires a json config in /var/configs with key region, output, and database.

cloud/gcpprovider.go

@@ -19,6 +19,7 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	compute "google.golang.org/api/compute/v1"
+	"google.golang.org/api/iterator"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -36,26 +37,87 @@ func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error)
 
 // GCP implements a provider interface for GCP
 type GCP struct {
-	Pricing      map[string]*GCPPricing
-	Clientset    *kubernetes.Clientset
-	APIKey       string
-	BaseCPUPrice string
+	Pricing            map[string]*GCPPricing
+	Clientset          *kubernetes.Clientset
+	APIKey             string
+	BaseCPUPrice       string
+	ProjectID          string
+	BillingDataDataset string
 }
 
-// QuerySQL should query BigQuery for billing data for out of cluster costs. TODO: Implement.
-func (*GCP) QuerySQL(query string) ([]byte, error) {
+type gcpAllocation struct {
+	Aggregator  bigquery.NullString
+	Environment bigquery.NullString
+	Service     string
+	Cost        float64
+}
+
+func gcpAllocationToOutOfClusterAllocation(gcpAlloc gcpAllocation) *OutOfClusterAllocation {
+	var aggregator string
+	if gcpAlloc.Aggregator.Valid {
+		aggregator = gcpAlloc.Aggregator.StringVal
+	}
+
+	var environment string
+	if gcpAlloc.Environment.Valid {
+		environment = gcpAlloc.Environment.StringVal
+	}
+
+	return &OutOfClusterAllocation{
+		Aggregator:  aggregator,
+		Environment: environment,
+		Service:     gcpAlloc.Service,
+		Cost:        gcpAlloc.Cost,
+	}
+}
+
+func (gcp *GCP) ExternalAllocations(start string, end string) ([]*OutOfClusterAllocation, error) {
+	// start, end formatted like: "2019-04-20 00:00:00"
+	queryString := fmt.Sprintf(`SELECT
+					service,
+					labels.key as aggregator,
+					labels.value as environment,
+					SUM(cost) as cost
+				FROM  (SELECT 
+							service.description as service,
+							labels,
+							cost 
+						FROM %s
+						WHERE usage_start_time >= "%s" AND usage_start_time < "%s")
+						LEFT JOIN UNNEST(labels) as labels
+						ON labels.key = "namespace" OR labels.key = "container"
+				GROUP BY aggregator, environment, service;`, gcp.BillingDataDataset, start, end) // For example, "billing_data.gcp_billing_export_v1_01AC9F_74CF1D_5565A2"
+	klog.V(3).Infof("HERE IS THE PROJECT ID: %s", gcp.ProjectID)
+	klog.V(3).Infof("HERE IS THE QUERY STRING: %s", queryString)
+	return gcp.QuerySQL(queryString)
+}
+
+// QuerySQL should query BigQuery for billing data for out of cluster costs.
+func (gcp *GCP) QuerySQL(query string) ([]*OutOfClusterAllocation, error) {
 	ctx := context.Background()
-	client, err := bigquery.NewClient(ctx, "guestbook-219823")
+	client, err := bigquery.NewClient(ctx, gcp.ProjectID) // For example, "guestbook-227502"
 	if err != nil {
 		return nil, err
 	}
 
-	q := client.Query()
+	q := client.Query(query)
 	it, err := q.Read(ctx)
 	if err != nil {
 		return nil, err
 	}
-
+	var allocations []*OutOfClusterAllocation
+	for {
+		var a gcpAllocation
+		err := it.Next(&a)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		allocations = append(allocations, gcpAllocationToOutOfClusterAllocation(a))
+	}
+	return allocations, nil
 }
 
 // ClusterName returns the name of a GKE cluster, as provided by metadata.
@@ -306,6 +368,14 @@ func (gcp *GCP) parsePages(inputKeys map[string]bool) (map[string]*GCPPricing, e
 // DownloadPricingData fetches data from the GCP Pricing API. Requires a key-- a kubecost key is provided for quickstart, but should be replaced by a users.
 func (gcp *GCP) DownloadPricingData() error {
 
+	c, err := GetDefaultPricingData("gcp.json")
+	if err != nil {
+		klog.V(2).Infof("Error downloading default pricing data: %s", err.Error())
+	}
+	gcp.BaseCPUPrice = c.CPU
+	gcp.ProjectID = c.ProjectID
+	gcp.BillingDataDataset = c.BillingDataDataset
+
 	nodeList, err := gcp.Clientset.CoreV1().Nodes().List(metav1.ListOptions{})
 	if err != nil {
 		return err
@@ -324,11 +394,6 @@ func (gcp *GCP) DownloadPricingData() error {
 		return err
 	}
 	gcp.Pricing = pages
-	c, err := GetDefaultPricingData("default.json")
-	if err != nil {
-		klog.V(2).Infof("Error downloading default pricing data: %s", err.Error())
-	}
-	gcp.BaseCPUPrice = c.CPU
 
 	return nil
 }

cloud/provider.go

@@ -38,6 +38,15 @@ type Key interface {
 	Features() string // Features are a comma separated string of node metadata that could match pricing
 }
 
+// OutOfClusterAllocation represents a cloud provider cost not associated with kubernetes
+type OutOfClusterAllocation struct {
+	Aggregator  string  `json:"aggregator"`
+	Environment string  `json:"environment"`
+	Service     string  `json:"service"`
+	Cost        float64 `json:"cost"`
+	Cluster     string  `json:"cluster"`
+}
+
 // Provider represents a k8s provider.
 type Provider interface {
 	ClusterName() ([]byte, error)
@@ -48,7 +57,7 @@ type Provider interface {
 	DownloadPricingData() error
 	GetKey(map[string]string) Key
 
-	QuerySQL(string) ([]byte, error)
+	ExternalAllocations(string, string) ([]*OutOfClusterAllocation, error)
 }
 
 // GetDefaultPricingData will search for a json file representing pricing data in /models/ and use it for base pricing info.
@@ -71,20 +80,21 @@ func GetDefaultPricingData(fname string) (*CustomPricing, error) {
 }
 
 type CustomPricing struct {
-	Provider         string `json:"provider"`
-	Description      string `json:"description"`
-	CPU              string `json:"CPU"`
-	SpotCPU          string `json:"spotCPU"`
-	RAM              string `json:"RAM"`
-	SpotRAM          string `json:"spotRAM"`
-	SpotLabel        string `json:"spotLabel,omitempty"`
-	SpotLabelValue   string `json:"spotLabelValue,omitempty"`
-	ServiceKeyName   string `json:"awsServiceKeyName,omitempty"`
-	ServiceKeySecret string `json:"awsServiceKeySecret,omitempty"`
-	SpotDataRegion   string `json:"awsSpotDataRegion,omitempty"`
-	SpotDataBucket   string `json:"awsSpotDataBucket,omitempty"`
-	SpotDataPrefix   string `json:"awsSpotDataPrefix,omitempty"`
-	ProjectID        string `json:"awsProjectID,omitempty"`
+	Provider           string `json:"provider"`
+	Description        string `json:"description"`
+	CPU                string `json:"CPU"`
+	SpotCPU            string `json:"spotCPU"`
+	RAM                string `json:"RAM"`
+	SpotRAM            string `json:"spotRAM"`
+	SpotLabel          string `json:"spotLabel,omitempty"`
+	SpotLabelValue     string `json:"spotLabelValue,omitempty"`
+	ServiceKeyName     string `json:"awsServiceKeyName,omitempty"`
+	ServiceKeySecret   string `json:"awsServiceKeySecret,omitempty"`
+	SpotDataRegion     string `json:"awsSpotDataRegion,omitempty"`
+	SpotDataBucket     string `json:"awsSpotDataBucket,omitempty"`
+	SpotDataPrefix     string `json:"awsSpotDataPrefix,omitempty"`
+	ProjectID          string `json:"projectID,omitempty"`
+	BillingDataDataset string `json:"billingDataDataset,omitempty"`
 }
 
 type NodePrice struct {
@@ -172,6 +182,10 @@ func (c *CustomProvider) GetKey(labels map[string]string) Key {
 	}
 }
 
+func (*CustomProvider) ExternalAllocations(start string, end string) ([]*OutOfClusterAllocation, error) {
+	return nil, nil // TODO: transform the QuerySQL lines into the new OutOfClusterAllocation Struct
+}
+
 func (*CustomProvider) QuerySQL(query string) ([]byte, error) {
 	return nil, nil
 }

main.go

@@ -104,6 +104,17 @@ func (a *Accesses) CostDataModelRange(w http.ResponseWriter, r *http.Request, ps
 	w.Write(wrapData(data, err))
 }
 
+func (a *Accesses) OutofClusterCosts(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	start := r.URL.Query().Get("start")
+	end := r.URL.Query().Get("end")
+
+	data, err := a.Cloud.ExternalAllocations(start, end)
+	w.Write(wrapData(data, err))
+}
+
 func (p *Accesses) GetAllNodePricing(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -252,6 +263,7 @@ func main() {
 	router := httprouter.New()
 	router.GET("/costDataModel", a.CostDataModel)
 	router.GET("/costDataModelRange", a.CostDataModelRange)
+	router.GET("/outOfClusterCosts", a.OutofClusterCosts)
 	router.GET("/allNodePricing", a.GetAllNodePricing)
 	router.GET("/healthz", Healthz)
 	router.POST("/refreshPricing", a.RefreshPricingData)