cloud stubs

cloud/awsprovider.go

@@ -323,10 +323,6 @@ func (aws *AWS) DownloadPricingData() error {
 		}
 	}
 
-	if err != nil {
-		return err
-	}
-
 	sp, err := parseSpotData(aws.SpotDataBucket, aws.SpotDataPrefix, aws.ProjectID, aws.SpotDataRegion, aws.ServiceKeyName, aws.ServiceKeySecret)
 	if err != nil {
 		klog.V(1).Infof("Error downloading spot data %s", err.Error())
@@ -714,10 +710,6 @@ func parseSpotData(bucket string, prefix string, projectID string, region string
 			return nil, err
 		}
 	}
-	s3Prefix := projectID
-	if len(prefix) != 0 {
-		s3Prefix = prefix + "/" + s3Prefix
-	}
 
 	c := aws.NewConfig().WithRegion(region)
 
@@ -729,30 +721,22 @@ func parseSpotData(bucket string, prefix string, projectID string, region string
 	tOneDayAgo := tNow.Add(time.Duration(-24) * time.Hour) // Also get files from one day ago to avoid boundary conditions
 	ls := &s3.ListObjectsInput{
 		Bucket: aws.String(bucket),
-		Prefix: aws.String(s3Prefix + "." + tOneDayAgo.Format("2006-01-02")),
+		Prefix: aws.String(prefix + "/" + projectID + "." + tOneDayAgo.Format("2006-01-02")),
 	}
 	ls2 := &s3.ListObjectsInput{
 		Bucket: aws.String(bucket),
-		Prefix: aws.String(s3Prefix + "." + tNow.Format("2006-01-02")),
+		Prefix: aws.String(prefix + "/" + projectID + "." + tNow.Format("2006-01-02")),
 	}
 	lso, err := s3Svc.ListObjects(ls)
 	if err != nil {
 		return nil, err
 	}
-	lsoLen := len(lso.Contents)
-	klog.V(2).Infof("Found %d spot data files from yesterday", lsoLen)
-	if lsoLen == 0 {
-		klog.V(5).Infof("ListObjects \"s3://%s/%s\" produced no keys", ls.Bucket, ls.Prefix)
-	}
+	klog.V(2).Infof("Found %d spot data files from yesterday", len(lso.Contents))
 	lso2, err := s3Svc.ListObjects(ls2)
 	if err != nil {
 		return nil, err
 	}
-	lso2Len := len(lso2.Contents)
-	klog.V(2).Infof("Found %d spot data files from today", lso2Len)
-	if lso2Len == 0 {
-		klog.V(5).Infof("ListObjects \"s3://%s/%s\" produced no keys", ls2.Bucket, ls2.Prefix)
-	}
+	klog.V(2).Infof("Found %d spot data files from today", len(lso2.Contents))
 
 	var keys []*string
 	for _, obj := range lso.Contents {

cloud/gcpprovider.go

@@ -1,6 +1,7 @@
 package cloud
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -13,6 +14,7 @@ import (
 
 	"k8s.io/klog"
 
+	"cloud.google.com/go/bigquery"
 	"cloud.google.com/go/compute/metadata"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -42,7 +44,18 @@ type GCP struct {
 
 // QuerySQL should query BigQuery for billing data for out of cluster costs. TODO: Implement.
 func (*GCP) QuerySQL(query string) ([]byte, error) {
-	return nil, nil
+	ctx := context.Background()
+	client, err := bigquery.NewClient(ctx, "guestbook-219823")
+	if err != nil {
+		return nil, err
+	}
+
+	q := client.Query()
+	it, err := q.Read(ctx)
+	if err != nil {
+		return nil, err
+	}
+
 }
 
 // ClusterName returns the name of a GKE cluster, as provided by metadata.

go.mod

@@ -8,6 +8,7 @@ require (
 	cloud.google.com/go v0.34.0
 	github.com/aws/aws-sdk-go v1.19.10
 	github.com/golang/mock v1.2.0
+	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
 	github.com/jszwec/csvutil v1.2.1
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829

go.sum

@@ -49,6 +49,8 @@ github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeq
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/uuid v0.0.0-20171113160352-8c31c18f31ed/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
+github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=