Merge pull request #345 from kubecost/develop

Merge develop into master

Dockerfile

@@ -17,6 +17,7 @@ RUN set -e ;\
     if test -n "`git status --porcelain --untracked-files=no | grep '\.go'`"; then \
       GIT_DIRTY='+dirty' ;\
     fi ;\
+    cd cmd/costmodel;\
     CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
     go build -a -installsuffix cgo \
         -ldflags "-X main.gitCommit=${GIT_COMMIT}${GIT_DIRTY}" \
@@ -25,9 +26,9 @@ RUN set -e ;\
 FROM alpine:3.10.2
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build-env /go/bin/app /go/bin/app
-ADD ./cloud/default.json /models/default.json
-ADD ./cloud/azure.json /models/azure.json
-ADD ./cloud/aws.json /models/aws.json
-ADD ./cloud/gcp.json /models/gcp.json
+ADD ./configs/default.json /models/default.json
+ADD ./configs/azure.json /models/azure.json
+ADD ./configs/aws.json /models/aws.json
+ADD ./configs/gcp.json /models/gcp.json
 USER 1001
 ENTRYPOINT ["/go/bin/app"]

main.go → cmd/costmodel/main.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 
 	"github.com/julienschmidt/httprouter"
-	"github.com/kubecost/cost-model/costmodel"
+	"github.com/kubecost/cost-model/pkg/costmodel"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/klog"
 )

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/Azure/go-autorest v11.3.2+incompatible
 	github.com/aws/aws-sdk-go v1.28.9
 	github.com/dimchansky/utfbom v1.1.0 // indirect
-	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
+	github.com/etcd-io/bbolt v1.3.3
 	github.com/golang/mock v1.2.0
 	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/google/uuid v1.1.1
@@ -19,13 +19,14 @@ require (
 	github.com/jszwec/csvutil v1.2.1
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/lib/pq v1.2.0
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.8.1 // indirect
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
+	go.etcd.io/bbolt v1.3.3 // indirect
 	golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 // indirect
 	golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
@@ -36,4 +37,7 @@ require (
 	k8s.io/apimachinery v0.0.0-20190913075812-e119e5e154b6
 	k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
 	k8s.io/klog v0.4.0
+	sigs.k8s.io/yaml v1.1.0
 )
+
+go 1.13

go.sum

@@ -14,6 +14,7 @@ github.com/Azure/go-autorest v11.3.2+incompatible h1:2bRmoaLvtIXW5uWpZVoIkc0C1z7
 github.com/Azure/go-autorest v11.3.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
@@ -21,6 +22,7 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.19.10 h1:WHIaUrU98WsWIXxlxeMCmbuB5HowxuUnk8eBH4iGl/g=
 github.com/aws/aws-sdk-go v1.19.10/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0=
@@ -31,13 +33,24 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY=
 github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
@@ -46,14 +59,18 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
+github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac=
 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
@@ -64,6 +81,7 @@ github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@@ -82,6 +100,7 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
@@ -107,23 +126,34 @@ github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8 h1:L9JPKrt
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gophercloud/gophercloud v0.2.0 h1:lD2Bce2xBAMNNcFZ0dObTpXkGLlVIb33RPVUNVpw6ic=
 github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE=
 github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be h1:AHimNtVIpiBjPUhEF5KNCkrUyqTSA5zWUl8sQ2bfGBE=
 github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -133,8 +163,10 @@ github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62F
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jszwec/csvutil v1.2.1 h1:9+vmGqMdYxIbeDmVbTrVryibx2izwHAfKdPwl4GPNHM=
 github.com/jszwec/csvutil v1.2.1/go.mod h1:8YHz6C3KVdIeCxLMvwbbIVDCTA/Wi2df93AZlQNaE2U=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -147,11 +179,16 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kubecost/cost-model v0.0.0-20190415210323-992655b79eac/go.mod h1:NxiMjOpYdrBQBjo3bGcJOpB+KZd1NWpTbWaWlMq3f+Q=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -162,6 +199,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -174,6 +212,8 @@ github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
@@ -186,6 +226,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -193,41 +234,75 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJ
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
+github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A=
 go.opencensus.io v0.19.2 h1:ZZpq6xI6kv/LuE/5s5UQvBU5vMjvRnPb8PvJrIntAnc=
 go.opencensus.io v0.19.2/go.mod h1:NO/8qkisMZLZ1FCsKNqtJPwc8/TaclWyY0B6wcYNg9M=
 go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180808211826-de0752318171/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -260,6 +335,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
 golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc h1:gkKoSkUmnU6bpS/VhkuO27bzQeSA51uaEfbOW5dNb68=
 golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -284,6 +360,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
@@ -305,6 +382,7 @@ golang.org/x/time v0.0.0-20161028155119-f51c12702a4d h1:TnM+PKb3ylGmZvyPXmo9m/wk
 golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -315,6 +393,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU=
 google.golang.org/api v0.3.0 h1:UIJY20OEo3+tK5MBlcdx37kmdH6EnRjGkW78mc6+EeA=
@@ -336,6 +415,8 @@ google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
@@ -343,12 +424,16 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o=
 gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

cloud/awsprovider.go → pkg/cloud/awsprovider.go

@@ -19,7 +19,7 @@ import (
 
 	"k8s.io/klog"
 
-	"github.com/kubecost/cost-model/clustercache"
+	"github.com/kubecost/cost-model/pkg/clustercache"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -303,7 +303,7 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 			c.AthenaTable = a.AthenaTable
 			c.ServiceKeyName = a.ServiceKeyName
 			c.ServiceKeySecret = a.ServiceKeySecret
-			c.ProjectID = a.AccountID
+			c.AthenaProjectID = a.AccountID
 		} else {
 			a := make(map[string]interface{})
 			err := json.NewDecoder(r).Decode(&a)
@@ -399,14 +399,16 @@ type awsPVKey struct {
 	StorageClassParameters map[string]string
 	StorageClassName       string
 	Name                   string
+	DefaultRegion          string
 }
 
-func (aws *AWS) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string) PVKey {
+func (aws *AWS) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
 	return &awsPVKey{
 		Labels:                 pv.Labels,
 		StorageClassName:       pv.Spec.StorageClassName,
 		StorageClassParameters: parameters,
 		Name:                   pv.Name,
+		DefaultRegion:          defaultRegion,
 	}
 }
 
@@ -505,7 +507,7 @@ func (aws *AWS) DownloadPricingData() error {
 			klog.V(2).Infof("Unable to find params for storageClassName %s, falling back to default pricing", pv.Spec.StorageClassName)
 			continue
 		}
-		key := aws.GetPVKey(pv, params)
+		key := aws.GetPVKey(pv, params, "")
 		pvkeys[key.Features()] = key
 	}
 
@@ -1056,16 +1058,29 @@ func ConvertToGlueColumnFormat(column_name string) string {
 	return final
 }
 
+func generateAWSGroupBy(lastIdx int) string {
+	sequence := []string{}
+	for i := 1; i < lastIdx+1; i++ {
+		sequence = append(sequence, strconv.Itoa(i))
+	}
+	return strings.Join(sequence, ",")
+}
+
 // ExternalAllocations represents tagged assets outside the scope of kubernetes.
 // "start" and "end" are dates of the format YYYY-MM-DD
 // "aggregator" is the tag used to determine how to allocate those assets, ie namespace, pod, etc.
-func (a *AWS) ExternalAllocations(start string, end string, aggregator string, filterType string, filterValue string) ([]*OutOfClusterAllocation, error) {
+func (a *AWS) ExternalAllocations(start string, end string, aggregators []string, filterType string, filterValue string) ([]*OutOfClusterAllocation, error) {
 	customPricing, err := a.GetConfig()
 	if err != nil {
 		return nil, err
 	}
-	aggregator_column_name := "resource_tags_user_" + aggregator
-	aggregator_column_name = ConvertToGlueColumnFormat(aggregator_column_name)
+	formattedAggregators := []string{}
+	for _, agg := range aggregators {
+		aggregator_column_name := "resource_tags_user_" + agg
+		aggregator_column_name = ConvertToGlueColumnFormat(aggregator_column_name)
+		formattedAggregators = append(formattedAggregators, aggregator_column_name)
+	}
+	aggregatorNames := strings.Join(formattedAggregators, ",")
 
 	filter_column_name := "resource_tags_user_" + filterType
 	filter_column_name = ConvertToGlueColumnFormat(filter_column_name)
@@ -1073,6 +1088,8 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregator string, f
 	var query string
 	var lastIdx int
 	if filterType != "kubernetes_" { // This gets appended upstream and is equivalent to no filter.
+		lastIdx = len(formattedAggregators) + 3
+		groupby := generateAWSGroupBy(lastIdx)
 		query = fmt.Sprintf(`SELECT   
 			CAST(line_item_usage_start_date AS DATE) as start_date,
 			%s,
@@ -1080,11 +1097,11 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregator string, f
 			%s,
 			SUM(line_item_blended_cost) as blended_cost
 		FROM %s as cost_data
-		WHERE (%s='%s' OR %s='') AND line_item_usage_start_date BETWEEN date '%s' AND date '%s'
-		GROUP BY 1,2,3,4`, aggregator_column_name, filter_column_name, customPricing.AthenaTable, filter_column_name, filterValue, filter_column_name, start, end)
-		lastIdx = 4
+		WHERE (%s='%s') AND line_item_usage_start_date BETWEEN date '%s' AND date '%s'
+		GROUP BY %s`, aggregatorNames, filter_column_name, customPricing.AthenaTable, filter_column_name, filterValue, start, end, groupby)
 	} else {
-		lastIdx = 3
+		lastIdx = len(formattedAggregators) + 2
+		groupby := generateAWSGroupBy(lastIdx)
 		query = fmt.Sprintf(`SELECT   
 			CAST(line_item_usage_start_date AS DATE) as start_date,
 			%s,
@@ -1092,7 +1109,7 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregator string, f
 			SUM(line_item_blended_cost) as blended_cost
 		FROM %s as cost_data
 		WHERE line_item_usage_start_date BETWEEN date '%s' AND date '%s'
-		GROUP BY 1,2,3`, aggregator_column_name, customPricing.AthenaTable, start, end)
+		GROUP BY %s`, aggregatorNames, customPricing.AthenaTable, start, end, groupby)
 	}
 
 	klog.V(3).Infof("Running Query: %s", query)
@@ -1110,7 +1127,6 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregator string, f
 	region := aws.String(customPricing.AthenaRegion)
 	resultsBucket := customPricing.AthenaBucketName
 	database := customPricing.AthenaDatabase
-
 	c := &aws.Config{
 		Region: region,
 	}
@@ -1169,17 +1185,36 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregator string, f
 				if err != nil {
 					return nil, err
 				}
+				environment := ""
+				for _, d := range r.Data[1 : len(formattedAggregators)+1] {
+					if *d.VarCharValue != "" {
+						environment = *d.VarCharValue // just set to the first nonempty match
+					}
+					break
+				}
 				ooc := &OutOfClusterAllocation{
-					Aggregator:  aggregator,
-					Environment: *r.Data[1].VarCharValue,
-					Service:     *r.Data[2].VarCharValue,
+					Aggregator:  strings.Join(aggregators, ","),
+					Environment: environment,
+					Service:     *r.Data[len(formattedAggregators)+1].VarCharValue,
 					Cost:        cost,
 				}
 				oocAllocs = append(oocAllocs, ooc)
 			}
 		} else {
-			klog.V(1).Infof("No results available for %s at database %s between %s and %s", aggregator_column_name, customPricing.AthenaTable, start, end)
+			klog.V(1).Infof("No results available for %s at database %s between %s and %s", strings.Join(formattedAggregators, ","), customPricing.AthenaTable, start, end)
+		}
+	}
+
+	if customPricing.BillingDataDataset != "" { // There is GCP data, meaning someone has tried to configure a GCP out-of-cluster allocation.
+		gcp, err := NewCrossClusterProvider("gcp", "gcp.json", a.Clientset)
+		if err != nil {
+			klog.Infof("Could not instantiate cross-cluster provider %s", err.Error())
+		}
+		gcpOOC, err := gcp.ExternalAllocations(start, end, aggregators, filterType, filterValue)
+		if err != nil {
+			klog.Infof("Could not fetch cross-cluster costs %s", err.Error())
 		}
+		oocAllocs = append(oocAllocs, gcpOOC...)
 	}
 
 	return oocAllocs, nil // TODO: transform the QuerySQL lines into the new OutOfClusterAllocation Struct

cloud/azureprovider.go → pkg/cloud/azureprovider.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/kubecost/cost-model/clustercache"
+	"github.com/kubecost/cost-model/pkg/clustercache"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-09-01/skus"
 	"github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2018-03-31/containerservice"
@@ -26,6 +26,11 @@ import (
 	"k8s.io/klog"
 )
 
+const (
+	AzurePremiumStorageClass  = "premium"
+	AzureStandardStorageClass = "standard"
+)
+
 var (
 	regionCodeMappings = map[string]string{
 		"ap": "asia",
@@ -161,8 +166,14 @@ func checkRegionID(regionID string, regions map[string]string) bool {
 	return false
 }
 
+// AzurePricing either contains a Node or PV
+type AzurePricing struct {
+	Node *Node
+	PV   *PV
+}
+
 type Azure struct {
-	allPrices               map[string]*Node
+	Pricing                 map[string]*AzurePricing
 	DownloadPricingDataLock sync.RWMutex
 	Clientset               clustercache.ClusterCache
 	Config                  *ProviderConfig
@@ -307,6 +318,7 @@ func (az *Azure) DownloadPricingData() error {
 	if err != nil {
 		return err
 	}
+
 	var authorizer autorest.Authorizer
 
 	if config.AzureClientID != "" && config.AzureClientSecret != "" && config.AzureTenantID != "" {
@@ -352,7 +364,7 @@ func (az *Azure) DownloadPricingData() error {
 	if err != nil {
 		return err
 	}
-	allPrices := make(map[string]*Node)
+	allPrices := make(map[string]*AzurePricing)
 	regions, err := getRegions("compute", sClient, providersClient, config.AzureSubscriptionID)
 	if err != nil {
 		return err
@@ -365,61 +377,101 @@ func (az *Azure) DownloadPricingData() error {
 	baseCPUPrice := c.CPU
 
 	for _, v := range *result.Meters {
-		if !strings.Contains(*v.MeterSubCategory, "Windows") && strings.Contains(*v.MeterCategory, "Virtual Machines") {
+		meterName := *v.MeterName
+		meterRegion := *v.MeterRegion
+		meterCategory := *v.MeterCategory
+		meterSubCategory := *v.MeterSubCategory
 
-			region, err := toRegionID(*v.MeterRegion, regions)
-			if err != nil {
-				continue
-			}
+		region, err := toRegionID(meterRegion, regions)
+		if err != nil {
+			continue
+		}
 
-			meterName := *v.MeterName
-			sc := *v.MeterSubCategory
+		if !strings.Contains(meterSubCategory, "Windows") {
 
-			// not available now
-			if strings.Contains(sc, "Promo") {
-				continue
-			}
+			if strings.Contains(meterCategory, "Storage") {
+				if strings.Contains(meterSubCategory, "HDD") || strings.Contains(meterSubCategory, "SSD") {
+					var storageClass string = ""
+					if strings.Contains(meterName, "S4 ") {
+						storageClass = AzureStandardStorageClass
+					} else if strings.Contains(meterName, "P4 ") {
+						storageClass = AzurePremiumStorageClass
+					}
 
-			usageType := ""
-			if !strings.Contains(meterName, "Low Priority") {
-				usageType = "ondemand"
-			} else {
-				usageType = "preemptible"
-			}
+					if storageClass != "" {
+						var priceInUsd float64
 
-			var instanceTypes []string
-			name := strings.TrimSuffix(meterName, " Low Priority")
-			instanceType := strings.Split(name, "/")
-			for _, it := range instanceType {
-				instanceTypes = append(instanceTypes, strings.Replace(it, " ", "_", 1))
+						if len(v.MeterRates) < 1 {
+							klog.V(1).Infof("missing rate info %+v", map[string]interface{}{"MeterSubCategory": *v.MeterSubCategory, "region": region})
+							continue
+						}
+						for _, rate := range v.MeterRates {
+							priceInUsd += *rate
+						}
+						priceStr := fmt.Sprintf("%f", priceInUsd)
+
+						key := region + "," + storageClass
+						klog.V(4).Infof("Adding PV.Key: %s, Cost: %s", key, priceStr)
+						allPrices[key] = &AzurePricing{
+							PV: &PV{
+								Cost:   priceStr,
+								Region: region,
+							},
+						}
+					}
+				}
 			}
 
-			instanceTypes = transformMachineType(sc, instanceTypes)
-			if strings.Contains(name, "Expired") {
-				instanceTypes = []string{}
-			}
+			if strings.Contains(meterCategory, "Virtual Machines") {
 
-			var priceInUsd float64
+				// not available now
+				if strings.Contains(meterSubCategory, "Promo") {
+					continue
+				}
 
-			if len(v.MeterRates) < 1 {
-				klog.V(1).Infof("missing rate info %+v", map[string]interface{}{"MeterSubCategory": *v.MeterSubCategory, "region": region})
-				continue
-			}
-			for _, rate := range v.MeterRates {
-				priceInUsd += *rate
-			}
-			priceStr := fmt.Sprintf("%f", priceInUsd)
-			for _, instanceType := range instanceTypes {
+				usageType := ""
+				if !strings.Contains(meterName, "Low Priority") {
+					usageType = "ondemand"
+				} else {
+					usageType = "preemptible"
+				}
+
+				var instanceTypes []string
+				name := strings.TrimSuffix(meterName, " Low Priority")
+				instanceType := strings.Split(name, "/")
+				for _, it := range instanceType {
+					instanceTypes = append(instanceTypes, strings.Replace(it, " ", "_", 1))
+				}
+
+				instanceTypes = transformMachineType(meterSubCategory, instanceTypes)
+				if strings.Contains(name, "Expired") {
+					instanceTypes = []string{}
+				}
 
-				key := fmt.Sprintf("%s,%s,%s", region, instanceType, usageType)
-				allPrices[key] = &Node{
-					Cost:         priceStr,
-					BaseCPUPrice: baseCPUPrice,
+				var priceInUsd float64
+
+				if len(v.MeterRates) < 1 {
+					klog.V(1).Infof("missing rate info %+v", map[string]interface{}{"MeterSubCategory": *v.MeterSubCategory, "region": region})
+					continue
+				}
+				for _, rate := range v.MeterRates {
+					priceInUsd += *rate
+				}
+				priceStr := fmt.Sprintf("%f", priceInUsd)
+				for _, instanceType := range instanceTypes {
+
+					key := fmt.Sprintf("%s,%s,%s", region, instanceType, usageType)
+					allPrices[key] = &AzurePricing{
+						Node: &Node{
+							Cost:         priceStr,
+							BaseCPUPrice: baseCPUPrice,
+						},
+					}
 				}
 			}
 		}
 	}
-	az.allPrices = allPrices
+	az.Pricing = allPrices
 	return nil
 }
 
@@ -427,19 +479,19 @@ func (az *Azure) DownloadPricingData() error {
 func (az *Azure) AllNodePricing() (interface{}, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
-	return az.allPrices, nil
+	return az.Pricing, nil
 }
 
 // NodePricing returns Azure pricing data for a single node
 func (az *Azure) NodePricing(key Key) (*Node, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
-	if n, ok := az.allPrices[key.Features()]; ok {
+	if n, ok := az.Pricing[key.Features()]; ok {
 		klog.V(4).Infof("Returning pricing for node %s: %+v from key %s", key, n, key.Features())
 		if key.GPUType() != "" {
-			n.GPU = "1" // TODO: support multiple GPUs
+			n.Node.GPU = "1" // TODO: support multiple GPUs
 		}
-		return n, nil
+		return n.Node, nil
 	}
 	klog.V(1).Infof("[Warning] no pricing data found for %s: %s", key.Features(), key)
 	c, err := az.GetConfig()
@@ -491,13 +543,15 @@ type azurePvKey struct {
 	Labels                 map[string]string
 	StorageClass           string
 	StorageClassParameters map[string]string
+	DefaultRegion          string
 }
 
-func (az *Azure) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string) PVKey {
+func (az *Azure) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
 	return &azurePvKey{
 		Labels:                 pv.Labels,
 		StorageClass:           pv.Spec.StorageClassName,
 		StorageClassParameters: parameters,
+		DefaultRegion:          defaultRegion,
 	}
 }
 
@@ -506,13 +560,17 @@ func (key *azurePvKey) GetStorageClass() string {
 }
 
 func (key *azurePvKey) Features() string {
-	storageClass := key.StorageClassParameters["type"]
-	if storageClass == "pd-ssd" {
-		storageClass = "ssd"
-	} else if storageClass == "pd-standard" {
-		storageClass = "pdstandard"
+	storageClass := key.StorageClassParameters["storageaccounttype"]
+	if strings.EqualFold(storageClass, "Premium_LRS") {
+		storageClass = AzurePremiumStorageClass
+	} else if strings.EqualFold(storageClass, "Standard_LRS") {
+		storageClass = AzureStandardStorageClass
+	}
+	if region, ok := key.Labels[v1.LabelZoneRegion]; ok {
+		return region + "," + storageClass
 	}
-	return key.Labels[v1.LabelZoneRegion] + "," + storageClass
+
+	return key.DefaultRegion + "," + storageClass
 }
 
 func (*Azure) GetDisks() ([]byte, error) {
@@ -608,7 +666,7 @@ func (az *Azure) GetConfig() (*CustomPricing, error) {
 	return c, nil
 }
 
-func (az *Azure) ExternalAllocations(string, string, string, string, string) ([]*OutOfClusterAllocation, error) {
+func (az *Azure) ExternalAllocations(string, string, []string, string, string) ([]*OutOfClusterAllocation, error) {
 	return nil, nil
 }
 
@@ -616,8 +674,16 @@ func (az *Azure) ApplyReservedInstancePricing(nodes map[string]*Node) {
 
 }
 
-func (az *Azure) PVPricing(PVKey) (*PV, error) {
-	return nil, nil
+func (az *Azure) PVPricing(pvk PVKey) (*PV, error) {
+	az.DownloadPricingDataLock.RLock()
+	defer az.DownloadPricingDataLock.RUnlock()
+
+	pricing, ok := az.Pricing[pvk.Features()]
+	if !ok {
+		klog.V(4).Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
+		return &PV{}, nil
+	}
+	return pricing.PV, nil
 }
 
 func (az *Azure) GetLocalStorageQuery(window, offset string, rate bool) string {

cloud/customprovider.go → pkg/cloud/customprovider.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/kubecost/cost-model/clustercache"
+	"github.com/kubecost/cost-model/pkg/clustercache"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -191,7 +191,7 @@ func (cp *CustomProvider) GetKey(labels map[string]string) Key {
 // ExternalAllocations represents tagged assets outside the scope of kubernetes.
 // "start" and "end" are dates of the format YYYY-MM-DD
 // "aggregator" is the tag used to determine how to allocate those assets, ie namespace, pod, etc.
-func (*CustomProvider) ExternalAllocations(start string, end string, aggregator string, filterType string, filterValue string) ([]*OutOfClusterAllocation, error) {
+func (*CustomProvider) ExternalAllocations(start string, end string, aggregator []string, filterType string, filterValue string) ([]*OutOfClusterAllocation, error) {
 	return nil, nil // TODO: transform the QuerySQL lines into the new OutOfClusterAllocation Struct
 }
 
@@ -234,10 +234,12 @@ func (cp *CustomProvider) NetworkPricing() (*Network, error) {
 	}, nil
 }
 
-func (*CustomProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string) PVKey {
+func (*CustomProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
 	return &awsPVKey{
-		Labels:           pv.Labels,
-		StorageClassName: pv.Spec.StorageClassName,
+		Labels:                 pv.Labels,
+		StorageClassName:       pv.Spec.StorageClassName,
+		StorageClassParameters: parameters,
+		DefaultRegion:          defaultRegion,
 	}
 }
 

cloud/gcpprovider.go → pkg/cloud/gcpprovider.go

@@ -20,7 +20,7 @@ import (
 
 	"cloud.google.com/go/bigquery"
 	"cloud.google.com/go/compute/metadata"
-	"github.com/kubecost/cost-model/clustercache"
+	"github.com/kubecost/cost-model/pkg/clustercache"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	compute "google.golang.org/api/compute/v1"
@@ -62,6 +62,40 @@ type gcpAllocation struct {
 	Cost        float64
 }
 
+type multiKeyGCPAllocation struct {
+	Keys    bigquery.NullString
+	Service string
+	Cost    float64
+}
+
+func multiKeyGCPAllocationToOutOfClusterAllocation(gcpAlloc multiKeyGCPAllocation, aggregatorNames []string) *OutOfClusterAllocation {
+	var keys []map[string]string
+	var environment string
+	var usedAggregatorName string
+	if gcpAlloc.Keys.Valid {
+		err := json.Unmarshal([]byte(gcpAlloc.Keys.StringVal), &keys)
+		if err != nil {
+			klog.Infof("Invalid unmarshaling response from BigQuery filtered query: %s", err.Error())
+		}
+	keyloop:
+		for _, label := range keys {
+			for _, aggregatorName := range aggregatorNames {
+				if label["key"] == aggregatorName {
+					environment = label["value"]
+					usedAggregatorName = label["key"]
+					break keyloop
+				}
+			}
+		}
+	}
+	return &OutOfClusterAllocation{
+		Aggregator:  usedAggregatorName,
+		Environment: environment,
+		Service:     gcpAlloc.Service,
+		Cost:        gcpAlloc.Cost,
+	}
+}
+
 func gcpAllocationToOutOfClusterAllocation(gcpAlloc gcpAllocation) *OutOfClusterAllocation {
 	var aggregator string
 	if gcpAlloc.Aggregator.Valid {
@@ -171,6 +205,19 @@ func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 			if err != nil {
 				return err
 			}
+		} else if updateType == AthenaInfoUpdateType {
+			a := AwsAthenaInfo{}
+			err := json.NewDecoder(r).Decode(&a)
+			if err != nil {
+				return err
+			}
+			c.AthenaBucketName = a.AthenaBucketName
+			c.AthenaRegion = a.AthenaRegion
+			c.AthenaDatabase = a.AthenaDatabase
+			c.AthenaTable = a.AthenaTable
+			c.ServiceKeyName = a.ServiceKeyName
+			c.ServiceKeySecret = a.ServiceKeySecret
+			c.AthenaProjectID = a.AccountID
 		} else {
 			a := make(map[string]interface{})
 			err := json.NewDecoder(r).Decode(&a)
@@ -211,28 +258,118 @@ func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 // ExternalAllocations represents tagged assets outside the scope of kubernetes.
 // "start" and "end" are dates of the format YYYY-MM-DD
 // "aggregator" is the tag used to determine how to allocate those assets, ie namespace, pod, etc.
-func (gcp *GCP) ExternalAllocations(start string, end string, aggregator string, filterType string, filterValue string) ([]*OutOfClusterAllocation, error) {
+func (gcp *GCP) ExternalAllocations(start string, end string, aggregators []string, filterType string, filterValue string) ([]*OutOfClusterAllocation, error) {
 	c, err := gcp.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
 	}
-	// start, end formatted like: "2019-04-20 00:00:00"
-	queryString := fmt.Sprintf(`SELECT
-					service,
-					labels.key as aggregator,
-					labels.value as environment,
-					SUM(cost) as cost
-					FROM  (SELECT 
-							service.description as service,
-							labels,
-							cost 
-						FROM %s
-						WHERE usage_start_time >= "%s" AND usage_start_time < "%s")
-						LEFT JOIN UNNEST(labels) as labels
-						ON labels.key = "%s"
-				GROUP BY aggregator, environment, service;`, c.BillingDataDataset, start, end, aggregator) // For example, "billing_data.gcp_billing_export_v1_01AC9F_74CF1D_5565A2"
-	klog.V(4).Infof("Querying \"%s\" with : %s", c.ProjectID, queryString)
-	return gcp.QuerySQL(queryString)
+	var s []*OutOfClusterAllocation
+	if c.ServiceKeyName != "" && c.ServiceKeySecret != "" {
+		aws, err := NewCrossClusterProvider("aws", "gcp.json", gcp.Clientset)
+		if err != nil {
+			klog.Infof("Could not instantiate cross-cluster provider %s", err.Error())
+		}
+		awsOOC, err := aws.ExternalAllocations(start, end, aggregators, filterType, filterValue)
+		if err != nil {
+			klog.Infof("Could not fetch cross-cluster costs %s", err.Error())
+		}
+		s = append(s, awsOOC...)
+	}
+
+	formattedAggregators := []string{}
+	for _, a := range aggregators {
+		formattedAggregators = append(formattedAggregators, strconv.Quote(a))
+	}
+
+	aggregator := strings.Join(formattedAggregators, ",")
+
+	var qerr error
+	if filterType == "kubernetes_" {
+		// start, end formatted like: "2019-04-20 00:00:00"
+		/* OLD METHOD: supported getting all data, including unaggregated.
+		queryString := fmt.Sprintf(`SELECT
+						service,
+						labels.key as aggregator,
+						labels.value as environment,
+						SUM(cost) as cost
+						FROM  (SELECT
+								service.description as service,
+								labels,
+								cost
+							FROM %s
+							WHERE usage_start_time >= "%s" AND usage_start_time < "%s")
+							LEFT JOIN UNNEST(labels) as labels
+							ON labels.key = "%s"
+					GROUP BY aggregator, environment, service;`, c.BillingDataDataset, start, end, aggregator) // For example, "billing_data.gcp_billing_export_v1_01AC9F_74CF1D_5565A2"
+		klog.V(3).Infof("Querying \"%s\" with : %s", c.ProjectID, queryString)
+		gcpOOC, err := gcp.QuerySQL(queryString)
+		s = append(s, gcpOOC...)
+		qerr = err
+		*/
+		queryString := fmt.Sprintf(`(SELECT
+			service.description as service,
+			TO_JSON_STRING(labels) as keys,
+			SUM(cost) as cost
+		  	FROM  %s
+		 	WHERE
+				EXISTS(SELECT * FROM UNNEST(labels) AS l2 WHERE l2.key IN (%s))
+				AND usage_start_time >= "%s" AND usage_start_time < "%s"
+			GROUP BY  service,keys)`, c.BillingDataDataset, aggregator, start, end)
+		klog.V(3).Infof("Querying \"%s\" with : %s", c.ProjectID, queryString)
+		gcpOOC, err := gcp.multiLabelQuery(queryString, aggregators)
+		s = append(s, gcpOOC...)
+		qerr = err
+	} else {
+		queryString := fmt.Sprintf(`(SELECT
+			service.description as service,
+			TO_JSON_STRING(labels) as keys,
+			SUM(cost) as cost
+		  	FROM  %s
+		 	WHERE
+				EXISTS (SELECT * FROM UNNEST(labels) AS l WHERE l.key = "%s" AND l.value = "%s")
+				AND EXISTS(SELECT * FROM UNNEST(labels) AS l2 WHERE l2.key IN (%s))
+				AND usage_start_time >= "%s" AND usage_start_time < "%s"
+			GROUP BY  service,keys)`, c.BillingDataDataset, filterType, filterValue, aggregator, start, end)
+		klog.V(3).Infof("Querying \"%s\" with : %s", c.ProjectID, queryString)
+		gcpOOC, err := gcp.multiLabelQuery(queryString, aggregators)
+		s = append(s, gcpOOC...)
+		qerr = err
+	}
+	if qerr != nil {
+		klog.Infof("Error querying gcp: %s", qerr)
+	}
+	return s, qerr
+}
+
+func (gcp *GCP) multiLabelQuery(query string, aggregators []string) ([]*OutOfClusterAllocation, error) {
+	c, err := gcp.Config.GetCustomPricingData()
+	if err != nil {
+		return nil, err
+	}
+	ctx := context.Background()
+	client, err := bigquery.NewClient(ctx, c.ProjectID) // For example, "guestbook-227502"
+	if err != nil {
+		return nil, err
+	}
+
+	q := client.Query(query)
+	it, err := q.Read(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var allocations []*OutOfClusterAllocation
+	for {
+		var a multiKeyGCPAllocation
+		err := it.Next(&a)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		allocations = append(allocations, multiKeyGCPAllocationToOutOfClusterAllocation(a, aggregators))
+	}
+	return allocations, nil
 }
 
 // QuerySQL should query BigQuery for billing data for out of cluster costs.
@@ -742,7 +879,7 @@ func (gcp *GCP) DownloadPricingData() error {
 			klog.Infof("Unable to find params for storageClassName %s", pv.Name)
 			continue
 		}
-		key := gcp.GetPVKey(pv, params)
+		key := gcp.GetPVKey(pv, params, "")
 		pvkeys[key.Features()] = key
 	}
 
@@ -1030,17 +1167,19 @@ type pvKey struct {
 	Labels                 map[string]string
 	StorageClass           string
 	StorageClassParameters map[string]string
+	DefaultRegion          string
 }
 
 func (key *pvKey) GetStorageClass() string {
 	return key.StorageClass
 }
 
-func (gcp *GCP) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string) PVKey {
+func (gcp *GCP) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
 	return &pvKey{
 		Labels:                 pv.Labels,
 		StorageClass:           pv.Spec.StorageClassName,
 		StorageClassParameters: parameters,
+		DefaultRegion:          defaultRegion,
 	}
 }
 

cloud/provider.go → pkg/cloud/provider.go

@@ -12,7 +12,7 @@ import (
 	"k8s.io/klog"
 
 	"cloud.google.com/go/compute/metadata"
-	"github.com/kubecost/cost-model/clustercache"
+	"github.com/kubecost/cost-model/pkg/clustercache"
 
 	v1 "k8s.io/api/core/v1"
 )
@@ -58,6 +58,7 @@ type Node struct {
 	GPU              string                `json:"gpu"` // GPU represents the number of GPU on the instance
 	GPUName          string                `json:"gpuName"`
 	GPUCost          string                `json:"gpuCost"`
+	InstanceType     string                `json:"instanceType,omitempty"`
 	Reserved         *ReservedInstanceData `json:"reserved,omitempty"`
 }
 
@@ -129,6 +130,7 @@ type CustomPricing struct {
 	SpotDataBucket        string            `json:"awsSpotDataBucket,omitempty"`
 	SpotDataPrefix        string            `json:"awsSpotDataPrefix,omitempty"`
 	ProjectID             string            `json:"projectID,omitempty"`
+	AthenaProjectID       string            `json:"athenaProjectID,omitempty"`
 	AthenaBucketName      string            `json:"athenaBucketName"`
 	AthenaRegion          string            `json:"athenaRegion"`
 	AthenaDatabase        string            `json:"athenaDatabase"`
@@ -163,13 +165,13 @@ type Provider interface {
 	AllNodePricing() (interface{}, error)
 	DownloadPricingData() error
 	GetKey(map[string]string) Key
-	GetPVKey(*v1.PersistentVolume, map[string]string) PVKey
+	GetPVKey(*v1.PersistentVolume, map[string]string, string) PVKey
 	UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error)
 	UpdateConfigFromConfigMap(map[string]string) (*CustomPricing, error)
 	GetConfig() (*CustomPricing, error)
 	GetManagementPlatform() (string, error)
 	GetLocalStorageQuery(string, string, bool) string
-	ExternalAllocations(string, string, string, string, string) ([]*OutOfClusterAllocation, error)
+	ExternalAllocations(string, string, []string, string, string) ([]*OutOfClusterAllocation, error)
 	ApplyReservedInstancePricing(map[string]*Node)
 }
 
@@ -203,6 +205,24 @@ func CustomPricesEnabled(p Provider) bool {
 	return config.CustomPricesEnabled == "true"
 }
 
+func NewCrossClusterProvider(ctype string, overrideConfigPath string, cache clustercache.ClusterCache) (Provider, error) {
+	if ctype == "aws" {
+		return &AWS{
+			Clientset: cache,
+			Config:    NewProviderConfig(overrideConfigPath),
+		}, nil
+	} else if ctype == "gcp" {
+		return &GCP{
+			Clientset: cache,
+			Config:    NewProviderConfig(overrideConfigPath),
+		}, nil
+	}
+	return &CustomProvider{
+		Clientset: cache,
+		Config:    NewProviderConfig(overrideConfigPath),
+	}, nil
+}
+
 // NewProvider looks at the nodespec or provider metadata server to decide which provider to instantiate.
 func NewProvider(cache clustercache.ClusterCache, apiKey string) (Provider, error) {
 	if metadata.OnGCE() {

cloud/providerconfig.go → pkg/cloud/providerconfig.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/kubecost/cost-model/pkg/util"
+
 	"k8s.io/klog"
 )
 
@@ -194,16 +196,7 @@ func SetCustomPricingField(obj *CustomPricing, name string, value string) error
 // but the error isn't relevant to the path. This can happen when the current
 // user doesn't have permission to access the file.
 func fileExists(filename string) (bool, error) {
-	info, err := os.Stat(filename)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-
-		return false, err
-	}
-
-	return !info.IsDir(), nil
+	return util.FileExists(filename) // delegate to utility method
 }
 
 // Returns the configuration directory concatenated with a specific config file name

pkg/clustermanager/boltdbstorage.go

@@ -0,0 +1,93 @@
+package clustermanager
+
+import (
+	bolt "github.com/etcd-io/bbolt"
+	_ "k8s.io/klog"
+)
+
+type BoltDBClusterStorage struct {
+	bucket []byte
+	db     *bolt.DB
+}
+
+func NewBoltDBClusterStorage(bucket string, db *bolt.DB) (ClusterStorage, error) {
+	bucketKey := []byte(bucket)
+
+	err := db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(bucketKey)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &BoltDBClusterStorage{
+		bucket: bucketKey,
+		db:     db,
+	}, nil
+}
+
+// Adds the entry if the key does not exist
+func (cs *BoltDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
+	return cs.db.Update(func(tx *bolt.Tx) error {
+		k := []byte(key)
+		bucket := tx.Bucket(cs.bucket)
+
+		if bucket.Get(k) != nil {
+			return nil
+		}
+		return bucket.Put(k, cluster)
+	})
+}
+
+// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+// value with the provided.
+func (cs *BoltDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
+	return cs.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(cs.bucket)
+
+		return bucket.Put([]byte(key), cluster)
+	})
+}
+
+// Removes a key from the cluster storage
+func (cs *BoltDBClusterStorage) Remove(key string) error {
+	return cs.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(cs.bucket)
+
+		return bucket.Delete([]byte(key))
+	})
+}
+
+// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+// an error, the iteration stops.
+func (cs *BoltDBClusterStorage) Each(handler func(string, []byte) error) error {
+	return cs.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(cs.bucket)
+
+		return bucket.ForEach(func(k, v []byte) error {
+			// Allow the bytes to live outside transaction by copy
+			key := make([]byte, len(k))
+			value := make([]byte, len(v))
+
+			copy(key, k)
+			copy(value, v)
+
+			if err := handler(string(key), value); err != nil {
+				return err
+			}
+
+			return nil
+		})
+	})
+}
+
+// Closes the backing storage
+func (cs *BoltDBClusterStorage) Close() error {
+	return cs.db.Close()
+}

pkg/clustermanager/clustermanager.go

@@ -0,0 +1,164 @@
+package clustermanager
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	"github.com/google/uuid"
+
+	"github.com/kubecost/cost-model/pkg/util"
+
+	"k8s.io/klog"
+	"sigs.k8s.io/yaml"
+)
+
+// Cluster definition from a configuration yaml
+type ClusterConfigEntry struct {
+	Name    string `yaml:"name"`
+	Address string `yaml:"address"`
+}
+
+// ClusterDefinition
+type ClusterDefinition struct {
+	ID      string                 `json:"id,omitempty"`
+	Name    string                 `json:"name"`
+	Address string                 `json:"address"`
+	Details map[string]interface{} `json:"details,omitempty"`
+}
+
+// ClusterStorage interface defines an implementation prototype for a storage responsible
+// for ClusterDefinition instances
+type ClusterStorage interface {
+	// Add only if the key does not exist
+	AddIfNotExists(key string, cluster []byte) error
+
+	// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+	// value with the provided.
+	AddOrUpdate(key string, cluster []byte) error
+
+	// Removes a key from the cluster storage
+	Remove(key string) error
+
+	// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+	// an error, the iteration stops.
+	Each(handler func(string, []byte) error) error
+
+	// Closes the backing storage
+	Close() error
+}
+
+type ClusterManager struct {
+	storage ClusterStorage
+	// cache   map[string]*ClusterDefinition
+}
+
+// Creates a new ClusterManager instance using the provided storage
+func NewClusterManager(storage ClusterStorage) *ClusterManager {
+	return &ClusterManager{
+		storage: storage,
+	}
+}
+
+// Creates a new ClusterManager instance using the provided storage and populates a
+// yaml configured list of clusters
+func NewConfiguredClusterManager(storage ClusterStorage, config string) *ClusterManager {
+	clusterManager := NewClusterManager(storage)
+
+	exists, err := util.FileExists(config)
+	if !exists {
+		if err != nil {
+			klog.V(1).Infof("[Error] Failed to load config file: %s. Error: %s", config, err.Error())
+		}
+		return clusterManager
+	}
+
+	data, err := ioutil.ReadFile(config)
+	if err != nil {
+		return clusterManager
+	}
+
+	var entries []ClusterConfigEntry
+	err = yaml.Unmarshal(data, &entries)
+	if err != nil {
+		return clusterManager
+	}
+
+	for _, entry := range entries {
+		clusterManager.Add(ClusterDefinition{
+			ID:      entry.Name,
+			Name:    entry.Name,
+			Address: entry.Address,
+		})
+	}
+
+	return clusterManager
+}
+
+// Adds, but will not update an existing entry.
+func (cm *ClusterManager) Add(cluster ClusterDefinition) (*ClusterDefinition, error) {
+	// First time add
+	if cluster.ID == "" {
+		cluster.ID = uuid.New().String()
+	}
+
+	data, err := json.Marshal(cluster)
+	if err != nil {
+		return nil, err
+	}
+
+	err = cm.storage.AddIfNotExists(cluster.ID, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cluster, nil
+}
+
+func (cm *ClusterManager) AddOrUpdate(cluster ClusterDefinition) (*ClusterDefinition, error) {
+	// First time add
+	if cluster.ID == "" {
+		cluster.ID = uuid.New().String()
+	}
+
+	data, err := json.Marshal(cluster)
+	if err != nil {
+		return nil, err
+	}
+
+	err = cm.storage.AddOrUpdate(cluster.ID, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cluster, nil
+}
+
+func (cm *ClusterManager) Remove(id string) error {
+	return cm.storage.Remove(id)
+}
+
+func (cm *ClusterManager) GetAll() []*ClusterDefinition {
+	clusters := []*ClusterDefinition{}
+
+	err := cm.storage.Each(func(key string, cluster []byte) error {
+		var cd ClusterDefinition
+		err := json.Unmarshal(cluster, &cd)
+		if err != nil {
+			klog.V(1).Infof("[Error] Failed to unmarshal json cluster definition for key: %s", key)
+			return nil
+		}
+
+		clusters = append(clusters, &cd)
+		return nil
+	})
+
+	if err != nil {
+		klog.Infof("[Error] Failed to load list of clusters: %s", err.Error())
+	}
+
+	return clusters
+}
+
+func (cm *ClusterManager) Close() error {
+	return cm.storage.Close()
+}

pkg/clustermanager/clustersendpoints.go

@@ -0,0 +1,104 @@
+package clustermanager
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/julienschmidt/httprouter"
+
+	"k8s.io/klog"
+)
+
+// DataEnvelope is a generic wrapper struct for http response data
+type DataEnvelope struct {
+	Code   int         `json:"code"`
+	Status string      `json:"status"`
+	Data   interface{} `json:"data"`
+}
+
+type ClusterManagerEndpoints struct {
+	manager *ClusterManager
+}
+
+func NewClusterManagerEndpoints(manager *ClusterManager) *ClusterManagerEndpoints {
+	return &ClusterManagerEndpoints{
+		manager: manager,
+	}
+}
+
+func (cme *ClusterManagerEndpoints) GetAllClusters(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	clusters := cme.manager.GetAll()
+	w.Write(wrapData(clusters, nil))
+}
+
+func (cme *ClusterManagerEndpoints) PutCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		w.Write(wrapData(nil, err))
+		return
+	}
+
+	var clusterDef ClusterDefinition
+	err = json.Unmarshal(data, &clusterDef)
+	if err != nil {
+		w.Write(wrapData(nil, err))
+		return
+	}
+
+	cd, err := cme.manager.AddOrUpdate(clusterDef)
+	if err != nil {
+		w.Write(wrapData(nil, err))
+		return
+	}
+
+	w.Write(wrapData(cd, nil))
+}
+
+func (cme *ClusterManagerEndpoints) DeleteCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	clusterID := ps.ByName("id")
+	if clusterID == "" {
+		w.Write(wrapData(nil, errors.New("Failed to locate cluster with empty id.")))
+		return
+	}
+
+	err := cme.manager.Remove(clusterID)
+	if err != nil {
+		w.Write(wrapData(nil, err))
+		return
+	}
+
+	w.Write(wrapData("success", nil))
+}
+
+func wrapData(data interface{}, err error) []byte {
+	var resp []byte
+
+	if err != nil {
+		klog.V(1).Infof("Error returned to client: %s", err.Error())
+		resp, _ = json.Marshal(&DataEnvelope{
+			Code:   http.StatusInternalServerError,
+			Status: "error",
+			Data:   err.Error(),
+		})
+	} else {
+		resp, _ = json.Marshal(&DataEnvelope{
+			Code:   http.StatusOK,
+			Status: "success",
+			Data:   data,
+		})
+
+	}
+
+	return resp
+}

pkg/clustermanager/mapdbstorage.go

@@ -0,0 +1,55 @@
+package clustermanager
+
+import (
+	_ "k8s.io/klog"
+)
+
+type MapDBClusterStorage struct {
+	store map[string][]byte
+}
+
+func NewMapDBClusterStorage() ClusterStorage {
+	return &MapDBClusterStorage{
+		store: make(map[string][]byte),
+	}
+}
+
+// Adds the entry if the key does not exist
+func (cs *MapDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
+	if _, ok := cs.store[key]; !ok {
+		cs.store[key] = cluster
+	}
+	return nil
+}
+
+// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+// value with the provided.
+func (cs *MapDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
+	cs.store[key] = cluster
+	return nil
+}
+
+// Removes a key from the cluster storage
+func (cs *MapDBClusterStorage) Remove(key string) error {
+	delete(cs.store, key)
+	return nil
+}
+
+// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+// an error, the iteration stops.
+func (cs *MapDBClusterStorage) Each(handler func(string, []byte) error) error {
+	for k, v := range cs.store {
+		value := make([]byte, len(v))
+		copy(value, v)
+
+		if err := handler(k, value); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Closes the backing storage
+func (cs *MapDBClusterStorage) Close() error {
+	return nil
+}

costmodel/cluster.go → pkg/costmodel/cluster.go

@@ -6,8 +6,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/kubecost/cost-model/cloud"
-	"github.com/kubecost/cost-model/util"
+	"github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/util"
 	prometheus "github.com/prometheus/client_golang/api"
 	"k8s.io/klog"
 )
@@ -42,7 +42,7 @@ type PromQueryContext struct {
 }
 
 // TODO move this to a package-accessible helper function once dependencies are able to
-// be extricated from costmodel package (PromQueryResult -> Vector). Otherwise, circular deps.
+// be extricated from costmodel package (PromQueryResult -> util.Vector). Otherwise, circular deps.
 func AsyncPromQuery(query string, resultCh chan []*PromQueryResult, ctx PromQueryContext) {
 	if ctx.wg != nil {
 		defer ctx.wg.Done()

costmodel/costmodel.go → pkg/costmodel/costmodel.go

@@ -12,8 +12,9 @@ import (
 	"sync"
 	"time"
 
-	costAnalyzerCloud "github.com/kubecost/cost-model/cloud"
-	"github.com/kubecost/cost-model/clustercache"
+	costAnalyzerCloud "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/util"
 	prometheusClient "github.com/prometheus/client_golang/api"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -75,15 +76,15 @@ type CostData struct {
 	Daemonsets      []string                     `json:"daemonsets,omitempty"`
 	Statefulsets    []string                     `json:"statefulsets,omitempty"`
 	Jobs            []string                     `json:"jobs,omitempty"`
-	RAMReq          []*Vector                    `json:"ramreq,omitempty"`
-	RAMUsed         []*Vector                    `json:"ramused,omitempty"`
-	RAMAllocation   []*Vector                    `json:"ramallocated,omitempty"`
-	CPUReq          []*Vector                    `json:"cpureq,omitempty"`
-	CPUUsed         []*Vector                    `json:"cpuused,omitempty"`
-	CPUAllocation   []*Vector                    `json:"cpuallocated,omitempty"`
-	GPUReq          []*Vector                    `json:"gpureq,omitempty"`
+	RAMReq          []*util.Vector               `json:"ramreq,omitempty"`
+	RAMUsed         []*util.Vector               `json:"ramused,omitempty"`
+	RAMAllocation   []*util.Vector               `json:"ramallocated,omitempty"`
+	CPUReq          []*util.Vector               `json:"cpureq,omitempty"`
+	CPUUsed         []*util.Vector               `json:"cpuused,omitempty"`
+	CPUAllocation   []*util.Vector               `json:"cpuallocated,omitempty"`
+	GPUReq          []*util.Vector               `json:"gpureq,omitempty"`
 	PVCData         []*PersistentVolumeClaimData `json:"pvcData,omitempty"`
-	NetworkData     []*Vector                    `json:"network,omitempty"`
+	NetworkData     []*util.Vector               `json:"network,omitempty"`
 	Labels          map[string]string            `json:"labels,omitempty"`
 	NamespaceLabels map[string]string            `json:"namespaceLabels,omitempty"`
 	ClusterID       string                       `json:"clusterId"`
@@ -266,7 +267,7 @@ func ValidatePrometheus(cli prometheusClient.Client, isThanos bool) (*Prometheus
 	}
 }
 
-func getUptimeData(qr interface{}) ([]*Vector, bool, error) {
+func getUptimeData(qr interface{}) ([]*util.Vector, bool, error) {
 	data, ok := qr.(map[string]interface{})["data"]
 	if !ok {
 		e, err := wrapPrometheusError(qr)
@@ -283,7 +284,7 @@ func getUptimeData(qr interface{}) ([]*Vector, bool, error) {
 	if !ok {
 		return nil, false, fmt.Errorf("Improperly formatted results from prometheus, result field is not a slice")
 	}
-	jobData := []*Vector{}
+	jobData := []*util.Vector{}
 	kubecostMetrics := false
 	for _, val := range results {
 		// For now, just do this for validation. TODO: This can be parsed to figure out the exact running jobs.
@@ -308,7 +309,7 @@ func getUptimeData(qr interface{}) ([]*Vector, bool, error) {
 		}
 		strVal := dataPoint[1].(string)
 		v, _ := strconv.ParseFloat(strVal, 64)
-		toReturn := &Vector{
+		toReturn := &util.Vector{
 			Timestamp: dataPoint[0].(float64),
 			Value:     v,
 		}
@@ -499,6 +500,8 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 		return nil, err
 	}
 
+	// Unmounted PVs represent the PVs that are not mounted or tied to a volume on a container
+	unmountedPVs := make(map[string][]*PersistentVolumeClaimData)
 	pvClaimMapping, err := GetPVInfo(resultPVRequests, clusterID)
 	if err != nil {
 		klog.Infof("[Warning] Unable to get PV Data: %s", err.Error())
@@ -508,6 +511,10 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 		if err != nil {
 			return nil, err
 		}
+		// copy claim mappings into zombies, then remove as they're discovered
+		for k, v := range pvClaimMapping {
+			unmountedPVs[k] = []*PersistentVolumeClaimData{v}
+		}
 	}
 
 	networkUsageMap, err := GetNetworkUsageData(resultNetZoneRequests, resultNetRegionRequests, resultNetInternetRequests, clusterID)
@@ -611,13 +618,17 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 			for _, vol := range podClaims {
 				if vol.PersistentVolumeClaim != nil {
 					name := vol.PersistentVolumeClaim.ClaimName
-					if pvClaim, ok := pvClaimMapping[ns+","+name+","+clusterID]; ok {
+					key := ns + "," + name + "," + clusterID
+					if pvClaim, ok := pvClaimMapping[key]; ok {
 						podPVs = append(podPVs, pvClaim)
+
+						// Remove entry from potential unmounted pvs
+						delete(unmountedPVs, key)
 					}
 				}
 			}
 
-			var podNetCosts []*Vector
+			var podNetCosts []*util.Vector
 			if usage, ok := networkUsageMap[ns+","+podName+","+clusterID]; ok {
 				netCosts, err := GetNetworkCost(usage, cp)
 				if err != nil {
@@ -645,31 +656,31 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 				RAMReqV, ok := RAMReqMap[newKey]
 				if !ok {
 					klog.V(4).Info("no RAM requests for " + newKey)
-					RAMReqV = []*Vector{&Vector{}}
+					RAMReqV = []*util.Vector{&util.Vector{}}
 				}
 				RAMUsedV, ok := RAMUsedMap[newKey]
 				if !ok {
 					klog.V(4).Info("no RAM usage for " + newKey)
-					RAMUsedV = []*Vector{&Vector{}}
+					RAMUsedV = []*util.Vector{&util.Vector{}}
 				}
 				CPUReqV, ok := CPUReqMap[newKey]
 				if !ok {
 					klog.V(4).Info("no CPU requests for " + newKey)
-					CPUReqV = []*Vector{&Vector{}}
+					CPUReqV = []*util.Vector{&util.Vector{}}
 				}
 				GPUReqV, ok := GPUReqMap[newKey]
 				if !ok {
 					klog.V(4).Info("no GPU requests for " + newKey)
-					GPUReqV = []*Vector{&Vector{}}
+					GPUReqV = []*util.Vector{&util.Vector{}}
 				}
 				CPUUsedV, ok := CPUUsedMap[newKey]
 				if !ok {
 					klog.V(4).Info("no CPU usage for " + newKey)
-					CPUUsedV = []*Vector{&Vector{}}
+					CPUUsedV = []*util.Vector{&util.Vector{}}
 				}
 
 				var pvReq []*PersistentVolumeClaimData
-				var netReq []*Vector
+				var netReq []*util.Vector
 				if i == 0 { // avoid duplicating by just assigning all claims to the first container.
 					pvReq = podPVs
 					netReq = podNetCosts
@@ -715,27 +726,27 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 			RAMReqV, ok := RAMReqMap[key]
 			if !ok {
 				klog.V(4).Info("no RAM requests for " + key)
-				RAMReqV = []*Vector{&Vector{}}
+				RAMReqV = []*util.Vector{&util.Vector{}}
 			}
 			RAMUsedV, ok := RAMUsedMap[key]
 			if !ok {
 				klog.V(4).Info("no RAM usage for " + key)
-				RAMUsedV = []*Vector{&Vector{}}
+				RAMUsedV = []*util.Vector{&util.Vector{}}
 			}
 			CPUReqV, ok := CPUReqMap[key]
 			if !ok {
 				klog.V(4).Info("no CPU requests for " + key)
-				CPUReqV = []*Vector{&Vector{}}
+				CPUReqV = []*util.Vector{&util.Vector{}}
 			}
 			GPUReqV, ok := GPUReqMap[key]
 			if !ok {
 				klog.V(4).Info("no GPU requests for " + key)
-				GPUReqV = []*Vector{&Vector{}}
+				GPUReqV = []*util.Vector{&util.Vector{}}
 			}
 			CPUUsedV, ok := CPUUsedMap[key]
 			if !ok {
 				klog.V(4).Info("no CPU usage for " + key)
-				CPUUsedV = []*Vector{&Vector{}}
+				CPUUsedV = []*util.Vector{&util.Vector{}}
 			}
 
 			node, ok := nodes[c.NodeName]
@@ -778,11 +789,24 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 			}
 		}
 	}
-	err = findDeletedNodeInfo(cli, missingNodes, window)
+	// Use unmounted pvs to create a mapping of "Unmounted-<Namespace>" containers
+	// to pass along the cost data
+	unmounted := findUnmountedPVCostData(unmountedPVs, namespaceLabelsMapping)
+	for k, costs := range unmounted {
+		klog.V(3).Infof("Unmounted PVs in Namespace/ClusterID: %s/%s", costs.Namespace, costs.ClusterID)
+
+		if filterNamespace == "" {
+			containerNameCost[k] = costs
+		} else if costs.Namespace == filterNamespace {
+			containerNameCost[k] = costs
+		}
+	}
 
+	err = findDeletedNodeInfo(cli, missingNodes, window)
 	if err != nil {
 		klog.V(1).Infof("Error fetching historical node data: %s", err.Error())
 	}
+
 	err = findDeletedPodInfo(cli, missingContainers, window)
 	if err != nil {
 		klog.V(1).Infof("Error fetching historical pod data: %s", err.Error())
@@ -790,6 +814,50 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, clientset kube
 	return containerNameCost, err
 }
 
+func findUnmountedPVCostData(unmountedPVs map[string][]*PersistentVolumeClaimData, namespaceLabelsMapping map[string]map[string]string) map[string]*CostData {
+	costs := make(map[string]*CostData)
+	if len(unmountedPVs) == 0 {
+		return costs
+	}
+
+	for k, pv := range unmountedPVs {
+		keyParts := strings.Split(k, ",")
+		if len(keyParts) != 3 {
+			klog.V(1).Infof("Unmounted PV used key with incorrect parts: %s", k)
+			continue
+		}
+
+		ns, _, clusterID := keyParts[0], keyParts[1], keyParts[2]
+
+		namespacelabels, ok := namespaceLabelsMapping[ns+","+clusterID]
+		if !ok {
+			klog.V(3).Infof("Missing data for namespace %s", ns)
+		}
+
+		// Should be a unique "Unmounted" cost data type
+		name := "unmounted-pvs"
+
+		metric := newContainerMetricFromValues(ns, name, name, "", clusterID)
+		key := metric.Key()
+
+		if costData, ok := costs[key]; !ok {
+			costs[key] = &CostData{
+				Name:            name,
+				PodName:         name,
+				NodeName:        "",
+				Namespace:       ns,
+				NamespaceLabels: namespacelabels,
+				ClusterID:       clusterID,
+				PVCData:         pv,
+			}
+		} else {
+			costData.PVCData = append(costData.PVCData, pv...)
+		}
+	}
+
+	return costs
+}
+
 func findDeletedPodInfo(cli prometheusClient.Client, missingContainers map[string]*CostData, window string) error {
 	if len(missingContainers) > 0 {
 		queryHistoricalPodLabels := fmt.Sprintf(`kube_pod_labels{}[%s]`, window)
@@ -935,9 +1003,9 @@ func findDeletedNodeInfo(cli prometheusClient.Client, missingNodes map[string]*c
 	return nil
 }
 
-func getContainerAllocation(req []*Vector, used []*Vector, allocationType string) []*Vector {
-	// The result of the normalize operation will be a new []*Vector to replace the requests
-	allocationOp := func(r *Vector, x *float64, y *float64) bool {
+func getContainerAllocation(req []*util.Vector, used []*util.Vector, allocationType string) []*util.Vector {
+	// The result of the normalize operation will be a new []*util.Vector to replace the requests
+	allocationOp := func(r *util.Vector, x *float64, y *float64) bool {
 		if x != nil && y != nil {
 			x1 := *x
 			if math.IsNaN(x1) {
@@ -960,7 +1028,7 @@ func getContainerAllocation(req []*Vector, used []*Vector, allocationType string
 		return true
 	}
 
-	return ApplyVectorOp(req, used, allocationOp)
+	return util.ApplyVectorOp(req, used, allocationOp)
 }
 
 func addPVData(cache clustercache.ClusterCache, pvClaimMapping map[string]*PersistentVolumeClaimData, cloud costAnalyzerCloud.Provider) error {
@@ -968,6 +1036,13 @@ func addPVData(cache clustercache.ClusterCache, pvClaimMapping map[string]*Persi
 	if err != nil {
 		return err
 	}
+	// Pull a region from the first node
+	var defaultRegion string
+	nodeList := cache.GetAllNodes()
+	if len(nodeList) > 0 {
+		defaultRegion = nodeList[0].Labels[v1.LabelZoneRegion]
+	}
+
 	storageClasses := cache.GetAllStorageClasses()
 	storageClassMap := make(map[string]map[string]string)
 	for _, storageClass := range storageClasses {
@@ -986,12 +1061,19 @@ func addPVData(cache clustercache.ClusterCache, pvClaimMapping map[string]*Persi
 		if !ok {
 			klog.V(4).Infof("Unable to find parameters for storage class \"%s\". Does pv \"%s\" have a storageClassName?", pv.Spec.StorageClassName, pv.Name)
 		}
+		var region string
+		if r, ok := pv.Labels[v1.LabelZoneRegion]; ok {
+			region = r
+		} else {
+			region = defaultRegion
+		}
+
 		cacPv := &costAnalyzerCloud.PV{
 			Class:      pv.Spec.StorageClassName,
-			Region:     pv.Labels[v1.LabelZoneRegion],
+			Region:     region,
 			Parameters: parameters,
 		}
-		err := GetPVCost(cacPv, pv, cloud)
+		err := GetPVCost(cacPv, pv, cloud, region)
 		if err != nil {
 			return err
 		}
@@ -1008,15 +1090,16 @@ func addPVData(cache clustercache.ClusterCache, pvClaimMapping map[string]*Persi
 			}
 		}
 	}
+
 	return nil
 }
 
-func GetPVCost(pv *costAnalyzerCloud.PV, kpv *v1.PersistentVolume, cp costAnalyzerCloud.Provider) error {
+func GetPVCost(pv *costAnalyzerCloud.PV, kpv *v1.PersistentVolume, cp costAnalyzerCloud.Provider, defaultRegion string) error {
 	cfg, err := cp.GetConfig()
 	if err != nil {
 		return err
 	}
-	key := cp.GetPVKey(kpv, pv.Parameters)
+	key := cp.GetPVKey(kpv, pv.Parameters, defaultRegion)
 	pvWithCost, err := cp.PVPricing(key)
 	if err != nil {
 		pv.Cost = cfg.Storage
@@ -1058,6 +1141,9 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			}
 		}
 		newCnode := *cnode
+		if newCnode.InstanceType == "" {
+			newCnode.InstanceType = n.Labels[v1.LabelInstanceType]
+		}
 
 		var cpu float64
 		if newCnode.VCPU == "" {
@@ -1929,12 +2015,16 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, clientset kubern
 		klog.V(1).Infof("Unable to get PV Hourly Cost Data: %s", err.Error())
 	}
 
+	unmountedPVs := make(map[string][]*PersistentVolumeClaimData)
 	pvAllocationMapping, err := GetPVAllocationMetrics(pvPodAllocationResults, clusterID)
 	if err != nil {
 		klog.V(1).Infof("Unable to get PV Allocation Cost Data: %s", err.Error())
 	}
 	if pvAllocationMapping != nil {
 		addMetricPVData(pvAllocationMapping, pvCostMapping, cp)
+		for k, v := range pvAllocationMapping {
+			unmountedPVs[k] = v
+		}
 	}
 
 	measureTime(profileStart, profileThreshold, fmt.Sprintf("costDataRange(%fh): process PV data", durHrs))
@@ -2092,37 +2182,37 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, clientset kubern
 		RAMReqV, ok := RAMReqMap[key]
 		if !ok {
 			klog.V(4).Info("no RAM requests for " + key)
-			RAMReqV = []*Vector{}
+			RAMReqV = []*util.Vector{}
 		}
 		RAMUsedV, ok := RAMUsedMap[key]
 		if !ok {
 			klog.V(4).Info("no RAM usage for " + key)
-			RAMUsedV = []*Vector{}
+			RAMUsedV = []*util.Vector{}
 		}
 		CPUReqV, ok := CPUReqMap[key]
 		if !ok {
 			klog.V(4).Info("no CPU requests for " + key)
-			CPUReqV = []*Vector{}
+			CPUReqV = []*util.Vector{}
 		}
 		CPUUsedV, ok := CPUUsedMap[key]
 		if !ok {
 			klog.V(4).Info("no CPU usage for " + key)
-			CPUUsedV = []*Vector{}
+			CPUUsedV = []*util.Vector{}
 		}
 		RAMAllocsV, ok := RAMAllocMap[key]
 		if !ok {
 			klog.V(4).Info("no RAM allocation for " + key)
-			RAMAllocsV = []*Vector{}
+			RAMAllocsV = []*util.Vector{}
 		}
 		CPUAllocsV, ok := CPUAllocMap[key]
 		if !ok {
 			klog.V(4).Info("no CPU allocation for " + key)
-			CPUAllocsV = []*Vector{}
+			CPUAllocsV = []*util.Vector{}
 		}
 		GPUReqV, ok := GPUReqMap[key]
 		if !ok {
 			klog.V(4).Info("no GPU requests for " + key)
-			GPUReqV = []*Vector{}
+			GPUReqV = []*util.Vector{}
 		}
 
 		var node *costAnalyzerCloud.Node
@@ -2179,7 +2269,7 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, clientset kubern
 		}
 
 		var podPVs []*PersistentVolumeClaimData
-		var podNetCosts []*Vector
+		var podNetCosts []*util.Vector
 
 		// For PVC data, we'll need to find the claim mapping and cost data. Will need to append
 		// cost data since that was populated by cluster data previously. We do this with
@@ -2189,9 +2279,12 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, clientset kubern
 			klog.V(4).Infof("Failed to locate pv allocation mapping for missing pod.")
 		}
 
+		// Delete the current pod key from potentially unmounted pvs
+		delete(unmountedPVs, podKey)
+
 		// For network costs, we'll use existing map since it should still contain the
 		// correct data.
-		var podNetworkCosts []*Vector
+		var podNetworkCosts []*util.Vector
 		if usage, ok := networkUsageMap[podKey]; ok {
 			netCosts, err := GetNetworkCost(usage, cp)
 			if err != nil {
@@ -2247,6 +2340,15 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, clientset kubern
 
 	measureTime(profileStart, profileThreshold, fmt.Sprintf("costDataRange(%fh): build CostData map", durHrs))
 
+	unmounted := findUnmountedPVCostData(unmountedPVs, namespaceLabelsMapping)
+	for k, costs := range unmounted {
+		klog.V(3).Infof("Unmounted PVs in Namespace/ClusterID: %s/%s", costs.Namespace, costs.ClusterID)
+
+		if costDataPassesFilters(costs, filterNamespace, filterCluster) {
+			containerNameCost[k] = costs
+		}
+	}
+
 	w := end.Sub(start)
 	w += window
 	if w.Minutes() > 0 {
@@ -2260,9 +2362,9 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, clientset kubern
 	return containerNameCost, err
 }
 
-func applyAllocationToRequests(allocationMap map[string][]*Vector, requestMap map[string][]*Vector) {
-	// The result of the normalize operation will be a new []*Vector to replace the requests
-	normalizeOp := func(r *Vector, x *float64, y *float64) bool {
+func applyAllocationToRequests(allocationMap map[string][]*util.Vector, requestMap map[string][]*util.Vector) {
+	// The result of the normalize operation will be a new []*util.Vector to replace the requests
+	normalizeOp := func(r *util.Vector, x *float64, y *float64) bool {
 		// Omit data (return false) if both x and y inputs don't exist
 		if x == nil || y == nil {
 			return false
@@ -2289,7 +2391,7 @@ func applyAllocationToRequests(allocationMap map[string][]*Vector, requestMap ma
 		}
 
 		// Replace request map with normalized
-		requestMap[k] = ApplyVectorOp(allocations, requests, normalizeOp)
+		requestMap[k] = util.ApplyVectorOp(allocations, requests, normalizeOp)
 	}
 }
 
@@ -2375,11 +2477,11 @@ type PersistentVolumeClaimData struct {
 	ClusterID  string                `json:"clusterId"`
 	VolumeName string                `json:"volumeName"`
 	Volume     *costAnalyzerCloud.PV `json:"persistentVolume"`
-	Values     []*Vector             `json:"values"`
+	Values     []*util.Vector        `json:"values"`
 }
 
-func getCost(qr interface{}) (map[string][]*Vector, error) {
-	toReturn := make(map[string][]*Vector)
+func getCost(qr interface{}) (map[string][]*util.Vector, error) {
+	toReturn := make(map[string][]*util.Vector)
 	result, err := NewQueryResults(qr)
 	if err != nil {
 		return toReturn, err
@@ -2475,14 +2577,14 @@ func getNormalization(qr interface{}) (float64, error) {
 }
 
 //todo: don't cast, implement unmarshaler interface
-func getNormalizations(qr interface{}) ([]*Vector, error) {
+func getNormalizations(qr interface{}) ([]*util.Vector, error) {
 	queryResults, err := NewQueryResults(qr)
 	if err != nil {
 		return nil, err
 	}
 
 	if len(queryResults) > 0 {
-		vectors := []*Vector{}
+		vectors := []*util.Vector{}
 		for _, value := range queryResults {
 			vectors = append(vectors, value.Values...)
 		}
@@ -2615,13 +2717,13 @@ func NewKeyTuple(key string) (*KeyTuple, error) {
 	}, nil
 }
 
-func GetContainerMetricVector(qr interface{}, normalize bool, normalizationValue float64, defaultClusterID string) (map[string][]*Vector, error) {
+func GetContainerMetricVector(qr interface{}, normalize bool, normalizationValue float64, defaultClusterID string) (map[string][]*util.Vector, error) {
 	result, err := NewQueryResults(qr)
 	if err != nil {
 		return nil, err
 	}
 
-	containerData := make(map[string][]*Vector)
+	containerData := make(map[string][]*util.Vector)
 	for _, val := range result {
 		containerMetric, err := newContainerMetricFromPrometheus(val.Metric, defaultClusterID)
 		if err != nil {
@@ -2638,13 +2740,13 @@ func GetContainerMetricVector(qr interface{}, normalize bool, normalizationValue
 	return containerData, nil
 }
 
-func GetContainerMetricVectors(qr interface{}, defaultClusterID string) (map[string][]*Vector, error) {
+func GetContainerMetricVectors(qr interface{}, defaultClusterID string) (map[string][]*util.Vector, error) {
 	result, err := NewQueryResults(qr)
 	if err != nil {
 		return nil, err
 	}
 
-	containerData := make(map[string][]*Vector)
+	containerData := make(map[string][]*util.Vector)
 	for _, val := range result {
 		containerMetric, err := newContainerMetricFromPrometheus(val.Metric, defaultClusterID)
 		if err != nil {
@@ -2655,19 +2757,19 @@ func GetContainerMetricVectors(qr interface{}, defaultClusterID string) (map[str
 	return containerData, nil
 }
 
-func GetNormalizedContainerMetricVectors(qr interface{}, normalizationValues []*Vector, defaultClusterID string) (map[string][]*Vector, error) {
+func GetNormalizedContainerMetricVectors(qr interface{}, normalizationValues []*util.Vector, defaultClusterID string) (map[string][]*util.Vector, error) {
 	result, err := NewQueryResults(qr)
 	if err != nil {
 		return nil, err
 	}
 
-	containerData := make(map[string][]*Vector)
+	containerData := make(map[string][]*util.Vector)
 	for _, val := range result {
 		containerMetric, err := newContainerMetricFromPrometheus(val.Metric, defaultClusterID)
 		if err != nil {
 			return nil, err
 		}
-		containerData[containerMetric.Key()] = NormalizeVectorByVector(val.Values, normalizationValues)
+		containerData[containerMetric.Key()] = util.NormalizeVectorByVector(val.Values, normalizationValues)
 	}
 	return containerData, nil
 }

costmodel/metrics.go → pkg/costmodel/metrics.go

@@ -23,13 +23,13 @@ func kubeLabelsToPrometheusLabels(labels map[string]string) ([]string, []string)
 
 	labelValues := make([]string, 0, len(labels))
 	for i, k := range labelKeys {
-		labelKeys[i] = "label_" + sanitizeLabelName(k)
+		labelKeys[i] = "label_" + SanitizeLabelName(k)
 		labelValues = append(labelValues, labels[k])
 	}
 	return labelKeys, labelValues
 }
 
-func sanitizeLabelName(s string) string {
+func SanitizeLabelName(s string) string {
 	return invalidLabelCharRE.ReplaceAllString(s, "_")
 }
 

costmodel/networkcosts.go → pkg/costmodel/networkcosts.go

@@ -1,7 +1,8 @@
 package costmodel
 
 import (
-	costAnalyzerCloud "github.com/kubecost/cost-model/cloud"
+	costAnalyzerCloud "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/util"
 	"k8s.io/klog"
 )
 
@@ -10,9 +11,9 @@ type NetworkUsageData struct {
 	ClusterID             string
 	PodName               string
 	Namespace             string
-	NetworkZoneEgress     []*Vector
-	NetworkRegionEgress   []*Vector
-	NetworkInternetEgress []*Vector
+	NetworkZoneEgress     []*util.Vector
+	NetworkRegionEgress   []*util.Vector
+	NetworkInternetEgress []*util.Vector
 }
 
 // NetworkUsageVector contains a network usage vector for egress network traffic
@@ -20,7 +21,7 @@ type NetworkUsageVector struct {
 	ClusterID string
 	PodName   string
 	Namespace string
-	Values    []*Vector
+	Values    []*util.Vector
 }
 
 // GetNetworkUsageData performs a join of the the results of zone, region, and internet usage queries to return a single
@@ -91,8 +92,8 @@ func GetNetworkUsageData(zr interface{}, rr interface{}, ir interface{}, default
 }
 
 // GetNetworkCost computes the actual cost for NetworkUsageData based on data provided by the Provider.
-func GetNetworkCost(usage *NetworkUsageData, cloud costAnalyzerCloud.Provider) ([]*Vector, error) {
-	var results []*Vector
+func GetNetworkCost(usage *NetworkUsageData, cloud costAnalyzerCloud.Provider) ([]*util.Vector, error) {
+	var results []*util.Vector
 
 	pricing, err := cloud.NetworkPricing()
 	if err != nil {
@@ -126,7 +127,7 @@ func GetNetworkCost(usage *NetworkUsageData, cloud costAnalyzerCloud.Provider) (
 			timestamp = usage.NetworkInternetEgress[i].Timestamp
 		}
 
-		results = append(results, &Vector{
+		results = append(results, &util.Vector{
 			Value:     cost,
 			Timestamp: timestamp,
 		})

costmodel/promparsers.go → pkg/costmodel/promparsers.go

@@ -6,14 +6,15 @@ import (
 	"strconv"
 	"strings"
 
-	costAnalyzerCloud "github.com/kubecost/cost-model/cloud"
+	costAnalyzerCloud "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/util"
 	"k8s.io/klog"
 )
 
 // PromQueryResult contains a single result from a prometheus query
 type PromQueryResult struct {
 	Metric map[string]interface{}
-	Values []*Vector
+	Values []*util.Vector
 }
 
 func (pqr *PromQueryResult) GetString(field string) (string, error) {
@@ -102,7 +103,7 @@ func NewQueryResults(queryResult interface{}) ([]*PromQueryResult, error) {
 		// Determine if the result is a ranged data set or single value
 		_, isRange := resultInterface["values"]
 
-		var vectors []*Vector
+		var vectors []*util.Vector
 		if !isRange {
 			dataPoint, ok := resultInterface["value"]
 			if !ok {
@@ -139,7 +140,7 @@ func NewQueryResults(queryResult interface{}) ([]*PromQueryResult, error) {
 	return result, nil
 }
 
-func parseDataPoint(dataPoint interface{}, labels func() string) (*Vector, error) {
+func parseDataPoint(dataPoint interface{}, labels func() string) (*util.Vector, error) {
 	value, ok := dataPoint.([]interface{})
 	if !ok || len(value) != 2 {
 		return nil, fmt.Errorf("Improperly formatted datapoint from Prometheus")
@@ -160,7 +161,7 @@ func parseDataPoint(dataPoint interface{}, labels func() string) (*Vector, error
 		v = 0.0
 	}
 
-	return &Vector{
+	return &util.Vector{
 		Timestamp: math.Round(value[0].(float64)/10) * 10,
 		Value:     v,
 	}, nil
@@ -343,7 +344,14 @@ func GetPodLabelsMetrics(queryResult interface{}, defaultClusterID string) (map[
 		}
 
 		nsKey := ns + "," + pod + "," + clusterID
-		toReturn[nsKey] = val.GetLabels()
+		if labels, ok := toReturn[nsKey]; ok {
+			newlabels := val.GetLabels()
+			for k, v := range newlabels {
+				labels[k] = v
+			}
+		} else {
+			toReturn[nsKey] = val.GetLabels()
+		}
 	}
 
 	return toReturn, nil

costmodel/router.go → pkg/costmodel/router.go

@@ -17,15 +17,18 @@ import (
 	"k8s.io/klog"
 
 	"github.com/julienschmidt/httprouter"
-	costAnalyzerCloud "github.com/kubecost/cost-model/cloud"
-	"github.com/kubecost/cost-model/clustercache"
-	"github.com/patrickmn/go-cache"
+	costAnalyzerCloud "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	cm "github.com/kubecost/cost-model/pkg/clustermanager"
 	prometheusClient "github.com/prometheus/client_golang/api"
 	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
-	"github.com/prometheus/client_golang/prometheus"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	bolt "github.com/etcd-io/bbolt"
+	"github.com/patrickmn/go-cache"
+	"github.com/prometheus/client_golang/prometheus"
+
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
@@ -48,6 +51,7 @@ type Accesses struct {
 	PrometheusClient              prometheusClient.Client
 	ThanosClient                  prometheusClient.Client
 	KubeClientSet                 kubernetes.Interface
+	ClusterManager                *cm.ClusterManager
 	Cloud                         costAnalyzerCloud.Provider
 	CPUPriceRecorder              *prometheus.GaugeVec
 	RAMPriceRecorder              *prometheus.GaugeVec
@@ -422,6 +426,26 @@ func (a *Accesses) CostDataModelRangeLarge(w http.ResponseWriter, r *http.Reques
 	w.Write(WrapData(data, err))
 }
 
+func parseAggregations(customAggregation, aggregator, filterType string) (string, []string, string) {
+	var key string
+	var filter string
+	var val []string
+	if customAggregation != "" {
+		key = customAggregation
+		filter = filterType
+		val = strings.Split(customAggregation, ",")
+	} else {
+		aggregations := strings.Split(aggregator, ",")
+		for i, agg := range aggregations {
+			aggregations[i] = "kubernetes_" + agg
+		}
+		key = strings.Join(aggregations, ",")
+		filter = "kubernetes_" + filterType
+		val = aggregations
+	}
+	return key, val, filter
+}
+
 func (a *Accesses) OutofClusterCosts(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -434,11 +458,8 @@ func (a *Accesses) OutofClusterCosts(w http.ResponseWriter, r *http.Request, ps
 	filterValue := r.URL.Query().Get("filterValue")
 	var data []*costAnalyzerCloud.OutOfClusterAllocation
 	var err error
-	if customAggregation != "" {
-		data, err = a.Cloud.ExternalAllocations(start, end, customAggregation, filterType, filterValue)
-	} else {
-		data, err = a.Cloud.ExternalAllocations(start, end, "kubernetes_"+aggregator, "kubernetes_"+filterType, filterValue)
-	}
+	_, aggregations, filter := parseAggregations(customAggregation, aggregator, filterType)
+	data, err = a.Cloud.ExternalAllocations(start, end, aggregations, filter, filterValue)
 	w.Write(WrapData(data, err))
 }
 
@@ -464,11 +485,7 @@ func (a *Accesses) OutOfClusterCostsWithCache(w http.ResponseWriter, r *http.Req
 	filterType := r.URL.Query().Get("filterType")
 	filterValue := r.URL.Query().Get("filterValue")
 
-	aggregation := "kubernetes_" + kubernetesAggregation
-	filterType = "kubernetes_" + filterType
-	if customAggregation != "" {
-		aggregation = customAggregation
-	}
+	aggregationkey, aggregation, filter := parseAggregations(customAggregation, kubernetesAggregation, filterType)
 
 	// clear cache prior to checking the cache so that a clearCache=true
 	// request always returns a freshly computed value
@@ -477,7 +494,7 @@ func (a *Accesses) OutOfClusterCostsWithCache(w http.ResponseWriter, r *http.Req
 	}
 
 	// attempt to retrieve cost data from cache
-	key := fmt.Sprintf(`%s:%s:%s:%s:%s`, start, end, aggregation, filterType, filterValue)
+	key := fmt.Sprintf(`%s:%s:%s:%s:%s`, start, end, aggregationkey, filter, filterValue)
 	if value, found := a.OutOfClusterCache.Get(key); found && !disableCache {
 		if data, ok := value.([]*costAnalyzerCloud.OutOfClusterAllocation); ok {
 			w.Write(WrapDataWithMessage(data, nil, fmt.Sprintf("out of cluster cache hit: %s", key)))
@@ -486,7 +503,7 @@ func (a *Accesses) OutOfClusterCostsWithCache(w http.ResponseWriter, r *http.Req
 		klog.Errorf("caching error: failed to type cast data: %s", key)
 	}
 
-	data, err := a.Cloud.ExternalAllocations(start, end, aggregation, filterType, filterValue)
+	data, err := a.Cloud.ExternalAllocations(start, end, aggregation, filter, filterValue)
 	if err == nil {
 		a.OutOfClusterCache.Set(key, data, cache.DefaultExpiration)
 	}
@@ -622,6 +639,12 @@ func (a *Accesses) recordPrices() {
 			return strings.Split(key, ",")
 		}
 
+		var defaultRegion string = ""
+		nodeList := a.Model.Cache.GetAllNodes()
+		if len(nodeList) > 0 {
+			defaultRegion = nodeList[0].Labels[v1.LabelZoneRegion]
+		}
+
 		for {
 			klog.V(4).Info("Recording prices...")
 			podlist := a.Model.Cache.GetAllPods()
@@ -685,13 +708,14 @@ func (a *Accesses) recordPrices() {
 						gpuCost = 0
 					}
 				}
+				nodeType := node.InstanceType
 
 				totalCost := cpu*cpuCost + ramCost*(ram/1024/1024/1024) + gpu*gpuCost
 
-				a.CPUPriceRecorder.WithLabelValues(nodeName, nodeName).Set(cpuCost)
-				a.RAMPriceRecorder.WithLabelValues(nodeName, nodeName).Set(ramCost)
-				a.GPUPriceRecorder.WithLabelValues(nodeName, nodeName).Set(gpuCost)
-				a.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName).Set(totalCost)
+				a.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(cpuCost)
+				a.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(ramCost)
+				a.GPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(gpuCost)
+				a.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(totalCost)
 				labelKey := getKeyFromLabelStrings(nodeName, nodeName)
 				nodeSeen[labelKey] = true
 			}
@@ -747,12 +771,18 @@ func (a *Accesses) recordPrices() {
 					if !ok {
 						klog.V(4).Infof("Unable to find parameters for storage class \"%s\". Does pv \"%s\" have a storageClassName?", pv.Spec.StorageClassName, pv.Name)
 					}
+					var region string
+					if r, ok := pv.Labels[v1.LabelZoneRegion]; ok {
+						region = r
+					} else {
+						region = defaultRegion
+					}
 					cacPv := &costAnalyzerCloud.PV{
 						Class:      pv.Spec.StorageClassName,
-						Region:     pv.Labels[v1.LabelZoneRegion],
+						Region:     region,
 						Parameters: parameters,
 					}
-					GetPVCost(cacPv, pv, a.Cloud)
+					GetPVCost(cacPv, pv, a.Cloud, region)
 					c, _ := strconv.ParseFloat(cacPv.Cost, 64)
 					a.PersistentVolumePriceRecorder.WithLabelValues(pv.Name, pv.Name).Set(c)
 					labelKey := getKeyFromLabelStrings(pv.Name, pv.Name)
@@ -807,7 +837,33 @@ func (a *Accesses) recordPrices() {
 	}()
 }
 
-func Initialize() {
+// Creates a new ClusterManager instance using a boltdb storage. If that fails,
+// then we fall back to a memory-only storage.
+func newClusterManager() *cm.ClusterManager {
+	clustersConfigFile := "/var/configs/clusters/default-clusters.yaml"
+
+	path := os.Getenv("CONFIG_PATH")
+	db, err := bolt.Open(path+"costmodel.db", 0600, nil)
+	if err != nil {
+		klog.V(1).Infof("[Error] Failed to create costmodel.db: %s", err.Error())
+		return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
+	}
+
+	store, err := cm.NewBoltDBClusterStorage("clusters", db)
+	if err != nil {
+		klog.V(1).Infof("[Error] Failed to Create Cluster Storage: %s", err.Error())
+		return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
+	}
+
+	return cm.NewConfiguredClusterManager(store, clustersConfigFile)
+}
+
+type ConfigWatchers struct {
+	ConfigmapName string
+	WatchFunc     func(string, map[string]string) error
+}
+
+func Initialize(additionalConfigWatchers ...ConfigWatchers) {	
 	klog.InitFlags(nil)
 	flag.Set("v", "3")
 	flag.Parse()
@@ -871,7 +927,15 @@ func Initialize() {
 		if conf.GetName() == "pricing-configs" {
 			_, err := cloudProvider.UpdateConfigFromConfigMap(conf.Data)
 			if err != nil {
-				klog.Infof("ERROR UPDATING CONFIG: %s", err.Error())
+				klog.Infof("ERROR UPDATING %s CONFIG: %s", "pricing-configs", err.Error())
+			}
+		}
+		for _, cw := range additionalConfigWatchers {
+			if conf.GetName() == cw.ConfigmapName {
+				err := cw.WatchFunc(conf.GetName(), conf.Data)
+				if err != nil {
+					klog.Infof("ERROR UPDATING %s CONFIG: %s", cw.ConfigmapName, err.Error())
+				}
 			}
 		}
 	}
@@ -879,32 +943,47 @@ func Initialize() {
 	// We need an initial invocation because the init of the cache has happened before we had access to the provider.
 	configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get("pricing-configs", metav1.GetOptions{})
 	if err != nil {
-		klog.Infof("No configs found at installtime, using existing configs: %s", err.Error())
+		klog.Infof("No %s configmap found at installtime, using existing configs: %s", "pricing-configs", err.Error())
 	} else {
 		watchConfigFunc(configs)
 	}
 
+	for _, cw := range additionalConfigWatchers {
+		configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(cw.ConfigmapName, metav1.GetOptions{})
+		if err != nil {
+			klog.Infof("No %s configmap found at installtime, using existing configs: %s", cw.ConfigmapName, err.Error())
+		} else {
+			watchConfigFunc(configs)
+		}
+	}
+
 	k8sCache.SetConfigMapUpdateFunc(watchConfigFunc)
 
+	// TODO: General Architecture Note: Several passes have been made to modularize a lot of
+	// TODO: our code, but the router still continues to be the obvious entry point for new \
+	// TODO: features. We should look to spliting out the actual "router" functionality and
+	// TODO: implement a builder -> controller for stitching new features and other dependencies.
+	clusterManager := newClusterManager()
+
 	cpuGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_cpu_hourly_cost",
 		Help: "node_cpu_hourly_cost hourly cost for each cpu on this node",
-	}, []string{"instance", "node"})
+	}, []string{"instance", "node", "instance_type"})
 
 	ramGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_ram_hourly_cost",
 		Help: "node_ram_hourly_cost hourly cost for each gb of ram on this node",
-	}, []string{"instance", "node"})
+	}, []string{"instance", "node", "instance_type"})
 
 	gpuGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_gpu_hourly_cost",
 		Help: "node_gpu_hourly_cost hourly cost for each gpu on this node",
-	}, []string{"instance", "node"})
+	}, []string{"instance", "node", "instance_type"})
 
 	totalGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_total_hourly_cost",
 		Help: "node_total_hourly_cost Total node cost per hour",
-	}, []string{"instance", "node"})
+	}, []string{"instance", "node", "instance_type"})
 
 	pvGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "pv_hourly_cost",
@@ -975,6 +1054,7 @@ func Initialize() {
 	A = Accesses{
 		PrometheusClient:              promCli,
 		KubeClientSet:                 kubeClientset,
+		ClusterManager:                clusterManager,
 		Cloud:                         cloudProvider,
 		CPUPriceRecorder:              cpuGv,
 		RAMPriceRecorder:              ramGv,
@@ -1047,6 +1127,8 @@ func Initialize() {
 
 	A.recordPrices()
 
+	managerEndpoints := cm.NewClusterManagerEndpoints(A.ClusterManager)
+
 	Router.GET("/costDataModel", A.CostDataModel)
 	Router.GET("/costDataModelRange", A.CostDataModelRange)
 	Router.GET("/costDataModelRangeLarge", A.CostDataModelRangeLarge)
@@ -1059,4 +1141,7 @@ func Initialize() {
 	Router.GET("/managementPlatform", A.ManagementPlatform)
 	Router.GET("/clusterInfo", A.ClusterInfo)
 	Router.GET("/containerUptimes", A.ContainerUptimes)
+	Router.GET("/clusters", managerEndpoints.GetAllClusters)
+	Router.PUT("/clusters", managerEndpoints.PutCluster)
+	Router.DELETE("/clusters/:id", managerEndpoints.DeleteCluster)
 }

costmodel/sql.go → pkg/costmodel/sql.go

@@ -9,7 +9,8 @@ import (
 
 	"k8s.io/klog"
 
-	costAnalyzerCloud "github.com/kubecost/cost-model/cloud"
+	costAnalyzerCloud "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/util"
 	_ "github.com/lib/pq"
 )
 
@@ -141,7 +142,7 @@ func CostDataRangeFromSQL(field string, value string, window string, start strin
 
 		k := newContainerMetricFromValues(namespace, pod, container, instance, clusterid)
 		key := k.Key()
-		allocationVector := &Vector{
+		allocationVector := &util.Vector{
 			Timestamp: float64(t.Unix()),
 			Value:     sum,
 		}
@@ -163,9 +164,9 @@ func CostDataRangeFromSQL(field string, value string, window string, start strin
 				PodName:       pod,
 				NodeName:      instance,
 				NodeData:      node,
-				CPUAllocation: []*Vector{},
-				RAMAllocation: []*Vector{},
-				GPUReq:        []*Vector{},
+				CPUAllocation: []*util.Vector{},
+				RAMAllocation: []*util.Vector{},
+				GPUReq:        []*util.Vector{},
 				Namespace:     namespace,
 				ClusterID:     clusterid,
 			}
@@ -212,7 +213,7 @@ func CostDataRangeFromSQL(field string, value string, window string, start strin
 
 		k := newContainerMetricFromValues(namespace, pod, container, instance, clusterid)
 		key := k.Key()
-		allocationVector := &Vector{
+		allocationVector := &util.Vector{
 			Timestamp: float64(t.Unix()),
 			Value:     sum,
 		}
@@ -234,9 +235,9 @@ func CostDataRangeFromSQL(field string, value string, window string, start strin
 				PodName:       pod,
 				NodeName:      instance,
 				NodeData:      node,
-				CPUAllocation: []*Vector{},
-				RAMAllocation: []*Vector{},
-				GPUReq:        []*Vector{},
+				CPUAllocation: []*util.Vector{},
+				RAMAllocation: []*util.Vector{},
+				GPUReq:        []*util.Vector{},
 				Namespace:     namespace,
 				ClusterID:     clusterid,
 			}
@@ -336,7 +337,7 @@ func CostDataRangeFromSQL(field string, value string, window string, start strin
 			if err != nil {
 				return nil, err
 			}
-			allocationVector := &Vector{
+			allocationVector := &util.Vector{
 				Timestamp: float64(t.Unix()),
 				Value:     sum,
 			}

pkg/util/file.go

@@ -0,0 +1,23 @@
+package util
+
+import "os"
+
+// File exists has three different return cases that should be handled:
+//   1. File exists and is not a directory (true, nil)
+//   2. File does not exist (false, nil)
+//   3. File may or may not exist. Error occurred during stat (false, error)
+// The third case represents the scenario where the stat returns an error,
+// but the error isn't relevant to the path. This can happen when the current
+// user doesn't have permission to access the file.
+func FileExists(filename string) (bool, error) {
+	info, err := os.Stat(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return !info.IsDir(), nil
+}

costmodel/pool.go → pkg/util/pool.go

@@ -1,4 +1,4 @@
-package costmodel
+package util
 
 import (
 	"sync"

costmodel/vector.go → pkg/util/vector.go

@@ -1,4 +1,4 @@
-package costmodel
+package util
 
 import (
 	"math"

test/aggregation_test.go

@@ -6,8 +6,8 @@ import (
 
 	"gotest.tools/assert"
 
-	"github.com/kubecost/cost-model/cloud"
-	costModel "github.com/kubecost/cost-model/costmodel"
+	"github.com/kubecost/cost-model/pkg/cloud"
+	costModel "github.com/kubecost/cost-model/pkg/costmodel"
 )
 
 func TestAggregation(t *testing.T) {

test/historical_pod_test.go

@@ -14,8 +14,8 @@ import (
 
 	"gotest.tools/assert"
 
-	"github.com/kubecost/cost-model/cloud"
-	costModel "github.com/kubecost/cost-model/costmodel"
+	"github.com/kubecost/cost-model/pkg/cloud"
+	costModel "github.com/kubecost/cost-model/pkg/costmodel"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"