Merge pull request #982 from kubecost/develop

Merge develop to master for 1.88.0 release

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,3 +20,6 @@
 
 ## How was this PR tested?
 
+
+## Have you made an update to documentation?
+

Dockerfile.metrics

@@ -18,6 +18,10 @@ RUN set -e ;\
 FROM alpine:latest
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build-env /go/bin/app /go/bin/app
+ADD ./configs/default.json /models/default.json
+ADD ./configs/azure.json /models/azure.json
+ADD ./configs/aws.json /models/aws.json
+ADD ./configs/gcp.json /models/gcp.json
 
 USER 1001
 ENTRYPOINT ["/go/bin/app"]

cmd/kubemetrics/main.go

@@ -1,15 +1,25 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"net/http"
+	"time"
 
+	"github.com/kubecost/cost-model/pkg/cloud"
 	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/costmodel"
+	"github.com/kubecost/cost-model/pkg/costmodel/clusters"
 	"github.com/kubecost/cost-model/pkg/env"
-	"github.com/kubecost/cost-model/pkg/metrics"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/kubecost/cost-model/pkg/util/watcher"
 
+	prometheus "github.com/prometheus/client_golang/api"
+	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/rs/cors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -51,6 +61,45 @@ func newKubernetesClusterCache() (clustercache.ClusterCache, error) {
 	return k8sCache, nil
 }
 
+func newPrometheusClient() (prometheus.Client, error) {
+	address := env.GetPrometheusServerEndpoint()
+	if address == "" {
+		return nil, fmt.Errorf("No address for prometheus set in $%s. Aborting.", env.PrometheusServerEndpointEnvVar)
+	}
+
+	queryConcurrency := env.GetMaxQueryConcurrency()
+	klog.Infof("Prometheus Client Max Concurrency set to %d", queryConcurrency)
+
+	timeout := 120 * time.Second
+	keepAlive := 120 * time.Second
+
+	promCli, err := prom.NewPrometheusClient(address, timeout, keepAlive, queryConcurrency, "")
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create prometheus client, Error: %v", err)
+	}
+
+	m, err := prom.Validate(promCli)
+	if err != nil || !m.Running {
+		if err != nil {
+			klog.Errorf("Failed to query prometheus at %s. Error: %s . Troubleshooting help available at: %s", address, err.Error(), prom.PrometheusTroubleshootingURL)
+		} else if !m.Running {
+			klog.Errorf("Prometheus at %s is not running. Troubleshooting help available at: %s", address, prom.PrometheusTroubleshootingURL)
+		}
+	} else {
+		klog.V(1).Info("Success: retrieved the 'up' query against prometheus at: " + address)
+	}
+
+	api := prometheusAPI.NewAPI(promCli)
+	_, err = api.Config(context.Background())
+	if err != nil {
+		klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prom.PrometheusTroubleshootingURL)
+	} else {
+		klog.Infof("Retrieved a prometheus config file from: %s", address)
+	}
+
+	return promCli, nil
+}
+
 func main() {
 	klog.InitFlags(nil)
 	flag.Set("v", "3")
@@ -58,19 +107,73 @@ func main() {
 
 	klog.V(1).Infof("Starting kubecost-metrics...")
 
+	configWatchers := watcher.NewConfigMapWatchers()
+
+	scrapeInterval := time.Minute
+	promCli, err := newPrometheusClient()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Lookup scrape interval for kubecost job, update if found
+	si, err := prom.ScrapeIntervalFor(promCli, env.GetKubecostJobName())
+	if err == nil {
+		scrapeInterval = si
+	}
+
+	klog.Infof("Using scrape interval of %f", scrapeInterval.Seconds())
+
 	// initialize kubernetes client and cluster cache
 	clusterCache, err := newKubernetesClusterCache()
 	if err != nil {
 		panic(err.Error())
 	}
 
-	// initialize Kubernetes Metrics
-	metrics.InitKubeMetrics(clusterCache, &metrics.KubeMetricsOpts{
-		EmitKubecostControllerMetrics: true,
-		EmitNamespaceAnnotations:      env.IsEmitNamespaceAnnotationsMetric(),
-		EmitPodAnnotations:            env.IsEmitPodAnnotationsMetric(),
-		EmitKubeStateMetrics:          true,
-	})
+	cloudProviderKey := env.GetCloudProviderAPIKey()
+	cloudProvider, err := cloud.NewProvider(clusterCache, cloudProviderKey)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Append the pricing config watcher
+	configWatchers.AddWatcher(cloud.ConfigWatcherFor(cloudProvider))
+	watchConfigFunc := configWatchers.ToWatchFunc()
+	watchedConfigs := configWatchers.GetWatchedConfigs()
+
+	k8sClient := clusterCache.GetClient()
+	kubecostNamespace := env.GetKubecostNamespace()
+
+	// We need an initial invocation because the init of the cache has happened before we had access to the provider.
+	for _, cw := range watchedConfigs {
+		configs, err := k8sClient.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), cw, metav1.GetOptions{})
+		if err != nil {
+			klog.Infof("No %s configmap found at install time, using existing configs: %s", cw, err.Error())
+		} else {
+			watchConfigFunc(configs)
+		}
+	}
+
+	clusterCache.SetConfigMapUpdateFunc(watchConfigFunc)
+
+	// Initialize ClusterMap for maintaining ClusterInfo by ClusterID
+	clusterMap := clusters.NewClusterMap(
+		promCli,
+		costmodel.NewLocalClusterInfoProvider(k8sClient, cloudProvider),
+		5*time.Minute)
+
+	costModel := costmodel.NewCostModel(promCli, cloudProvider, clusterCache, clusterMap, scrapeInterval)
+
+	// initialize Kubernetes Metrics Emitter
+	metricsEmitter := costmodel.NewCostModelMetricsEmitter(promCli, clusterCache, cloudProvider, costModel)
+
+	// download pricing data
+	err = cloudProvider.DownloadPricingData()
+	if err != nil {
+		klog.Errorf("Error downloading pricing data: %s", err)
+	}
+
+	// start emitting metrics
+	metricsEmitter.Start()
 
 	rootMux := http.NewServeMux()
 	rootMux.HandleFunc("/healthz", Healthz)

pkg/cloud/awsprovider.go

@@ -62,7 +62,10 @@ func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 	sps := &PricingSource{
 		Name: SpotPricingSource,
 	}
-	sps.Error = aws.SpotPricingStatus
+	sps.Error = ""
+	if aws.SpotPricingError != nil {
+		sps.Error = aws.SpotPricingError.Error()
+	}
 	if sps.Error != "" {
 		sps.Available = false
 	} else if len(aws.SpotPricingByInstanceID) > 0 {
@@ -75,7 +78,10 @@ func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 	rps := &PricingSource{
 		Name: ReservedInstancePricingSource,
 	}
-	rps.Error = aws.RIPricingStatus
+	rps.Error = ""
+	if aws.RIPricingError != nil {
+		rps.Error = aws.RIPricingError.Error()
+	}
 	if rps.Error != "" {
 		rps.Available = false
 	} else {
@@ -124,9 +130,9 @@ type AWS struct {
 	SpotPricingUpdatedAt        *time.Time
 	SpotRefreshRunning          bool
 	SpotPricingLock             sync.RWMutex
-	SpotPricingStatus           string
+	SpotPricingError            error
 	RIPricingByInstanceID       map[string]*RIData
-	RIPricingStatus             string
+	RIPricingError              error
 	RIDataRunning               bool
 	RIDataLock                  sync.RWMutex
 	SavingsPlanDataByInstanceID map[string]*SavingsPlanData
@@ -150,6 +156,8 @@ type AWS struct {
 	Config                      *ProviderConfig
 	ServiceAccountChecks        map[string]*ServiceAccountCheck
 	clusterManagementPrice      float64
+	clusterAccountId            string
+	clusterRegion               string
 	clusterProvisioner          string
 	*CustomProvider
 }
@@ -936,10 +944,10 @@ func (aws *AWS) refreshSpotPricing(force bool) {
 	sp, err := aws.parseSpotData(aws.SpotDataBucket, aws.SpotDataPrefix, aws.ProjectID, aws.SpotDataRegion)
 	if err != nil {
 		klog.V(1).Infof("Skipping AWS spot data download: %s", err.Error())
-		aws.SpotPricingStatus = err.Error()
+		aws.SpotPricingError = err
 		return
 	}
-	aws.SpotPricingStatus = ""
+	aws.SpotPricingError = nil
 
 	// update time last updated
 	aws.SpotPricingUpdatedAt = &now
@@ -1174,6 +1182,8 @@ func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
 		m := make(map[string]string)
 		m["name"] = c.ClusterName
 		m["provider"] = "AWS"
+		m["account"] = c.AthenaProjectID // this value requires configuration but is unavailable else where
+		m["region"] = awsProvider.clusterRegion
 		m["id"] = env.GetClusterID()
 		m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 		m["provisioner"] = awsProvider.clusterProvisioner
@@ -1184,6 +1194,8 @@ func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
 		m := make(map[string]string)
 		m["name"] = clusterName
 		m["provider"] = "AWS"
+		m["account"] = c.AthenaProjectID // this value requires configuration but is unavailable else where
+		m["region"] = awsProvider.clusterRegion
 		m["id"] = env.GetClusterID()
 		m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 		return m, nil
@@ -1823,10 +1835,10 @@ func (a *AWS) GetReservationDataFromAthena() error {
 		query := fmt.Sprintf(q, cfg.AthenaTable, start, end)
 		op, err := a.QueryAthenaBillingData(query)
 		if err != nil {
-			a.RIPricingStatus = err.Error()
+			a.RIPricingError = err
 			return fmt.Errorf("Error fetching Reserved Instance Data: %s", err)
 		}
-		a.RIPricingStatus = ""
+		a.RIPricingError = nil
 		klog.Infof("Fetching RI data...")
 		if len(op.ResultSet.Rows) > 1 {
 			a.RIDataLock.Lock()
@@ -1860,7 +1872,7 @@ func (a *AWS) GetReservationDataFromAthena() error {
 		}
 	} else {
 		klog.Infof("No reserved data available in Athena")
-		a.RIPricingStatus = ""
+		a.RIPricingError = nil
 	}
 	return nil
 }

pkg/cloud/azureprovider.go

@@ -382,6 +382,9 @@ type Azure struct {
 	Clientset               clustercache.ClusterCache
 	Config                  *ProviderConfig
 	ServiceAccountChecks    map[string]*ServiceAccountCheck
+	RateCardPricingError    error
+	clusterAccountId        string
+	clusterRegion           string
 }
 
 type azureKey struct {
@@ -731,6 +734,7 @@ func (az *Azure) DownloadPricingData() error {
 
 	config, err := az.GetConfig()
 	if err != nil {
+		az.RateCardPricingError = err
 		return err
 	}
 
@@ -747,6 +751,7 @@ func (az *Azure) DownloadPricingData() error {
 		credentialsConfig := auth.NewClientCredentialsConfig(config.AzureClientID, config.AzureClientSecret, config.AzureTenantID)
 		a, err := credentialsConfig.Authorizer()
 		if err != nil {
+			az.RateCardPricingError = err
 			return err
 		}
 		authorizer = a
@@ -758,6 +763,7 @@ func (az *Azure) DownloadPricingData() error {
 		if err != nil { // Failed to create authorizer from environment, try from file
 			a, err := auth.NewAuthorizerFromFile(azure.PublicCloud.ResourceManagerEndpoint)
 			if err != nil {
+				az.RateCardPricingError = err
 				return err
 			}
 			authorizer = a
@@ -784,19 +790,17 @@ func (az *Azure) DownloadPricingData() error {
 	klog.Infof("Using ratecard query %s", rateCardFilter)
 	result, err := rcClient.Get(context.TODO(), rateCardFilter)
 	if err != nil {
+		az.RateCardPricingError = err
 		return err
 	}
 	allPrices := make(map[string]*AzurePricing)
 	regions, err := getRegions("compute", sClient, providersClient, config.AzureSubscriptionID)
 	if err != nil {
+		az.RateCardPricingError = err
 		return err
 	}
 
-	c, err := az.GetConfig()
-	if err != nil {
-		return err
-	}
-	baseCPUPrice := c.CPU
+	baseCPUPrice := config.CPU
 
 	for _, v := range *result.Meters {
 		meterName := *v.MeterName
@@ -915,6 +919,7 @@ func (az *Azure) DownloadPricingData() error {
 	}
 
 	az.Pricing = allPrices
+	az.RateCardPricingError = nil
 	return nil
 }
 
@@ -1127,6 +1132,8 @@ func (az *Azure) ClusterInfo() (map[string]string, error) {
 		m["name"] = c.ClusterName
 	}
 	m["provider"] = "azure"
+	m["account"] = az.clusterAccountId
+	m["region"] = az.clusterRegion
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 	m["id"] = env.GetClusterID()
 	return m, nil
@@ -1379,8 +1386,29 @@ func (az *Azure) ServiceAccountStatus() *ServiceAccountStatus {
 	}
 }
 
+const rateCardPricingSource = "Rate Card API"
+
+// PricingSourceStatus returns the status of the rate card api
 func (az *Azure) PricingSourceStatus() map[string]*PricingSource {
-	return make(map[string]*PricingSource)
+	sources := make(map[string]*PricingSource)
+	errMsg := ""
+	if az.RateCardPricingError != nil {
+		errMsg = az.RateCardPricingError.Error()
+	}
+	rcps := &PricingSource{
+		Name:  rateCardPricingSource,
+		Error: errMsg,
+	}
+	if rcps.Error != "" {
+		rcps.Available = false
+	} else if len(az.Pricing) == 0 {
+		rcps.Error = "No Pricing Data Available"
+		rcps.Available = false
+	} else {
+		rcps.Available = true
+	}
+	sources[rateCardPricingSource] = rcps
+	return sources
 }
 
 func (*Azure) ClusterManagementPricing() (string, float64, error) {
@@ -1394,3 +1422,15 @@ func (az *Azure) CombinedDiscountForNode(instanceType string, isPreemptible bool
 func (az *Azure) Regions() []string {
 	return azureRegions
 }
+
+func parseAzureSubscriptionID(id string) string {
+	// azure:///subscriptions/0badafdf-1234-abcd-wxyz-123456789/...
+	//  => 0badafdf-1234-abcd-wxyz-123456789
+	rx := regexp.MustCompile("azure:///subscriptions/([^/]*)/*")
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+	// Return empty string if an account could not be parsed from provided string
+	return ""
+}

pkg/cloud/azureprovider_test.go

@@ -0,0 +1,36 @@
+package cloud
+
+import (
+	"testing"
+)
+
+func TestParseAzureSubscriptionID(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "azure:///subscriptions/0badafdf-1234-abcd-wxyz-123456789/...",
+			expected: "0badafdf-1234-abcd-wxyz-123456789",
+		},
+		{
+			input:    "azure:/subscriptions/0badafdf-1234-abcd-wxyz-123456789/...",
+			expected: "",
+		},
+		{
+			input:    "azure:///subscriptions//",
+			expected: "",
+		},
+		{
+			input:    "",
+			expected: "",
+		},
+	}
+
+	for _, test := range cases {
+		result := parseAzureSubscriptionID(test.input)
+		if result != test.expected {
+			t.Errorf("Input: %s, Expected: %s, Actual: %s", test.input, test.expected, result)
+		}
+	}
+}

pkg/cloud/gcpprovider.go

@@ -92,6 +92,8 @@ type GCP struct {
 	ServiceKeyProvided      bool
 	ValidPricingKeys        map[string]bool
 	clusterManagementPrice  float64
+	clusterProjectId        string
+	clusterRegion           string
 	clusterProvisioner      string
 	*CustomProvider
 }
@@ -529,6 +531,8 @@ func (gcp *GCP) ClusterInfo() (map[string]string, error) {
 	m := make(map[string]string)
 	m["name"] = attribute
 	m["provider"] = "GCP"
+	m["project"] = gcp.clusterProjectId
+	m["region"] = gcp.clusterRegion
 	m["provisioner"] = gcp.clusterProvisioner
 	m["id"] = env.GetClusterID()
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
@@ -673,7 +677,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 				usageType := strings.ToLower(product.Category.UsageType)
 				instanceType := strings.ToLower(product.Category.ResourceGroup)
 
-				if instanceType == "ssd" && !strings.Contains(product.Description, "Regional") { // TODO: support regional
+				if instanceType == "ssd" && strings.Contains(product.Description, "SSD backed") && !strings.Contains(product.Description, "Regional") { // TODO: support regional
 					lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
 					var nanos float64
 					if lastRateIndex > -1 && len(product.PricingInfo) > 0 {
@@ -695,6 +699,28 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						}
 					}
 					continue
+				} else if instanceType == "ssd" && strings.Contains(product.Description, "SSD backed") && strings.Contains(product.Description, "Regional") { // TODO: support regional
+					lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
+					var nanos float64
+					if lastRateIndex > -1 && len(product.PricingInfo) > 0 {
+						nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
+					} else {
+						continue
+					}
+					hourlyPrice := (nanos * math.Pow10(-9)) / 730
+
+					for _, sr := range product.ServiceRegions {
+						region := sr
+						candidateKey := region + "," + "ssd" + "," + "regional"
+						if _, ok := pvKeys[candidateKey]; ok {
+							product.PV = &PV{
+								Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
+							}
+							gcpPricingList[candidateKey] = product
+							continue
+						}
+					}
+					continue
 				} else if instanceType == "pdstandard" && !strings.Contains(product.Description, "Regional") { // TODO: support regional
 					lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
 					var nanos float64
@@ -716,6 +742,27 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						}
 					}
 					continue
+				} else if instanceType == "pdstandard" && strings.Contains(product.Description, "Regional") { // TODO: support regional
+					lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
+					var nanos float64
+					if lastRateIndex > -1 && len(product.PricingInfo) > 0 {
+						nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
+					} else {
+						continue
+					}
+					hourlyPrice := (nanos * math.Pow10(-9)) / 730
+					for _, sr := range product.ServiceRegions {
+						region := sr
+						candidateKey := region + "," + "pdstandard" + "," + "regional"
+						if _, ok := pvKeys[candidateKey]; ok {
+							product.PV = &PV{
+								Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
+							}
+							gcpPricingList[candidateKey] = product
+							continue
+						}
+					}
+					continue
 				}
 
 				if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "CUSTOM") {
@@ -975,7 +1022,12 @@ func (gcp *GCP) parsePages(inputKeys map[string]Key, pvKeys map[string]PVKey) (m
 	}
 	klog.V(1).Infof("ALL PAGES: %+v", returnPages)
 	for k, v := range returnPages {
-		klog.V(1).Infof("Returned Page: %s : %+v", k, v.Node)
+		if v.Node != nil {
+			klog.V(1).Infof("Returned Page: %s : %+v", k, v.Node)
+		}
+		if v.PV != nil {
+			klog.V(1).Infof("Returned Page: %s : %+v", k, v.PV)
+		}
 	}
 	return returnPages, err
 }
@@ -998,13 +1050,17 @@ func (gcp *GCP) DownloadPricingData() error {
 	nodeList := gcp.Clientset.GetAllNodes()
 	inputkeys := make(map[string]Key)
 
+	defaultRegion := "" // Sometimes, PVs may be missing the region label. In that case assume that they are in the same region as the nodes
 	for _, n := range nodeList {
 		labels := n.GetObjectMeta().GetLabels()
 		if _, ok := labels["cloud.google.com/gke-nodepool"]; ok { // The node is part of a GKE nodepool, so you're paying a cluster management cost
 			gcp.clusterManagementPrice = 0.10
 			gcp.clusterProvisioner = "GKE"
 		}
-
+		r, _ := util.GetRegion(labels)
+		if r != "" {
+			defaultRegion = r
+		}
 		key := gcp.GetKey(labels, n)
 		inputkeys[key.Features()] = key
 	}
@@ -1028,7 +1084,7 @@ func (gcp *GCP) DownloadPricingData() error {
 			log.DedupedWarningf(5, "Unable to find params for storageClassName %s", pv.Name)
 			continue
 		}
-		key := gcp.GetPVKey(pv, params, "")
+		key := gcp.GetPVKey(pv, params, defaultRegion)
 		pvkeys[key.Features()] = key
 	}
 
@@ -1057,7 +1113,7 @@ func (gcp *GCP) PVPricing(pvk PVKey) (*PV, error) {
 	defer gcp.DownloadPricingDataLock.RUnlock()
 	pricing, ok := gcp.Pricing[pvk.Features()]
 	if !ok {
-		klog.V(4).Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
+		klog.V(3).Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
 		return &PV{}, nil
 	}
 	return pricing.PV, nil
@@ -1363,8 +1419,17 @@ func (key *pvKey) Features() string {
 	} else if storageClass == "pd-standard" {
 		storageClass = "pdstandard"
 	}
+	replicationType := ""
+	if rt, ok := key.StorageClassParameters["replication-type"]; ok {
+		if rt == "regional-pd" {
+			replicationType = ",regional"
+		}
+	}
 	region, _ := util.GetRegion(key.Labels)
-	return region + "," + storageClass
+	if region == "" {
+		region = key.DefaultRegion
+	}
+	return region + "," + storageClass + replicationType
 }
 
 type gcpKey struct {
@@ -1395,15 +1460,19 @@ func (gcp *gcpKey) GPUType() string {
 	return ""
 }
 
-// GetKey maps node labels to information needed to retrieve pricing data
-func (gcp *gcpKey) Features() string {
+func parseGCPInstanceTypeLabel(it string) string {
 	var instanceType string
-	it, _ := util.GetInstanceType(gcp.Labels)
-	if it == "" {
-		log.DedupedErrorf(1, "Missing or Unknown 'node.kubernetes.io/instance-type' node label")
+
+	splitByDash := strings.Split(it, "-")
+
+	// GKE nodes are labeled with the GCP instance type, but users can deploy on GCP
+	// with tools like K3s, whose instance type labels will be "k3s". This logic
+	// avoids a panic in the slice operation then there are no dashes (-) in the
+	// instance type label value.
+	if len(splitByDash) < 2 {
 		instanceType = "unknown"
 	} else {
-		instanceType = strings.ToLower(strings.Join(strings.Split(it, "-")[:2], ""))
+		instanceType = strings.ToLower(strings.Join(splitByDash[:2], ""))
 		if instanceType == "n1highmem" || instanceType == "n1highcpu" {
 			instanceType = "n1standard" // These are priced the same. TODO: support n1ultrahighmem
 		} else if instanceType == "n2highmem" || instanceType == "n2highcpu" {
@@ -1415,6 +1484,20 @@ func (gcp *gcpKey) Features() string {
 		}
 	}
 
+	return instanceType
+}
+
+// GetKey maps node labels to information needed to retrieve pricing data
+func (gcp *gcpKey) Features() string {
+	var instanceType string
+	it, _ := util.GetInstanceType(gcp.Labels)
+	if it == "" {
+		log.DedupedErrorf(1, "Missing or Unknown 'node.kubernetes.io/instance-type' node label")
+		instanceType = "unknown"
+	} else {
+		instanceType = parseGCPInstanceTypeLabel(it)
+	}
+
 	r, _ := util.GetRegion(gcp.Labels)
 	region := strings.ToLower(r)
 	var usageType string
@@ -1506,3 +1589,15 @@ func sustainedUseDiscount(class string, defaultDiscount float64, isPreemptible b
 	}
 	return discount
 }
+
+func parseGCPProjectID(id string) string {
+	// gce://guestbook-12345/...
+	//  => guestbook-12345
+	rx := regexp.MustCompile("gce://([^/]*)/*")
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+	// Return empty string if an account could not be parsed from provided string
+	return ""
+}

pkg/cloud/gcpprovider_test.go

@@ -0,0 +1,67 @@
+package cloud
+
+import (
+	"testing"
+)
+
+func TestParseGCPInstanceTypeLabel(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "n1-standard-2",
+			expected: "n1standard",
+		},
+		{
+			input:    "e2-medium",
+			expected: "e2medium",
+		},
+		{
+			input:    "k3s",
+			expected: "unknown",
+		},
+		{
+			input:    "custom-n1-standard-2",
+			expected: "custom",
+		},
+	}
+
+	for _, test := range cases {
+		result := parseGCPInstanceTypeLabel(test.input)
+		if result != test.expected {
+			t.Errorf("Input: %s, Expected: %s, Actual: %s", test.input, test.expected, result)
+		}
+	}
+}
+
+func TestParseGCPProjectID(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "gce://guestbook-12345/...",
+			expected: "guestbook-12345",
+		},
+		{
+			input:    "gce:/guestbook-12345/...",
+			expected: "",
+		},
+		{
+			input:    "asdfa",
+			expected: "",
+		},
+		{
+			input:    "",
+			expected: "",
+		},
+	}
+
+	for _, test := range cases {
+		result := parseGCPProjectID(test.input)
+		if result != test.expected {
+			t.Errorf("Input: %s, Expected: %s, Actual: %s", test.input, test.expected, result)
+		}
+	}
+}

pkg/cloud/provider.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"github.com/kubecost/cost-model/pkg/util"
 	"io"
 	"regexp"
 	"strconv"
@@ -17,6 +18,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/clustercache"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/util/watcher"
 
 	v1 "k8s.io/api/core/v1"
 )
@@ -298,6 +300,18 @@ func CustomPricesEnabled(p Provider) bool {
 	return config.CustomPricesEnabled == "true"
 }
 
+// ConfigWatcherFor returns a new ConfigWatcher instance which watches changes to the "pricing-configs"
+// configmap
+func ConfigWatcherFor(p Provider) *watcher.ConfigMapWatcher {
+	return &watcher.ConfigMapWatcher{
+		ConfigMapName: env.GetPricingConfigmapName(),
+		WatchFunc: func(name string, data map[string]string) error {
+			_, err := p.UpdateConfigFromConfigMap(data)
+			return err
+		},
+	}
+}
+
 // AllocateIdleByDefault returns true if the application settings specify to allocate idle by default
 func AllocateIdleByDefault(p Provider) bool {
 	config, err := p.GetConfig()
@@ -399,62 +413,92 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string) (Provider, erro
 		return nil, fmt.Errorf("Could not locate any nodes for cluster.")
 	}
 
-	provider := strings.ToLower(nodes[0].Spec.ProviderID)
+	cp := getClusterProperties(nodes[0])
 
-	if env.IsUseCSVProvider() {
+	switch cp.provider {
+	case "CSV":
 		klog.Infof("Using CSV Provider with CSV at %s", env.GetCSVPath())
-		configFileName := ""
-		if metadata.OnGCE() {
-			configFileName = "gcp.json"
-		} else if strings.HasPrefix(provider, "aws") {
-			configFileName = "aws.json"
-		} else if strings.HasPrefix(provider, "azure") {
-			configFileName = "azure.json"
-
-		} else {
-			configFileName = "default.json"
-		}
 		return &CSVProvider{
 			CSVLocation: env.GetCSVPath(),
 			CustomProvider: &CustomProvider{
 				Clientset: cache,
-				Config:    NewProviderConfig(configFileName),
+				Config:    NewProviderConfig(cp.configFileName),
 			},
 		}, nil
-	}
-	if metadata.OnGCE() {
+	case "GCP":
 		klog.V(3).Info("metadata reports we are in GCE")
 		if apiKey == "" {
 			return nil, errors.New("Supply a GCP Key to start getting data")
 		}
 		return &GCP{
-			Clientset: cache,
-			APIKey:    apiKey,
-			Config:    NewProviderConfig("gcp.json"),
+			Clientset:        cache,
+			APIKey:           apiKey,
+			Config:           NewProviderConfig(cp.configFileName),
+			clusterRegion:    cp.region,
+			clusterProjectId: cp.projectID,
 		}, nil
-	}
-
-	if strings.HasPrefix(provider, "aws") {
+	case "AWS":
 		klog.V(2).Info("Found ProviderID starting with \"aws\", using AWS Provider")
 		return &AWS{
-			Clientset: cache,
-			Config:    NewProviderConfig("aws.json"),
+			Clientset:        cache,
+			Config:           NewProviderConfig(cp.configFileName),
+			clusterRegion:    cp.region,
+			clusterAccountId: cp.accountID,
 		}, nil
-	} else if strings.HasPrefix(provider, "azure") {
+	case "AZURE":
 		klog.V(2).Info("Found ProviderID starting with \"azure\", using Azure Provider")
 		return &Azure{
-			Clientset: cache,
-			Config:    NewProviderConfig("azure.json"),
+			Clientset:        cache,
+			Config:           NewProviderConfig(cp.configFileName),
+			clusterRegion:    cp.region,
+			clusterAccountId: cp.accountID,
 		}, nil
-	} else {
+	default:
 		klog.V(2).Info("Unsupported provider, falling back to default")
 		return &CustomProvider{
 			Clientset: cache,
-			Config:    NewProviderConfig("default.json"),
+			Config:    NewProviderConfig(cp.configFileName),
 		}, nil
 	}
 }
 
+type clusterProperties struct {
+	provider       string
+	configFileName string
+	region         string
+	accountID      string
+	projectID      string
+}
+
+func getClusterProperties(node *v1.Node) (clusterProperties) {
+	providerID := strings.ToLower(node.Spec.ProviderID)
+	region, _ := util.GetRegion(node.Labels)
+	cp := clusterProperties{
+		provider: "DEFAULT",
+		configFileName: "default.json",
+		region: region,
+		accountID: "",
+		projectID: "",
+	}
+	if metadata.OnGCE() {
+		cp.provider = "GCP"
+		cp.configFileName = "gcp.json"
+		cp.projectID = parseGCPProjectID(providerID)
+	} else if strings.HasPrefix(providerID, "aws") {
+		cp.provider = "AWS"
+		cp.configFileName = "aws.json"
+	} else if strings.HasPrefix(providerID, "azure") {
+		cp.provider = "AZURE"
+		cp.configFileName = "azure.json"
+		cp.accountID = parseAzureSubscriptionID(providerID)
+	}
+	if env.IsUseCSVProvider() {
+		cp.provider = "CSV"
+	}
+
+	return cp
+}
+
 func UpdateClusterMeta(cluster_id, cluster_name string) error {
 	pw := env.GetRemotePW()
 	address := env.GetSQLAddress()

pkg/costmodel/clusters/clustermap.go

@@ -21,11 +21,15 @@ const (
 	LoadRetryDelay time.Duration = 10 * time.Second
 )
 
+// ClusterInfo holds attributes of Cluster from metrics pulled from Prometheus
 type ClusterInfo struct {
 	ID          string `json:"id"`
 	Name        string `json:"name"`
 	Profile     string `json:"profile"`
 	Provider    string `json:"provider"`
+	Account     string `json:"account"`
+	Project     string `json:"project"`
+	Region      string `json:"region"`
 	Provisioner string `json:"provisioner"`
 }
 
@@ -40,6 +44,9 @@ func (ci *ClusterInfo) Clone() *ClusterInfo {
 		Name:        ci.Name,
 		Profile:     ci.Profile,
 		Provider:    ci.Provider,
+		Account:     ci.Account,
+		Project:     ci.Project,
+		Region:      ci.Region,
 		Provisioner: ci.Provisioner,
 	}
 }
@@ -170,6 +177,21 @@ func (pcm *PrometheusClusterMap) loadClusters() (map[string]*ClusterInfo, error)
 			provider = ""
 		}
 
+		account, err := result.GetString("account")
+		if err != nil {
+			account = ""
+		}
+
+		project, err := result.GetString("project")
+		if err != nil {
+			project = ""
+		}
+
+		region, err := result.GetString("region")
+		if err != nil {
+			region = ""
+		}
+
 		provisioner, err := result.GetString("provisioner")
 		if err != nil {
 			provisioner = ""
@@ -180,6 +202,9 @@ func (pcm *PrometheusClusterMap) loadClusters() (map[string]*ClusterInfo, error)
 			Name:        name,
 			Profile:     profile,
 			Provider:    provider,
+			Account:     account,
+			Project:     project,
+			Region:      region,
 			Provisioner: provisioner,
 		}
 	}
@@ -218,14 +243,31 @@ func (pcm *PrometheusClusterMap) getLocalClusterInfo() (*ClusterInfo, error) {
 
 	var clusterProfile string
 	var provider string
+	var account string
+	var project string
+	var region string
 	var provisioner string
 
 	if cp, ok := info["clusterProfile"]; ok {
 		clusterProfile = cp
 	}
+
 	if pvdr, ok := info["provider"]; ok {
 		provider = pvdr
 	}
+
+	if acct, ok := info["account"]; ok {
+		account = acct
+	}
+
+	if proj, ok := info["project"]; ok {
+		project = proj
+	}
+
+	if reg, ok := info["region"]; ok {
+		region = reg
+	}
+
 	if pvsr, ok := info["provisioner"]; ok {
 		provisioner = pvsr
 	}
@@ -235,6 +277,9 @@ func (pcm *PrometheusClusterMap) getLocalClusterInfo() (*ClusterInfo, error) {
 		Name:        name,
 		Profile:     clusterProfile,
 		Provider:    provider,
+		Account:     account,
+		Project:     project,
+		Region:      region,
 		Provisioner: provisioner,
 	}, nil
 }

pkg/costmodel/metrics.go

@@ -263,17 +263,13 @@ func NewCostModelMetricsEmitter(promClient promclient.Client, clusterCache clust
 	// init will only actually execute once to register the custom gauges
 	initCostModelMetrics(clusterCache, provider)
 
-	// if the metrics pod is not enabled, we want to emit those metrics from this pod.
-	// NOTE: This is not optimal, as we calculate costs based on run times for other containers.
-	// NOTE: The metrics for run times should be emitted separate from cost-model
-	if !env.IsKubecostMetricsPodEnabled() {
-		metrics.InitKubeMetrics(clusterCache, &metrics.KubeMetricsOpts{
-			EmitKubecostControllerMetrics: true,
-			EmitNamespaceAnnotations:      env.IsEmitNamespaceAnnotationsMetric(),
-			EmitPodAnnotations:            env.IsEmitPodAnnotationsMetric(),
-			EmitKubeStateMetrics:          env.IsEmitKsmV1Metrics(),
-		})
-	}
+	metrics.InitKubeMetrics(clusterCache, &metrics.KubeMetricsOpts{
+		EmitKubecostControllerMetrics: true,
+		EmitNamespaceAnnotations:      env.IsEmitNamespaceAnnotationsMetric(),
+		EmitPodAnnotations:            env.IsEmitPodAnnotationsMetric(),
+		EmitKubeStateMetrics:          env.IsEmitKsmV1Metrics(),
+		EmitKubeStateMetricsV1Only:    env.IsEmitKsmV1MetricsOnly(),
+	})
 
 	return &CostModelMetricsEmitter{
 		PrometheusClient:              promClient,
@@ -325,6 +321,15 @@ func (cmme *CostModelMetricsEmitter) IsRunning() bool {
 	return cmme.recordingStop != nil
 }
 
+// NodeCostAverages tracks a running average of a node's cost attributes.
+// The averages are used to detect and discard spurrious outliers.
+type NodeCostAverages struct {
+	CpuCostAverage   float64
+	RamCostAverage   float64
+	NumCpuDataPoints float64
+	NumRamDataPoints float64
+}
+
 // StartCostModelMetricRecording starts the go routine that emits metrics used to determine
 // cluster costs.
 func (cmme *CostModelMetricsEmitter) Start() bool {
@@ -344,6 +349,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 		loadBalancerSeen := make(map[string]bool)
 		pvSeen := make(map[string]bool)
 		pvcSeen := make(map[string]bool)
+		nodeCostAverages := make(map[string]NodeCostAverages)
 
 		getKeyFromLabelStrings := func(labels ...string) string {
 			return strings.Join(labels, ",")
@@ -450,17 +456,54 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 
 				totalCost := cpu*cpuCost + ramCost*(ram/1024/1024/1024) + gpu*gpuCost
 
-				cmme.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(cpuCost)
-				cmme.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(ramCost)
-				cmme.GPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(gpuCost)
+				labelKey := getKeyFromLabelStrings(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID)
+
+				avgCosts, ok := nodeCostAverages[labelKey]
+
+				// initialize average cost tracking for this node if there is none
+				if !ok {
+					avgCosts = NodeCostAverages{
+						CpuCostAverage:   cpuCost,
+						RamCostAverage:   ramCost,
+						NumCpuDataPoints: 1,
+						NumRamDataPoints: 1,
+					}
+					nodeCostAverages[labelKey] = avgCosts
+				}
+
 				cmme.GPUCountRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(gpu)
-				cmme.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(totalCost)
+
+				const outlierFactor float64 = 30
+				// don't record cpuCost, ramCost, or gpuCost in the case of wild outliers
+				// k8s api sometimes causes cost spikes as described here:
+				// https://github.com/kubecost/cost-model/issues/927
+				if cpuCost < outlierFactor*avgCosts.CpuCostAverage {
+					cmme.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(cpuCost)
+					avgCosts.CpuCostAverage = (avgCosts.CpuCostAverage*avgCosts.NumCpuDataPoints + cpuCost) / (avgCosts.NumCpuDataPoints + 1)
+					avgCosts.NumCpuDataPoints += 1
+				} else {
+					log.Warningf("CPU cost outlier detected; skipping data point.")
+				}
+				if ramCost < outlierFactor*avgCosts.RamCostAverage {
+					cmme.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(ramCost)
+					avgCosts.RamCostAverage = (avgCosts.RamCostAverage*avgCosts.NumRamDataPoints + ramCost) / (avgCosts.NumRamDataPoints + 1)
+					avgCosts.NumRamDataPoints += 1
+				} else {
+					log.Warningf("RAM cost outlier detected; skipping data point.")
+				}
+				// skip redording totalCost if any constituent costs were outliers
+				if cpuCost < outlierFactor*avgCosts.CpuCostAverage &&
+					ramCost < outlierFactor*avgCosts.RamCostAverage {
+					cmme.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(totalCost)
+				}
+
+				nodeCostAverages[labelKey] = avgCosts
+
 				if node.IsSpot() {
 					cmme.NodeSpotRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(1.0)
 				} else {
 					cmme.NodeSpotRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(0.0)
 				}
-				labelKey := getKeyFromLabelStrings(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID)
 				nodeSeen[labelKey] = true
 			}
 
@@ -606,6 +649,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 						klog.Infof("FAILURE TO REMOVE %s from ramprice", labelString)
 					}
 					delete(nodeSeen, labelString)
+					delete(nodeCostAverages, labelString)
 				} else {
 					nodeSeen[labelString] = false
 				}

pkg/costmodel/promparsers.go

@@ -11,30 +11,8 @@ import (
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/util"
-	"gopkg.in/yaml.v2"
 )
 
-const DEFAULT_KUBECOST_JOB_NAME = "kubecost"
-
-type ScrapeConfig struct {
-	JobName        string `yaml:"job_name,omitempty"`
-	ScrapeInterval string `yaml:"scrape_interval,omitempty"`
-}
-
-type PromCfg struct {
-	ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs,omitempty"`
-}
-
-func GetPrometheusConfig(pcfg string) (PromCfg, error) {
-	var promCfg PromCfg
-	err := yaml.Unmarshal([]byte(pcfg), &promCfg)
-	return promCfg, err
-}
-
-func GetKubecostJobName() string {
-	return DEFAULT_KUBECOST_JOB_NAME // TODO: look this up from a prometheus variable?
-}
-
 func GetPVInfoLocal(cache clustercache.ClusterCache, defaultClusterID string) (map[string]*PersistentVolumeClaimData, error) {
 	toReturn := make(map[string]*PersistentVolumeClaimData)
 

pkg/costmodel/router.go

@@ -11,8 +11,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/services"
 	"github.com/kubecost/cost-model/pkg/util/httputil"
 	"github.com/kubecost/cost-model/pkg/util/timeutil"
+	"github.com/kubecost/cost-model/pkg/util/watcher"
 
 	"k8s.io/klog"
 
@@ -22,7 +24,6 @@ import (
 
 	"github.com/kubecost/cost-model/pkg/cloud"
 	"github.com/kubecost/cost-model/pkg/clustercache"
-	cm "github.com/kubecost/cost-model/pkg/clustermanager"
 	"github.com/kubecost/cost-model/pkg/costmodel/clusters"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/errors"
@@ -32,9 +33,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/thanos"
 	"github.com/kubecost/cost-model/pkg/util/json"
 	prometheus "github.com/prometheus/client_golang/api"
-	prometheusClient "github.com/prometheus/client_golang/api"
 	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/patrickmn/go-cache"
@@ -45,14 +44,13 @@ import (
 )
 
 const (
-	prometheusTroubleshootingEp = "http://docs.kubecost.com/custom-prom#troubleshoot"
-	RFC3339Milli                = "2006-01-02T15:04:05.000Z"
-	maxCacheMinutes1d           = 11
-	maxCacheMinutes2d           = 17
-	maxCacheMinutes7d           = 37
-	maxCacheMinutes30d          = 137
-	CustomPricingSetting        = "CustomPricing"
-	DiscountSetting             = "Discount"
+	RFC3339Milli         = "2006-01-02T15:04:05.000Z"
+	maxCacheMinutes1d    = 11
+	maxCacheMinutes2d    = 17
+	maxCacheMinutes7d    = 37
+	maxCacheMinutes30d   = 137
+	CustomPricingSetting = "CustomPricing"
+	DiscountSetting      = "Discount"
 )
 
 var (
@@ -64,10 +62,9 @@ var (
 // Prometheus, Kubernetes, the cloud provider, and caches.
 type Accesses struct {
 	Router            *httprouter.Router
-	PrometheusClient  prometheusClient.Client
-	ThanosClient      prometheusClient.Client
+	PrometheusClient  prometheus.Client
+	ThanosClient      prometheus.Client
 	KubeClientSet     kubernetes.Interface
-	ClusterManager    *cm.ClusterManager
 	ClusterMap        clusters.ClusterMap
 	CloudProvider     cloud.Provider
 	Model             *CostModel
@@ -84,13 +81,15 @@ type Accesses struct {
 	// settings will be published in a pub/sub model
 	settingsSubscribers map[string][]chan string
 	settingsMutex       sync.Mutex
+	// registered http service instances
+	httpServices services.HTTPServices
 }
 
 // GetPrometheusClient decides whether the default Prometheus client or the Thanos client
 // should be used.
-func (a *Accesses) GetPrometheusClient(remote bool) prometheusClient.Client {
+func (a *Accesses) GetPrometheusClient(remote bool) prometheus.Client {
 	// Use Thanos Client if it exists (enabled) and remote flag set
-	var pc prometheusClient.Client
+	var pc prometheus.Client
 
 	if remote && a.ThanosClient != nil {
 		pc = a.ThanosClient
@@ -410,7 +409,7 @@ func (a *Accesses) ClusterCosts(w http.ResponseWriter, r *http.Request, ps httpr
 		return
 	}
 
-	var client prometheusClient.Client
+	var client prometheus.Client
 	if useThanos {
 		client = a.ThanosClient
 		offsetDur = thanos.OffsetDuration()
@@ -493,7 +492,7 @@ func (a *Accesses) CostDataModelRange(w http.ResponseWriter, r *http.Request, ps
 	}
 
 	// Use Thanos Client if it exists (enabled) and remote flag set
-	var pClient prometheusClient.Client
+	var pClient prometheus.Client
 	if remote != "false" && a.ThanosClient != nil {
 		pClient = a.ThanosClient
 	} else {
@@ -911,40 +910,6 @@ func (a *Accesses) GetPrometheusMetrics(w http.ResponseWriter, _ *http.Request,
 	w.Write(WrapData(result, nil))
 }
 
-// Creates a new ClusterManager instance using a boltdb storage. If that fails,
-// then we fall back to a memory-only storage.
-func newClusterManager() *cm.ClusterManager {
-	clustersConfigFile := "/var/configs/clusters/default-clusters.yaml"
-
-	// Return a memory-backed cluster manager populated by configmap
-	return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
-
-	// NOTE: The following should be used with a persistent disk store. Since the
-	// NOTE: configmap approach is currently the "persistent" source (entries are read-only
-	// NOTE: on the backend), we don't currently need to store on disk.
-	/*
-		path := env.GetConfigPath()
-		db, err := bolt.Open(path+"costmodel.db", 0600, nil)
-		if err != nil {
-			klog.V(1).Infof("[Error] Failed to create costmodel.db: %s", err.Error())
-			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
-		}
-
-		store, err := cm.NewBoltDBClusterStorage("clusters", db)
-		if err != nil {
-			klog.V(1).Infof("[Error] Failed to Create Cluster Storage: %s", err.Error())
-			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
-		}
-
-		return cm.NewConfiguredClusterManager(store, clustersConfigFile)
-	*/
-}
-
-type ConfigWatchers struct {
-	ConfigmapName string
-	WatchFunc     func(string, map[string]string) error
-}
-
 // captures the panic event in sentry
 func capturePanicEvent(err string, stack string) {
 	msg := fmt.Sprintf("Panic: %s\nStackTrace: %s\n", err, stack)
@@ -975,12 +940,14 @@ func handlePanic(p errors.Panic) bool {
 	return p.Type == errors.PanicTypeHTTP
 }
 
-func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
+func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses {
 	klog.InitFlags(nil)
 	flag.Set("v", "3")
 	flag.Parse()
 	klog.V(1).Infof("Starting cost-model (git commit \"%s\")", env.GetAppVersion())
 
+	configWatchers := watcher.NewConfigMapWatchers(additionalConfigWatchers...)
+
 	var err error
 	if errorReportingEnabled {
 		err = sentry.Init(sentry.ClientOptions{Release: env.GetAppVersion()})
@@ -1004,59 +971,40 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 
 	timeout := 120 * time.Second
 	keepAlive := 120 * time.Second
-	scrapeInterval, _ := time.ParseDuration("1m")
+	scrapeInterval := time.Minute
 
 	promCli, err := prom.NewPrometheusClient(address, timeout, keepAlive, queryConcurrency, "")
 	if err != nil {
 		klog.Fatalf("Failed to create prometheus client, Error: %v", err)
 	}
 
-	api := prometheusAPI.NewAPI(promCli)
-	pcfg, err := api.Config(context.Background())
-	if err != nil {
-		klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prometheusTroubleshootingEp)
-	} else {
-		klog.V(1).Info("Retrieved a prometheus config file from: " + address)
-		sc, err := GetPrometheusConfig(pcfg.YAML)
-		if err != nil {
-			klog.Infof("Fix YAML error %s", err)
-		}
-		for _, scrapeconfig := range sc.ScrapeConfigs {
-			if scrapeconfig.JobName == GetKubecostJobName() {
-				if scrapeconfig.ScrapeInterval != "" {
-					si := scrapeconfig.ScrapeInterval
-					sid, err := time.ParseDuration(si)
-					if err != nil {
-						klog.Infof("error parseing scrapeConfig for %s", scrapeconfig.JobName)
-					} else {
-						klog.Infof("Found Kubecost job scrape interval of: %s", si)
-						scrapeInterval = sid
-					}
-				}
-			}
-		}
-	}
-	klog.Infof("Using scrape interval of %f", scrapeInterval.Seconds())
-
 	m, err := prom.Validate(promCli)
-	if err != nil || m.Running == false {
-		if err != nil {
-			klog.Errorf("Failed to query prometheus at %s. Error: %s . Troubleshooting help available at: %s", address, err.Error(), prometheusTroubleshootingEp)
-		} else if m.Running == false {
-			klog.Errorf("Prometheus at %s is not running. Troubleshooting help available at: %s", address, prometheusTroubleshootingEp)
-		}
-
-		api := prometheusAPI.NewAPI(promCli)
-		_, err = api.Config(context.Background())
+	if err != nil || !m.Running {
 		if err != nil {
-			klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prometheusTroubleshootingEp)
-		} else {
-			klog.V(1).Info("Retrieved a prometheus config file from: " + address)
+			klog.Errorf("Failed to query prometheus at %s. Error: %s . Troubleshooting help available at: %s", address, err.Error(), prom.PrometheusTroubleshootingURL)
+		} else if !m.Running {
+			klog.Errorf("Prometheus at %s is not running. Troubleshooting help available at: %s", address, prom.PrometheusTroubleshootingURL)
 		}
 	} else {
 		klog.V(1).Info("Success: retrieved the 'up' query against prometheus at: " + address)
 	}
 
+	api := prometheusAPI.NewAPI(promCli)
+	_, err = api.Config(context.Background())
+	if err != nil {
+		klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prom.PrometheusTroubleshootingURL)
+	} else {
+		klog.Infof("Retrieved a prometheus config file from: %s", address)
+	}
+
+	// Lookup scrape interval for kubecost job, update if found
+	si, err := prom.ScrapeIntervalFor(promCli, env.GetKubecostJobName())
+	if err == nil {
+		scrapeInterval = si
+	}
+
+	klog.Infof("Using scrape interval of %f", scrapeInterval.Seconds())
+
 	// Kubernetes API setup
 	var kc *rest.Config
 	if kubeconfig := env.GetKubeConfigPath(); kubeconfig != "" {
@@ -1083,37 +1031,17 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		panic(err.Error())
 	}
 
-	watchConfigFunc := func(c interface{}) {
-		conf := c.(*v1.ConfigMap)
-		if conf.GetName() == env.GetPricingConfigmapName() {
-			_, err := cloudProvider.UpdateConfigFromConfigMap(conf.Data)
-			if err != nil {
-				klog.Infof("ERROR UPDATING %s CONFIG: %s", "pricing-configs", err.Error())
-			}
-		}
-		for _, cw := range additionalConfigWatchers {
-			if conf.GetName() == cw.ConfigmapName {
-				err := cw.WatchFunc(conf.GetName(), conf.Data)
-				if err != nil {
-					klog.Infof("ERROR UPDATING %s CONFIG: %s", cw.ConfigmapName, err.Error())
-				}
-			}
-		}
-	}
+	// Append the pricing config watcher
+	configWatchers.AddWatcher(cloud.ConfigWatcherFor(cloudProvider))
+	watchConfigFunc := configWatchers.ToWatchFunc()
+	watchedConfigs := configWatchers.GetWatchedConfigs()
 
 	kubecostNamespace := env.GetKubecostNamespace()
 	// We need an initial invocation because the init of the cache has happened before we had access to the provider.
-	configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), env.GetPricingConfigmapName(), metav1.GetOptions{})
-	if err != nil {
-		klog.Infof("No %s configmap found at installtime, using existing configs: %s", env.GetPricingConfigmapName(), err.Error())
-	} else {
-		watchConfigFunc(configs)
-	}
-
-	for _, cw := range additionalConfigWatchers {
-		configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), cw.ConfigmapName, metav1.GetOptions{})
+	for _, cw := range watchedConfigs {
+		configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), cw, metav1.GetOptions{})
 		if err != nil {
-			klog.Infof("No %s configmap found at installtime, using existing configs: %s", cw.ConfigmapName, err.Error())
+			klog.Infof("No %s configmap found at install time, using existing configs: %s", cw, err.Error())
 		} else {
 			klog.Infof("Found configmap %s, watching...", configs.Name)
 			watchConfigFunc(configs)
@@ -1122,14 +1050,6 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 
 	k8sCache.SetConfigMapUpdateFunc(watchConfigFunc)
 
-	// TODO: General Architecture Note: Several passes have been made to modularize a lot of
-	// TODO: our code, but the router still continues to be the obvious entry point for new \
-	// TODO: features. We should look to split out the actual "router" functionality and
-	// TODO: implement a builder -> controller for stitching new features and other dependencies.
-	clusterManager := newClusterManager()
-
-	// Initialize metrics here
-
 	remoteEnabled := env.IsRemoteEnabled()
 	if remoteEnabled {
 		info, err := cloudProvider.ClusterInfo()
@@ -1144,7 +1064,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 	}
 
 	// Thanos Client
-	var thanosClient prometheusClient.Client
+	var thanosClient prometheus.Client
 	if thanos.IsEnabled() {
 		thanosAddress := thanos.QueryURL()
 
@@ -1208,7 +1128,6 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		PrometheusClient:  promCli,
 		ThanosClient:      thanosClient,
 		KubeClientSet:     kubeClientset,
-		ClusterManager:    clusterManager,
 		ClusterMap:        clusterMap,
 		CloudProvider:     cloudProvider,
 		Model:             costModel,
@@ -1219,6 +1138,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		OutOfClusterCache: outOfClusterCache,
 		SettingsCache:     settingsCache,
 		CacheExpiration:   cacheExpiration,
+		httpServices:      services.NewCostModelServices(),
 	}
 	// Use the Accesses instance, itself, as the CostModelAggregator. This is
 	// confusing and unconventional, but necessary so that we can swap it
@@ -1242,9 +1162,9 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		log.Infof("Init: AggregateCostModel cache warming disabled")
 	}
 
-	a.MetricsEmitter.Start()
-
-	managerEndpoints := cm.NewClusterManagerEndpoints(a.ClusterManager)
+	if !env.IsKubecostMetricsPodEnabled() {
+		a.MetricsEmitter.Start()
+	}
 
 	a.Router.GET("/costDataModel", a.CostDataModel)
 	a.Router.GET("/costDataModelRange", a.CostDataModelRange)
@@ -1274,10 +1194,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 	a.Router.GET("/diagnostics/requestQueue", a.GetPrometheusQueueState)
 	a.Router.GET("/diagnostics/prometheusMetrics", a.GetPrometheusMetrics)
 
-	// cluster manager endpoints
-	a.Router.GET("/clusters", managerEndpoints.GetAllClusters)
-	a.Router.PUT("/clusters", managerEndpoints.PutCluster)
-	a.Router.DELETE("/clusters/:id", managerEndpoints.DeleteCluster)
+	a.httpServices.RegisterAll(a.Router)
 
 	return a
 }

pkg/env/costmodelenv.go

@@ -39,6 +39,7 @@ const (
 	EmitNamespaceAnnotationsMetricEnvVar = "EMIT_NAMESPACE_ANNOTATIONS_METRIC"
 
 	EmitKsmV1MetricsEnvVar = "EMIT_KSM_V1_METRICS"
+	EmitKsmV1MetricsOnly   = "EMIT_KSM_V1_METRICS_ONLY"
 
 	ThanosEnabledEnvVar      = "THANOS_ENABLED"
 	ThanosQueryUrlEnvVar     = "THANOS_QUERY_URL"
@@ -72,7 +73,8 @@ const (
 
 	PromClusterIDLabelEnvVar = "PROM_CLUSTER_ID_LABEL"
 
-	PricingConfigmapName = "PRICING_CONFIGMAP_NAME"
+	PricingConfigmapName  = "PRICING_CONFIGMAP_NAME"
+	KubecostJobNameEnvVar = "KUBECOST_JOB_NAME"
 )
 
 func GetPricingConfigmapName() string {
@@ -82,7 +84,7 @@ func GetPricingConfigmapName() string {
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAppVersion() string {
-	return Get(AppVersionEnvVar, "1.87.3")
+	return Get(AppVersionEnvVar, "1.88.0")
 }
 
 // IsEmitNamespaceAnnotationsMetric returns true if cost-model is configured to emit the kube_namespace_annotations metric
@@ -103,6 +105,10 @@ func IsEmitKsmV1Metrics() bool {
 	return GetBool(EmitKsmV1MetricsEnvVar, true)
 }
 
+func IsEmitKsmV1MetricsOnly() bool {
+	return GetBool(EmitKsmV1MetricsOnly, false)
+}
+
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAWSAccessKeyID() string {
@@ -355,6 +361,11 @@ func GetParsedUTCOffset() time.Duration {
 	return offset
 }
 
+// GetKubecostJobName returns the environment variable value for KubecostJobNameEnvVar
+func GetKubecostJobName() string {
+	return Get(KubecostJobNameEnvVar, "kubecost")
+}
+
 func IsCacheWarmingEnabled() bool {
 	return GetBool(CacheWarmingEnabledEnvVar, true)
 }

pkg/kubecost/allocation.go

@@ -121,7 +121,7 @@ func (pv *PVAllocations) Clone() PVAllocations {
 		return nil
 	}
 	apv := *pv
-	clonePV := PVAllocations{}
+	clonePV := make(map[PVKey]*PVAllocation, len(apv))
 	for k, v := range apv {
 		clonePV[k] = &PVAllocation{
 			ByteHours: v.ByteHours,
@@ -1679,17 +1679,17 @@ func (as *AllocationSet) Clone() *AllocationSet {
 	as.RLock()
 	defer as.RUnlock()
 
-	allocs := map[string]*Allocation{}
+	allocs := make(map[string]*Allocation, len(as.allocations))
 	for k, v := range as.allocations {
 		allocs[k] = v.Clone()
 	}
 
-	externalKeys := map[string]bool{}
+	externalKeys := make(map[string]bool, len(as.externalKeys))
 	for k, v := range as.externalKeys {
 		externalKeys[k] = v
 	}
 
-	idleKeys := map[string]bool{}
+	idleKeys := make(map[string]bool, len(as.idleKeys))
 	for k, v := range as.idleKeys {
 		idleKeys[k] = v
 	}
@@ -1754,12 +1754,12 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 			}
 
 			cpuCost := node.CPUCost * (1.0 - node.Discount) * adjustmentRate
-			gpuCost := node.GPUCost * (1.0 - node.Discount) * adjustmentRate
 			ramCost := node.RAMCost * (1.0 - node.Discount) * adjustmentRate
+			gpuCost := node.GPUCost * (1.0) * adjustmentRate
 
 			assetClusterResourceCosts[node.Properties().Cluster]["cpu"] += cpuCost
-			assetClusterResourceCosts[node.Properties().Cluster]["gpu"] += gpuCost
 			assetClusterResourceCosts[node.Properties().Cluster]["ram"] += ramCost
+			assetClusterResourceCosts[node.Properties().Cluster]["gpu"] += gpuCost
 		}
 	})
 
@@ -1891,12 +1891,12 @@ func (as *AllocationSet) ComputeIdleAllocationsByNode(assetSet *AssetSet) (map[s
 			}
 
 			cpuCost := node.CPUCost * (1.0 - node.Discount) * adjustmentRate
-			gpuCost := node.GPUCost * (1.0 - node.Discount) * adjustmentRate
 			ramCost := node.RAMCost * (1.0 - node.Discount) * adjustmentRate
+			gpuCost := node.GPUCost * adjustmentRate
 
 			assetNodeResourceCosts[node.Properties().ProviderID]["cpu"] += cpuCost
-			assetNodeResourceCosts[node.Properties().ProviderID]["gpu"] += gpuCost
 			assetNodeResourceCosts[node.Properties().ProviderID]["ram"] += ramCost
+			assetNodeResourceCosts[node.Properties().ProviderID]["gpu"] += gpuCost
 		}
 	})
 

pkg/kubecost/allocationprops.go

@@ -133,13 +133,13 @@ func (p *AllocationProperties) Clone() *AllocationProperties {
 	}
 	clone.Services = services
 
-	labels := make(map[string]string)
+	labels := make(map[string]string, len(p.Labels))
 	for k, v := range p.Labels {
 		labels[k] = v
 	}
 	clone.Labels = labels
 
-	annotations := make(map[string]string)
+	annotations := make(map[string]string, len(p.Annotations))
 	for k, v := range p.Annotations {
 		annotations[k] = v
 	}

pkg/kubecost/asset.go

@@ -1,7 +1,6 @@
 package kubecost
 
 import (
-	"bytes"
 	"encoding"
 	"fmt"
 	"strings"
@@ -600,21 +599,6 @@ func (a *Any) Equal(that Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshaler
-func (a *Any) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncode(buffer, "properties", a.Properties(), ",")
-	jsonEncode(buffer, "labels", a.Labels(), ",")
-	jsonEncode(buffer, "window", a.Window(), ",")
-	jsonEncodeString(buffer, "start", a.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", a.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", a.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "adjustment", a.Adjustment(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (a *Any) String() string {
 	return toString(a)
@@ -756,7 +740,7 @@ func (ca *Cloud) Add(a Asset) Asset {
 	any.SetProperties(props)
 	any.SetLabels(labels)
 	any.adjustment = ca.Adjustment() + a.Adjustment()
-	any.Cost = (ca.TotalCost() - ca.Adjustment() - ca.Credit) + (a.TotalCost() - a.Adjustment() - ca.Credit)
+	any.Cost = (ca.TotalCost() - ca.Adjustment()) + (a.TotalCost() - a.Adjustment())
 
 	return any
 }
@@ -842,23 +826,6 @@ func (ca *Cloud) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshaler
-func (ca *Cloud) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", ca.Type().String(), ",")
-	jsonEncode(buffer, "properties", ca.Properties(), ",")
-	jsonEncode(buffer, "labels", ca.Labels(), ",")
-	jsonEncode(buffer, "window", ca.Window(), ",")
-	jsonEncodeString(buffer, "start", ca.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", ca.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", ca.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "adjustment", ca.Adjustment(), ",")
-	jsonEncodeFloat64(buffer, "credit", ca.Credit, ",")
-	jsonEncodeFloat64(buffer, "totalCost", ca.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (ca *Cloud) String() string {
 	return toString(ca)
@@ -1042,21 +1009,6 @@ func (cm *ClusterManagement) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshler
-func (cm *ClusterManagement) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", cm.Type().String(), ",")
-	jsonEncode(buffer, "properties", cm.Properties(), ",")
-	jsonEncode(buffer, "labels", cm.Labels(), ",")
-	jsonEncode(buffer, "window", cm.Window(), ",")
-	jsonEncodeString(buffer, "start", cm.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", cm.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", cm.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", cm.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (cm *ClusterManagement) String() string {
 	return toString(cm)
@@ -1323,25 +1275,6 @@ func (d *Disk) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements the json.Marshaler interface
-func (d *Disk) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", d.Type().String(), ",")
-	jsonEncode(buffer, "properties", d.Properties(), ",")
-	jsonEncode(buffer, "labels", d.Labels(), ",")
-	jsonEncode(buffer, "window", d.Window(), ",")
-	jsonEncodeString(buffer, "start", d.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", d.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", d.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "byteHours", d.ByteHours, ",")
-	jsonEncodeFloat64(buffer, "bytes", d.Bytes(), ",")
-	jsonEncode(buffer, "breakdown", d.Breakdown, ",")
-	jsonEncodeFloat64(buffer, "adjustment", d.Adjustment(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", d.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (d *Disk) String() string {
 	return toString(d)
@@ -1640,22 +1573,6 @@ func (n *Network) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshal interface
-func (n *Network) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", n.Type().String(), ",")
-	jsonEncode(buffer, "properties", n.Properties(), ",")
-	jsonEncode(buffer, "labels", n.Labels(), ",")
-	jsonEncode(buffer, "window", n.Window(), ",")
-	jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (n *Network) String() string {
 	return toString(n)
@@ -2000,36 +1917,6 @@ func (n *Node) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshal interface
-func (n *Node) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", n.Type().String(), ",")
-	jsonEncode(buffer, "properties", n.Properties(), ",")
-	jsonEncode(buffer, "labels", n.Labels(), ",")
-	jsonEncode(buffer, "window", n.Window(), ",")
-	jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
-	jsonEncodeString(buffer, "nodeType", n.NodeType, ",")
-	jsonEncodeFloat64(buffer, "cpuCores", n.CPUCores(), ",")
-	jsonEncodeFloat64(buffer, "ramBytes", n.RAMBytes(), ",")
-	jsonEncodeFloat64(buffer, "cpuCoreHours", n.CPUCoreHours, ",")
-	jsonEncodeFloat64(buffer, "ramByteHours", n.RAMByteHours, ",")
-	jsonEncodeFloat64(buffer, "GPUHours", n.GPUHours, ",")
-	jsonEncode(buffer, "cpuBreakdown", n.CPUBreakdown, ",")
-	jsonEncode(buffer, "ramBreakdown", n.RAMBreakdown, ",")
-	jsonEncodeFloat64(buffer, "preemptible", n.Preemptible, ",")
-	jsonEncodeFloat64(buffer, "discount", n.Discount, ",")
-	jsonEncodeFloat64(buffer, "cpuCost", n.CPUCost, ",")
-	jsonEncodeFloat64(buffer, "gpuCost", n.GPUCost, ",")
-	jsonEncodeFloat64(buffer, "gpuCount", n.GPUs(), ",")
-	jsonEncodeFloat64(buffer, "ramCost", n.RAMCost, ",")
-	jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (n *Node) String() string {
 	return toString(n)
@@ -2306,22 +2193,6 @@ func (lb *LoadBalancer) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshal
-func (lb *LoadBalancer) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", lb.Type().String(), ",")
-	jsonEncode(buffer, "properties", lb.Properties(), ",")
-	jsonEncode(buffer, "labels", lb.Labels(), ",")
-	jsonEncode(buffer, "window", lb.Window(), ",")
-	jsonEncodeString(buffer, "start", lb.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", lb.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", lb.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "adjustment", lb.Adjustment(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", lb.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (lb *LoadBalancer) String() string {
 	return toString(lb)
@@ -2515,28 +2386,18 @@ func (sa *SharedAsset) Equal(a Asset) bool {
 	return true
 }
 
-// MarshalJSON implements json.Marshaler
-func (sa *SharedAsset) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBufferString("{")
-	jsonEncodeString(buffer, "type", sa.Type().String(), ",")
-	jsonEncode(buffer, "properties", sa.Properties(), ",")
-	jsonEncode(buffer, "labels", sa.Labels(), ",")
-	jsonEncode(buffer, "properties", sa.Properties(), ",")
-	jsonEncode(buffer, "labels", sa.Labels(), ",")
-	jsonEncode(buffer, "window", sa.Window(), ",")
-	jsonEncodeString(buffer, "start", sa.Start().Format(time.RFC3339), ",")
-	jsonEncodeString(buffer, "end", sa.End().Format(time.RFC3339), ",")
-	jsonEncodeFloat64(buffer, "minutes", sa.Minutes(), ",")
-	jsonEncodeFloat64(buffer, "totalCost", sa.TotalCost(), "")
-	buffer.WriteString("}")
-	return buffer.Bytes(), nil
-}
-
 // String implements fmt.Stringer
 func (sa *SharedAsset) String() string {
 	return toString(sa)
 }
 
+// This type exists because only the assets map of AssetSet is marshaled to
+// json, which makes it impossible to recreate an AssetSet struct. Thus,
+// the type when unmarshaling a marshaled AssetSet,is AssetSetResponse
+type AssetSetResponse struct {
+	Assets map[string]Asset
+}
+
 // AssetSet stores a set of Assets, each with a unique name, that share
 // a window. An AssetSet is mutable, so treat it like a threadsafe map.
 type AssetSet struct {
@@ -2641,7 +2502,7 @@ func (as *AssetSet) Clone() *AssetSet {
 		aggregateBy = append([]string{}, as.aggregateBy...)
 	}
 
-	assets := map[string]Asset{}
+	assets := make(map[string]Asset, len(as.assets))
 	for k, v := range as.assets {
 		assets[k] = v.Clone()
 	}
@@ -2831,13 +2692,6 @@ func (as *AssetSet) Map() map[string]Asset {
 	return as.Clone().assets
 }
 
-// MarshalJSON JSON-encodes the AssetSet
-func (as *AssetSet) MarshalJSON() ([]byte, error) {
-	as.RLock()
-	defer as.RUnlock()
-	return json.Marshal(as.assets)
-}
-
 func (as *AssetSet) Set(asset Asset, aggregateBy []string) error {
 	if as.IsEmpty() {
 		as.Lock()
@@ -3045,6 +2899,14 @@ func (asr *AssetSetRange) MarshalJSON() ([]byte, error) {
 	return json.Marshal(asr.assets)
 }
 
+// As with AssetSet, AssetSetRange does not serialize all its fields,
+// making it impossible to reconstruct the AssetSetRange from its json.
+// Therefore, the type a marshaled AssetSetRange unmarshals to is
+// AssetSetRangeResponse
+type AssetSetRangeResponse struct {
+	Assets []*AssetSetResponse
+}
+
 func (asr *AssetSetRange) UTCOffset() time.Duration {
 	if asr.Length() == 0 {
 		return 0
@@ -3135,6 +2997,15 @@ func (asr *AssetSetRange) Minutes() float64 {
 	return duration.Minutes()
 }
 
+// This is a helper type. The Asset API returns a json which cannot be natively
+// unmarshaled into any Asset struct. Therefore, this struct IN COMBINATION WITH
+// DESERIALIZATION LOGIC DEFINED IN asset_unmarshal.go can unmarshal a json directly
+// from an Assets API query
+type AssetAPIResponse struct {
+	Code int                   `json:"code"`
+	Data AssetSetRangeResponse `json:"data"`
+}
+
 // Returns true if string slices a and b contain all of the same strings, in any order.
 func sameContents(a, b []string) bool {
 	if len(a) != len(b) {

pkg/kubecost/asset_unmarshal.go

@@ -0,0 +1,953 @@
+package kubecost
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"time"
+
+	// gojson is default golang json, required for RawMessage decoding
+	gojson "encoding/json"
+
+	"github.com/kubecost/cost-model/pkg/util/json"
+)
+
+// Encoding and decoding logic for Asset types
+
+// Any marshal and unmarshal
+
+// MarshalJSON implements json.Marshaler
+func (a *Any) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncode(buffer, "properties", a.Properties(), ",")
+	jsonEncode(buffer, "labels", a.Labels(), ",")
+	jsonEncode(buffer, "window", a.Window(), ",")
+	jsonEncodeString(buffer, "start", a.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", a.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", a.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "adjustment", a.Adjustment(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (a *Any) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = a.InterfaceToAny(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to Any, carrying over relevant fields
+func (a *Any) InterfaceToAny(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	a.properties = &properties
+	a.labels = labels
+	a.start = start
+	a.end = end
+	a.window = Window{
+		start: &start,
+		end:   &end,
+	}
+
+	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
+		a.adjustment = adjustment.(float64)
+	}
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		a.Cost = Cost.(float64) - a.adjustment
+	}
+
+	return nil
+}
+
+// Cloud marshal and unmarshal
+
+// MarshalJSON implements json.Marshaler
+func (ca *Cloud) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", ca.Type().String(), ",")
+	jsonEncode(buffer, "properties", ca.Properties(), ",")
+	jsonEncode(buffer, "labels", ca.Labels(), ",")
+	jsonEncode(buffer, "window", ca.Window(), ",")
+	jsonEncodeString(buffer, "start", ca.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", ca.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", ca.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "adjustment", ca.Adjustment(), ",")
+	jsonEncodeFloat64(buffer, "credit", ca.Credit, ",")
+	jsonEncodeFloat64(buffer, "totalCost", ca.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (ca *Cloud) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = ca.InterfaceToCloud(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to Cloud, carrying over relevant fields
+func (ca *Cloud) InterfaceToCloud(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	ca.properties = &properties
+	ca.labels = labels
+	ca.start = start
+	ca.end = end
+	ca.window = Window{
+		start: &start,
+		end:   &end,
+	}
+
+	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
+		ca.adjustment = adjustment.(float64)
+	}
+	if Credit, err := getTypedVal(fmap["credit"]); err == nil {
+		ca.Credit = Credit.(float64)
+	}
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		ca.Cost = Cost.(float64) - ca.adjustment - ca.Credit
+	}
+
+	return nil
+}
+
+// ClusterManagement marshal and unmarshal
+
+// MarshalJSON implements json.Marshler
+func (cm *ClusterManagement) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", cm.Type().String(), ",")
+	jsonEncode(buffer, "properties", cm.Properties(), ",")
+	jsonEncode(buffer, "labels", cm.Labels(), ",")
+	jsonEncode(buffer, "window", cm.Window(), ",")
+	jsonEncodeString(buffer, "start", cm.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", cm.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", cm.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", cm.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (cm *ClusterManagement) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = cm.InterfaceToClusterManagement(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to ClusterManagement, carrying over relevant fields
+func (cm *ClusterManagement) InterfaceToClusterManagement(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	cm.properties = &properties
+	cm.labels = labels
+	cm.window = Window{
+		start: &start,
+		end:   &end,
+	}
+
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		cm.Cost = Cost.(float64)
+	}
+
+	return nil
+}
+
+// Disk marshal and unmarshal
+
+// MarshalJSON implements the json.Marshaler interface
+func (d *Disk) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", d.Type().String(), ",")
+	jsonEncode(buffer, "properties", d.Properties(), ",")
+	jsonEncode(buffer, "labels", d.Labels(), ",")
+	jsonEncode(buffer, "window", d.Window(), ",")
+	jsonEncodeString(buffer, "start", d.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", d.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", d.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "byteHours", d.ByteHours, ",")
+	jsonEncodeFloat64(buffer, "bytes", d.Bytes(), ",")
+	jsonEncode(buffer, "breakdown", d.Breakdown, ",")
+	jsonEncodeFloat64(buffer, "adjustment", d.Adjustment(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", d.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (d *Disk) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = d.InterfaceToDisk(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to Disk, carrying over relevant fields
+func (d *Disk) InterfaceToDisk(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	fbreakdown := fmap["breakdown"].(map[string]interface{})
+
+	breakdown := toBreakdown(fbreakdown)
+
+	d.properties = &properties
+	d.labels = labels
+	d.start = start
+	d.end = end
+	d.window = Window{
+		start: &start,
+		end:   &end,
+	}
+	d.Breakdown = &breakdown
+
+	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
+		d.adjustment = adjustment.(float64)
+	}
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		d.Cost = Cost.(float64) - d.adjustment
+	}
+	if ByteHours, err := getTypedVal(fmap["byteHours"]); err == nil {
+		d.ByteHours = ByteHours.(float64)
+	}
+
+	// d.Local is not marhsaled, and cannot be calculated from marshaled values.
+	// Currently, it is just ignored and not set in the resulting unmarshal to Disk
+	//  be aware that this means a resulting Disk from an unmarshal is therefore NOT
+	// equal to the originally marshaled Disk.
+
+	return nil
+
+}
+
+// Network marshal and unmarshal
+
+// MarshalJSON implements json.Marshal interface
+func (n *Network) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", n.Type().String(), ",")
+	jsonEncode(buffer, "properties", n.Properties(), ",")
+	jsonEncode(buffer, "labels", n.Labels(), ",")
+	jsonEncode(buffer, "window", n.Window(), ",")
+	jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (n *Network) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = n.InterfaceToNetwork(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to Network, carrying over relevant fields
+func (n *Network) InterfaceToNetwork(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	n.properties = &properties
+	n.labels = labels
+	n.start = start
+	n.end = end
+	n.window = Window{
+		start: &start,
+		end:   &end,
+	}
+
+	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
+		n.adjustment = adjustment.(float64)
+	}
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		n.Cost = Cost.(float64) - n.adjustment
+	}
+
+	return nil
+
+}
+
+// Node marshal and unmarshal
+
+// MarshalJSON implements json.Marshal interface
+func (n *Node) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", n.Type().String(), ",")
+	jsonEncode(buffer, "properties", n.Properties(), ",")
+	jsonEncode(buffer, "labels", n.Labels(), ",")
+	jsonEncode(buffer, "window", n.Window(), ",")
+	jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
+	jsonEncodeString(buffer, "nodeType", n.NodeType, ",")
+	jsonEncodeFloat64(buffer, "cpuCores", n.CPUCores(), ",")
+	jsonEncodeFloat64(buffer, "ramBytes", n.RAMBytes(), ",")
+	jsonEncodeFloat64(buffer, "cpuCoreHours", n.CPUCoreHours, ",")
+	jsonEncodeFloat64(buffer, "ramByteHours", n.RAMByteHours, ",")
+	jsonEncodeFloat64(buffer, "GPUHours", n.GPUHours, ",")
+	jsonEncode(buffer, "cpuBreakdown", n.CPUBreakdown, ",")
+	jsonEncode(buffer, "ramBreakdown", n.RAMBreakdown, ",")
+	jsonEncodeFloat64(buffer, "preemptible", n.Preemptible, ",")
+	jsonEncodeFloat64(buffer, "discount", n.Discount, ",")
+	jsonEncodeFloat64(buffer, "cpuCost", n.CPUCost, ",")
+	jsonEncodeFloat64(buffer, "gpuCost", n.GPUCost, ",")
+	jsonEncodeFloat64(buffer, "gpuCount", n.GPUs(), ",")
+	jsonEncodeFloat64(buffer, "ramCost", n.RAMCost, ",")
+	jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (n *Node) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = n.InterfaceToNode(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to Node, carrying over relevant fields
+func (n *Node) InterfaceToNode(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	fcpuBreakdown := fmap["cpuBreakdown"].(map[string]interface{})
+	framBreakdown := fmap["ramBreakdown"].(map[string]interface{})
+
+	cpuBreakdown := toBreakdown(fcpuBreakdown)
+	ramBreakdown := toBreakdown(framBreakdown)
+
+	n.properties = &properties
+	n.labels = labels
+	n.start = start
+	n.end = end
+	n.window = Window{
+		start: &start,
+		end:   &end,
+	}
+	n.CPUBreakdown = &cpuBreakdown
+	n.RAMBreakdown = &ramBreakdown
+
+	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
+		n.adjustment = adjustment.(float64)
+	}
+	if NodeType, err := getTypedVal(fmap["nodeType"]); err == nil {
+		n.NodeType = NodeType.(string)
+	}
+	if CPUCoreHours, err := getTypedVal(fmap["cpuCoreHours"]); err == nil {
+		n.CPUCoreHours = CPUCoreHours.(float64)
+	}
+	if RAMByteHours, err := getTypedVal(fmap["ramByteHours"]); err == nil {
+		n.RAMByteHours = RAMByteHours.(float64)
+	}
+	if GPUHours, err := getTypedVal(fmap["GPUHours"]); err == nil {
+		n.GPUHours = GPUHours.(float64)
+	}
+	if CPUCost, err := getTypedVal(fmap["cpuCost"]); err == nil {
+		n.CPUCost = CPUCost.(float64)
+	}
+	if GPUCost, err := getTypedVal(fmap["gpuCost"]); err == nil {
+		n.GPUCost = GPUCost.(float64)
+	}
+	if GPUCount, err := getTypedVal(fmap["gpuCount"]); err == nil {
+		n.GPUCount = GPUCount.(float64)
+	}
+	if RAMCost, err := getTypedVal(fmap["ramCost"]); err == nil {
+		n.RAMCost = RAMCost.(float64)
+	}
+	if Discount, err := getTypedVal(fmap["discount"]); err == nil {
+		n.Discount = Discount.(float64)
+	}
+	if Preemptible, err := getTypedVal(fmap["preemptible"]); err == nil {
+		n.Preemptible = Preemptible.(float64)
+	}
+
+	return nil
+}
+
+// Loadbalancer marshal and unmarshal
+
+// MarshalJSON implements json.Marshal
+func (lb *LoadBalancer) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", lb.Type().String(), ",")
+	jsonEncode(buffer, "properties", lb.Properties(), ",")
+	jsonEncode(buffer, "labels", lb.Labels(), ",")
+	jsonEncode(buffer, "window", lb.Window(), ",")
+	jsonEncodeString(buffer, "start", lb.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", lb.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", lb.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "adjustment", lb.Adjustment(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", lb.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (lb *LoadBalancer) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = lb.InterfaceToLoadBalancer(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to LoadBalancer, carrying over relevant fields
+func (lb *LoadBalancer) InterfaceToLoadBalancer(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	lb.properties = &properties
+	lb.labels = labels
+	lb.start = start
+	lb.end = end
+	lb.window = Window{
+		start: &start,
+		end:   &end,
+	}
+
+	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
+		lb.adjustment = adjustment.(float64)
+	}
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		lb.Cost = Cost.(float64) - lb.adjustment
+	}
+
+	return nil
+
+}
+
+// SharedAsset marshal and unmarshal
+
+// MarshalJSON implements json.Marshaler
+func (sa *SharedAsset) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "type", sa.Type().String(), ",")
+	jsonEncode(buffer, "properties", sa.Properties(), ",")
+	jsonEncode(buffer, "labels", sa.Labels(), ",")
+	jsonEncode(buffer, "properties", sa.Properties(), ",")
+	jsonEncode(buffer, "labels", sa.Labels(), ",")
+	jsonEncode(buffer, "window", sa.Window(), ",")
+	jsonEncodeString(buffer, "start", sa.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", sa.End().Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", sa.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "totalCost", sa.TotalCost(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+func (sa *SharedAsset) UnmarshalJSON(b []byte) error {
+
+	var f interface{}
+
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	err = sa.InterfaceToSharedAsset(f)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Converts interface{} to SharedAsset, carrying over relevant fields
+func (sa *SharedAsset) InterfaceToSharedAsset(itf interface{}) error {
+
+	fmap := itf.(map[string]interface{})
+
+	// parse properties map to AssetProperties
+	fproperties := fmap["properties"].(map[string]interface{})
+	properties := toAssetProp(fproperties)
+
+	// parse labels map to AssetLabels
+	labels := make(map[string]string)
+	for k, v := range fmap["labels"].(map[string]interface{}) {
+		labels[k] = v.(string)
+	}
+
+	// parse start and end strings to time.Time
+	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
+	if err != nil {
+		return err
+	}
+	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
+	if err != nil {
+		return err
+	}
+
+	sa.properties = &properties
+	sa.labels = labels
+	sa.window = Window{
+		start: &start,
+		end:   &end,
+	}
+
+	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
+		sa.Cost = Cost.(float64)
+	}
+
+	return nil
+
+}
+
+// AssetSet marshal
+
+// MarshalJSON JSON-encodes the AssetSet
+func (as *AssetSet) MarshalJSON() ([]byte, error) {
+	as.RLock()
+	defer as.RUnlock()
+	return json.Marshal(as.assets)
+}
+
+// AssetSetResponse for unmarshaling of AssetSet.assets into AssetSet
+
+// Unmarshals a marshaled AssetSet json into AssetSetResponse
+func (asr *AssetSetResponse) UnmarshalJSON(b []byte) error {
+
+	// gojson used here, as jsonitter UnmarshalJSON won't work with RawMessage
+	var assetMap map[string]*gojson.RawMessage
+
+	// Partial unmarshal to map of json RawMessage
+	err := gojson.Unmarshal(b, &assetMap)
+	if err != nil {
+		return err
+	}
+
+	err = asr.RawMessageToAssetSetResponse(assetMap)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (asr *AssetSetResponse) RawMessageToAssetSetResponse(assetMap map[string]*gojson.RawMessage) error {
+
+	newAssetMap := make(map[string]Asset)
+
+	// For each item in asset map, unmarshal to appropriate type
+	for key, rawMessage := range assetMap {
+
+		var f interface{}
+
+		err := json.Unmarshal(*rawMessage, &f)
+		if err != nil {
+			return err
+		}
+
+		fmap := f.(map[string]interface{})
+
+		switch t := fmap["type"]; t {
+		case "Cloud":
+
+			var ca Cloud
+			err := ca.InterfaceToCloud(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &ca
+
+		case "ClusterManagement":
+
+			var cm ClusterManagement
+			err := cm.InterfaceToClusterManagement(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &cm
+
+		case "Disk":
+
+			var d Disk
+			err := d.InterfaceToDisk(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &d
+
+		case "Network":
+
+			var nw Network
+			err := nw.InterfaceToNetwork(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &nw
+
+		case "Node":
+
+			var n Node
+			err := n.InterfaceToNode(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &n
+
+		case "LoadBalancer":
+
+			var lb LoadBalancer
+			err := lb.InterfaceToLoadBalancer(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &lb
+
+		case "Shared":
+
+			var sa SharedAsset
+			err := sa.InterfaceToSharedAsset(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &sa
+
+		default:
+
+			var a Any
+			err := a.InterfaceToAny(f)
+
+			if err != nil {
+				return err
+			}
+
+			newAssetMap[key] = &a
+
+		}
+	}
+
+	asr.Assets = newAssetMap
+
+	return nil
+}
+
+func (asrr *AssetSetRangeResponse) UnmarshalJSON(b []byte) error {
+
+	// gojson used here, as jsonitter UnmarshalJSON won't work with RawMessage
+	var assetMapList []map[string]*gojson.RawMessage
+
+	// Partial unmarshal to map of json RawMessage
+	err := gojson.Unmarshal(b, &assetMapList)
+	if err != nil {
+		return err
+	}
+
+	var assetSetList []*AssetSetResponse
+
+	for _, rawm := range assetMapList {
+
+		var asresp AssetSetResponse
+		err = asresp.RawMessageToAssetSetResponse(rawm)
+		if err != nil {
+			return err
+		}
+
+		assetSetList = append(assetSetList, &asresp)
+
+	}
+
+	asrr.Assets = assetSetList
+
+	return nil
+}
+
+// Extra decoding util functions, for clarity
+
+// Creates an AssetProperties directly from map[string]interface{}
+func toAssetProp(fproperties map[string]interface{}) AssetProperties {
+	var properties AssetProperties
+
+	if category, v := fproperties["category"].(string); v {
+		properties.Category = category
+	}
+	if provider, v := fproperties["provider"].(string); v {
+		properties.Provider = provider
+	}
+	if account, v := fproperties["account"].(string); v {
+		properties.Account = account
+	}
+	if project, v := fproperties["project"].(string); v {
+		properties.Project = project
+	}
+	if service, v := fproperties["service"].(string); v {
+		properties.Service = service
+	}
+	if cluster, v := fproperties["cluster"].(string); v {
+		properties.Cluster = cluster
+	}
+	if name, v := fproperties["name"].(string); v {
+		properties.Name = name
+	}
+	if providerID, v := fproperties["providerID"].(string); v {
+		properties.ProviderID = providerID
+	}
+
+	return properties
+
+}
+
+// Creates an Breakdown directly from map[string]interface{}
+func toBreakdown(fproperties map[string]interface{}) Breakdown {
+	var breakdown Breakdown
+
+	if idle, v := fproperties["idle"].(float64); v {
+		breakdown.Idle = idle
+	}
+	if other, v := fproperties["other"].(float64); v {
+		breakdown.Other = other
+	}
+	if system, v := fproperties["system"].(float64); v {
+		breakdown.System = system
+	}
+	if user, v := fproperties["user"].(float64); v {
+		breakdown.User = user
+	}
+
+	return breakdown
+
+}
+
+// Not strictly nessesary, but cleans up the code and is a secondary check
+// for correct types
+func getTypedVal(itf interface{}) (interface{}, error) {
+	switch itf := itf.(type) {
+	case float64:
+		return float64(itf), nil
+	case string:
+		return string(itf), nil
+	default:
+		unktype := reflect.ValueOf(itf)
+		return nil, fmt.Errorf("Type %v is an invalid type", unktype)
+	}
+}

pkg/kubecost/asset_unmarshal_test.go

@@ -0,0 +1,552 @@
+package kubecost
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+var s = time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
+var e = start1.Add(day)
+var unmarshalWindow = NewWindow(&s, &e)
+
+func TestAny_Unmarshal(t *testing.T) {
+
+	any1 := NewAsset(*unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	any1.SetProperties(&AssetProperties{
+		Name:       "any1",
+		Cluster:    "cluster1",
+		ProviderID: "any1",
+	})
+	any1.Cost = 9.0
+	any1.SetAdjustment(1.0)
+
+	bytes, _ := json.Marshal(any1)
+
+	var testany Any
+	any2 := &testany
+
+	err := json.Unmarshal(bytes, any2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("Any Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial Any equal those in Any from unmarshal
+	if !any1.properties.Equal(any2.properties) {
+		t.Fatalf("Any Unmarshal: properties mutated in unmarshal")
+	}
+	if !any1.labels.Equal(any2.labels) {
+		t.Fatalf("Any Unmarshal: labels mutated in unmarshal")
+	}
+	if !any1.window.Equal(any2.window) {
+		t.Fatalf("Any Unmarshal: window mutated in unmarshal")
+	}
+	if !any1.start.Equal(any2.start) {
+		t.Fatalf("Any Unmarshal: start mutated in unmarshal")
+	}
+	if !any1.end.Equal(any2.end) {
+		t.Fatalf("Any Unmarshal: end mutated in unmarshal")
+	}
+	if any1.adjustment != any2.adjustment {
+		t.Fatalf("Any Unmarshal: adjustment mutated in unmarshal")
+	}
+	if any1.Cost != any2.Cost {
+		t.Fatalf("Any Unmarshal: cost mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !any1.Equal(any2) {
+		t.Fatalf("Any Unmarshal: Any mutated in unmarshal")
+	}
+
+}
+
+func TestCloud_Unmarshal(t *testing.T) {
+
+	cloud1 := NewCloud("Compute", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	cloud1.SetLabels(map[string]string{
+		"namespace": "namespace1",
+		"env":       "env1",
+		"product":   "product1",
+	})
+	cloud1.Cost = 10.00
+	cloud1.Credit = -1.0
+
+	bytes, _ := json.Marshal(cloud1)
+
+	var testcloud Cloud
+	cloud2 := &testcloud
+
+	err := json.Unmarshal(bytes, cloud2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("Cloud Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial Cloud equal those in Cloud from unmarshal
+	if !cloud1.properties.Equal(cloud2.properties) {
+		t.Fatalf("Cloud Unmarshal: properties mutated in unmarshal")
+	}
+	if !cloud1.labels.Equal(cloud2.labels) {
+		t.Fatalf("Cloud Unmarshal: labels mutated in unmarshal")
+	}
+	if !cloud1.window.Equal(cloud2.window) {
+		t.Fatalf("Cloud Unmarshal: window mutated in unmarshal")
+	}
+	if !cloud1.start.Equal(cloud2.start) {
+		t.Fatalf("Cloud Unmarshal: start mutated in unmarshal")
+	}
+	if !cloud1.end.Equal(cloud2.end) {
+		t.Fatalf("Cloud Unmarshal: end mutated in unmarshal")
+	}
+	if cloud1.adjustment != cloud2.adjustment {
+		t.Fatalf("Cloud Unmarshal: adjustment mutated in unmarshal")
+	}
+	if cloud1.Cost != cloud2.Cost {
+		t.Fatalf("Cloud Unmarshal: cost mutated in unmarshal")
+	}
+	if cloud1.Credit != cloud2.Credit {
+		t.Fatalf("Cloud Unmarshal: credit mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !cloud1.Equal(cloud2) {
+		t.Fatalf("Cloud Unmarshal: Cloud mutated in unmarshal")
+	}
+
+}
+
+func TestClusterManagement_Unmarshal(t *testing.T) {
+
+	cm1 := NewClusterManagement("gcp", "cluster1", unmarshalWindow)
+	cm1.Cost = 9.0
+
+	bytes, _ := json.Marshal(cm1)
+
+	var testcm ClusterManagement
+	cm2 := &testcm
+
+	err := json.Unmarshal(bytes, cm2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("ClusterManagement Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial ClusterManagement equal those in ClusterManagement from unmarshal
+	if !cm1.properties.Equal(cm2.properties) {
+		t.Fatalf("ClusterManagement Unmarshal: properties mutated in unmarshal")
+	}
+	if !cm1.labels.Equal(cm2.labels) {
+		t.Fatalf("ClusterManagement Unmarshal: labels mutated in unmarshal")
+	}
+	if !cm1.window.Equal(cm2.window) {
+		t.Fatalf("ClusterManagement Unmarshal: window mutated in unmarshal")
+	}
+	if cm1.Cost != cm2.Cost {
+		t.Fatalf("ClusterManagement Unmarshal: cost mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !cm1.Equal(cm2) {
+		t.Fatalf("ClusterManagement Unmarshal: ClusterManagement mutated in unmarshal")
+	}
+
+}
+
+func TestDisk_Unmarshal(t *testing.T) {
+
+	hours := unmarshalWindow.Duration().Hours()
+
+	disk1 := NewDisk("disk1", "cluster1", "disk1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	disk1.ByteHours = 60.0 * gb * hours
+	disk1.Cost = 4.0
+	disk1.Local = 1.0
+	disk1.SetAdjustment(1.0)
+	disk1.Breakdown = &Breakdown{
+		Idle:   0.1,
+		System: 0.2,
+		User:   0.3,
+		Other:  0.4,
+	}
+
+	bytes, _ := json.Marshal(disk1)
+
+	var testdisk Disk
+	disk2 := &testdisk
+
+	err := json.Unmarshal(bytes, disk2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("Disk Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial Disk equal those in Disk from unmarshal
+	if !disk1.properties.Equal(disk2.properties) {
+		t.Fatalf("Disk Unmarshal: properties mutated in unmarshal")
+	}
+	if !disk1.labels.Equal(disk2.labels) {
+		t.Fatalf("Disk Unmarshal: labels mutated in unmarshal")
+	}
+	if !disk1.window.Equal(disk2.window) {
+		t.Fatalf("Disk Unmarshal: window mutated in unmarshal")
+	}
+	if !disk1.Breakdown.Equal(disk2.Breakdown) {
+		t.Fatalf("Disk Unmarshal: Breakdown mutated in unmarshal")
+	}
+	if !disk1.start.Equal(disk2.start) {
+		t.Fatalf("Disk Unmarshal: start mutated in unmarshal")
+	}
+	if !disk1.end.Equal(disk2.end) {
+		t.Fatalf("Disk Unmarshal: end mutated in unmarshal")
+	}
+	if disk1.adjustment != disk2.adjustment {
+		t.Fatalf("Disk Unmarshal: adjustment mutated in unmarshal")
+	}
+	if disk1.ByteHours != disk2.ByteHours {
+		t.Fatalf("Disk Unmarshal: ByteHours mutated in unmarshal")
+	}
+	if disk1.Cost != disk2.Cost {
+		t.Fatalf("Disk Unmarshal: cost mutated in unmarshal")
+	}
+
+	// Local from Disk is not marhsaled, and cannot be calculated from marshaled values.
+	// Currently, it is just ignored and not set in the resulting unmarshal to Disk. Thus,
+	// it is also ignored in this test; be aware that this means a resulting Disk from an
+	// unmarshal is therefore NOT equal to the originally marshaled Disk.
+
+}
+
+func TestNetwork_Unmarshal(t *testing.T) {
+
+	network1 := NewNetwork("network1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	network1.Cost = 4.0
+	network1.SetAdjustment(1.0)
+
+	bytes, _ := json.Marshal(network1)
+
+	var testnw Network
+	network2 := &testnw
+
+	err := json.Unmarshal(bytes, network2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("Network Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial Network equal those in Network from unmarshal
+	if !network1.properties.Equal(network2.properties) {
+		t.Fatalf("Network Unmarshal: properties mutated in unmarshal")
+	}
+	if !network1.labels.Equal(network2.labels) {
+		t.Fatalf("Network Unmarshal: labels mutated in unmarshal")
+	}
+	if !network1.window.Equal(network2.window) {
+		t.Fatalf("Network Unmarshal: window mutated in unmarshal")
+	}
+	if !network1.start.Equal(network2.start) {
+		t.Fatalf("Network Unmarshal: start mutated in unmarshal")
+	}
+	if !network1.end.Equal(network2.end) {
+		t.Fatalf("Network Unmarshal: end mutated in unmarshal")
+	}
+	if network1.adjustment != network2.adjustment {
+		t.Fatalf("Network Unmarshal: adjustment mutated in unmarshal")
+	}
+	if network1.Cost != network2.Cost {
+		t.Fatalf("Network Unmarshal: cost mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !network1.Equal(network2) {
+		t.Fatalf("Network Unmarshal: Network mutated in unmarshal")
+	}
+
+}
+
+func TestNode_Unmarshal(t *testing.T) {
+
+	hours := unmarshalWindow.Duration().Hours()
+
+	node1 := NewNode("node1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	node1.CPUCoreHours = 1.0 * hours
+	node1.RAMByteHours = 2.0 * gb * hours
+	node1.GPUHours = 0.0 * hours
+	node1.GPUCost = 0.0
+	node1.CPUCost = 8.0
+	node1.RAMCost = 4.0
+	node1.Discount = 0.3
+	node1.CPUBreakdown = &Breakdown{
+		Idle:   0.6,
+		System: 0.2,
+		User:   0.2,
+		Other:  0.0,
+	}
+	node1.RAMBreakdown = &Breakdown{
+		Idle:   0.6,
+		System: 0.2,
+		User:   0.2,
+		Other:  0.0,
+	}
+	node1.SetAdjustment(1.6)
+
+	bytes, _ := json.Marshal(node1)
+
+	var testnode Node
+	node2 := &testnode
+
+	err := json.Unmarshal(bytes, node2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("Node Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial Node equal those in Node from unmarshal
+	if !node1.properties.Equal(node2.properties) {
+		t.Fatalf("Node Unmarshal: properties mutated in unmarshal")
+	}
+	if !node1.labels.Equal(node2.labels) {
+		t.Fatalf("Node Unmarshal: labels mutated in unmarshal")
+	}
+	if !node1.window.Equal(node2.window) {
+		t.Fatalf("Node Unmarshal: window mutated in unmarshal")
+	}
+	if !node1.CPUBreakdown.Equal(node2.CPUBreakdown) {
+		t.Fatalf("Node Unmarshal: CPUBreakdown mutated in unmarshal")
+	}
+	if !node1.RAMBreakdown.Equal(node2.RAMBreakdown) {
+		t.Fatalf("Node Unmarshal: RAMBreakdown mutated in unmarshal")
+	}
+	if !node1.start.Equal(node2.start) {
+		t.Fatalf("Node Unmarshal: start mutated in unmarshal")
+	}
+	if !node1.end.Equal(node2.end) {
+		t.Fatalf("Node Unmarshal: end mutated in unmarshal")
+	}
+	if node1.adjustment != node2.adjustment {
+		t.Fatalf("Node Unmarshal: adjustment mutated in unmarshal")
+	}
+	if node1.NodeType != node2.NodeType {
+		t.Fatalf("Node Unmarshal: NodeType mutated in unmarshal")
+	}
+	if node1.CPUCoreHours != node2.CPUCoreHours {
+		t.Fatalf("Node Unmarshal: CPUCoreHours mutated in unmarshal")
+	}
+	if node1.RAMByteHours != node2.RAMByteHours {
+		t.Fatalf("Node Unmarshal: RAMByteHours mutated in unmarshal")
+	}
+	if node1.GPUHours != node2.GPUHours {
+		t.Fatalf("Node Unmarshal: GPUHours mutated in unmarshal")
+	}
+	if node1.CPUCost != node2.CPUCost {
+		t.Fatalf("Node Unmarshal: CPUCost mutated in unmarshal")
+	}
+	if node1.GPUCost != node2.GPUCost {
+		t.Fatalf("Node Unmarshal: GPUCost mutated in unmarshal")
+	}
+	if node1.GPUCount != node2.GPUCount {
+		t.Fatalf("Node Unmarshal: GPUCount mutated in unmarshal")
+	}
+	if node1.RAMCost != node2.RAMCost {
+		t.Fatalf("Node Unmarshal: RAMCost mutated in unmarshal")
+	}
+	if node1.Discount != node2.Discount {
+		t.Fatalf("Node Unmarshal: Discount mutated in unmarshal")
+	}
+	if node1.Preemptible != node2.Preemptible {
+		t.Fatalf("Node Unmarshal: Preemptible mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !node1.Equal(node2) {
+		t.Fatalf("Node Unmarshal: Node mutated in unmarshal")
+	}
+
+}
+
+func TestLoadBalancer_Unmarshal(t *testing.T) {
+
+	lb1 := NewLoadBalancer("loadbalancer1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	lb1.Cost = 12.0
+	lb1.SetAdjustment(4.0)
+
+	bytes, _ := json.Marshal(lb1)
+
+	var testlb LoadBalancer
+	lb2 := &testlb
+
+	err := json.Unmarshal(bytes, lb2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("LoadBalancer Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial LoadBalancer equal those in LoadBalancer from unmarshal
+	if !lb1.properties.Equal(lb2.properties) {
+		t.Fatalf("LoadBalancer Unmarshal: properties mutated in unmarshal")
+	}
+	if !lb1.labels.Equal(lb2.labels) {
+		t.Fatalf("LoadBalancer Unmarshal: labels mutated in unmarshal")
+	}
+	if !lb1.window.Equal(lb2.window) {
+		t.Fatalf("LoadBalancer Unmarshal: window mutated in unmarshal")
+	}
+	if !lb1.start.Equal(lb2.start) {
+		t.Fatalf("LoadBalancer Unmarshal: start mutated in unmarshal")
+	}
+	if !lb1.end.Equal(lb2.end) {
+		t.Fatalf("LoadBalancer Unmarshal: end mutated in unmarshal")
+	}
+	if lb1.adjustment != lb2.adjustment {
+		t.Fatalf("LoadBalancer Unmarshal: adjustment mutated in unmarshal")
+	}
+	if lb1.Cost != lb2.Cost {
+		t.Fatalf("LoadBalancer Unmarshal: cost mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !lb1.Equal(lb2) {
+		t.Fatalf("LoadBalancer Unmarshal: LoadBalancer mutated in unmarshal")
+	}
+
+}
+
+func TestSharedAsset_Unmarshal(t *testing.T) {
+
+	sa1 := NewSharedAsset("sharedasset1", unmarshalWindow)
+	sa1.Cost = 7.0
+
+	bytes, _ := json.Marshal(sa1)
+
+	var testsa SharedAsset
+	sa2 := &testsa
+
+	err := json.Unmarshal(bytes, sa2)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("SharedAsset Unmarshal: unexpected error: %s", err)
+	}
+
+	// Check if all fields in initial SharedAsset equal those in SharedAsset from unmarshal
+	if !sa1.properties.Equal(sa2.properties) {
+		t.Fatalf("SharedAsset Unmarshal: properties mutated in unmarshal")
+	}
+	if !sa1.labels.Equal(sa2.labels) {
+		t.Fatalf("SharedAsset Unmarshal: labels mutated in unmarshal")
+	}
+	if !sa1.window.Equal(sa2.window) {
+		t.Fatalf("SharedAsset Unmarshal: window mutated in unmarshal")
+	}
+	if sa1.Cost != sa2.Cost {
+		t.Fatalf("SharedAsset Unmarshal: cost mutated in unmarshal")
+	}
+
+	// As a final check, make sure the above checks out
+	if !sa1.Equal(sa2) {
+		t.Fatalf("SharedAsset Unmarshal: SharedAsset mutated in unmarshal")
+	}
+
+}
+
+func TestAssetset_Unmarshal(t *testing.T) {
+
+	var s time.Time
+	var e time.Time
+	unmarshalWindow := NewWindow(&s, &e)
+
+	any := NewAsset(*unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	cloud := NewCloud("Compute", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	cm := NewClusterManagement("gcp", "cluster1", unmarshalWindow)
+	disk := NewDisk("disk1", "cluster1", "disk1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	network := NewNetwork("network1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	node := NewNode("node1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	lb := NewLoadBalancer("loadbalancer1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	sa := NewSharedAsset("sharedasset1", unmarshalWindow)
+
+	assetList := []Asset{any, cloud, cm, disk, network, node, lb, sa}
+
+	assetset := NewAssetSet(s, e, assetList...)
+	bytes, _ := json.Marshal(assetset)
+
+	var assetSetResponse AssetSetResponse
+	assetUnmarshalResponse := &assetSetResponse
+
+	err := json.Unmarshal(bytes, assetUnmarshalResponse)
+
+	// Check if unmarshal was successful
+	if err != nil {
+		t.Fatalf("AssetSet Unmarshal: unexpected error: %s", err)
+	}
+
+	// For each asset in unmarshaled AssetSetResponse, check if it is equal to the corresponding AssetSet asset
+	for key, asset := range assetset.assets {
+
+		if unmarshaledAsset, exists := assetUnmarshalResponse.Assets[key]; exists {
+
+			// As Disk is not marshaled with all fields, the resultant Disk will be unequal. Test all fields we have instead.
+			if unmarshaledAsset.Type().String() == "Disk" {
+
+				udiskEq := func(d1 Asset, d2 Asset) bool {
+
+					asset, _ := asset.(*Disk)
+					unmarshaledAsset, _ := unmarshaledAsset.(*Disk)
+
+					if !asset.Labels().Equal(unmarshaledAsset.Labels()) {
+						return false
+					}
+					if !asset.Properties().Equal(unmarshaledAsset.Properties()) {
+						return false
+					}
+
+					if !asset.Start().Equal(unmarshaledAsset.Start()) {
+						return false
+					}
+					if !asset.End().Equal(unmarshaledAsset.End()) {
+						return false
+					}
+					if !asset.window.Equal(unmarshaledAsset.window) {
+						return false
+					}
+					if asset.adjustment != unmarshaledAsset.adjustment {
+						return false
+					}
+					if asset.Cost != unmarshaledAsset.Cost {
+						return false
+					}
+					if asset.ByteHours != unmarshaledAsset.ByteHours {
+						return false
+					}
+					if !asset.Breakdown.Equal(unmarshaledAsset.Breakdown) {
+						return false
+					}
+
+					return true
+				}
+
+				if res := udiskEq(asset, unmarshaledAsset); !res {
+					t.Fatalf("AssetSet Unmarshal: asset at key '%s' from unmarshaled AssetSetResponse does not match corresponding asset from AssetSet", key)
+				}
+
+			} else {
+
+				if !asset.Equal(unmarshaledAsset) {
+					t.Fatalf("AssetSet Unmarshal: asset at key '%s' from unmarshaled AssetSetResponse does not match corresponding asset from AssetSet", key)
+				}
+
+			}
+
+		} else {
+			t.Fatalf("AssetSet Unmarshal: key '%s' from marshaled AssetSet does not exist in AssetSetResponse", key)
+		}
+
+	}
+
+}

pkg/kubecost/config_test.go

@@ -38,7 +38,7 @@ func TestLabelConfig_Map(t *testing.T) {
 
 func TestLabelConfig_GetExternalAllocationName(t *testing.T) {
 	// Make sure that AWS's Glue/Athena column formatting is supported
-	glueFormattedLabel := cloudutil.ConvertToGlueColumnFormat("Non__GlueFormattedLabel")
+	glueFormattedLabel := cloudutil.ConvertToGlueColumnFormat("___Non_&GlueFormattedLabel___&")
 
 	labels := map[string]string{
 		"kubens":                      "kubecost-staging",

pkg/metrics/kubemetrics.go

@@ -28,6 +28,7 @@ type KubeMetricsOpts struct {
 	EmitNamespaceAnnotations      bool
 	EmitPodAnnotations            bool
 	EmitKubeStateMetrics          bool
+	EmitKubeStateMetricsV1Only    bool
 }
 
 // DefaultKubeMetricsOpts returns KubeMetricsOpts with default values set
@@ -37,6 +38,7 @@ func DefaultKubeMetricsOpts() *KubeMetricsOpts {
 		EmitNamespaceAnnotations:      false,
 		EmitPodAnnotations:            false,
 		EmitKubeStateMetrics:          true,
+		EmitKubeStateMetricsV1Only:    false,
 	}
 }
 
@@ -93,6 +95,16 @@ func InitKubeMetrics(clusterCache clustercache.ClusterCache, opts *KubeMetricsOp
 			prometheus.MustRegister(KubeJobCollector{
 				KubeClusterCache: clusterCache,
 			})
+		} else if opts.EmitKubeStateMetricsV1Only {
+			prometheus.MustRegister(KubeNodeCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeNamespaceCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePodLabelsCollector{
+				KubeClusterCache: clusterCache,
+			})
 		}
 	})
 }

pkg/metrics/podlabelmetrics.go

@@ -0,0 +1,72 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostPodCollector
+//--------------------------------------------------------------------------
+
+// KubecostPodCollector is a prometheus collector that emits pod metrics
+type KubecostPodLabelsCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpmc KubecostPodLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_annotations", "All annotations for each pod prefix with annotation_", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubecostPodLabelsCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+
+		// Pod Annotations
+		labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
+		if len(labels) > 0 {
+			ch <- newPodAnnotationMetric("kube_pod_annotations", podNS, podName, labels, values)
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePodLabelsCollector
+//--------------------------------------------------------------------------
+
+// KubePodLabelsCollector is a prometheus collector that emits pod labels only
+type KubePodLabelsCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of pod labels only
+// collected by this Collector.
+func (kpmc KubePodLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_labels", "All labels for each pod prefixed with label_", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_owner", "Information about the Pod's owner", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubePodLabelsCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+		podUID := string(pod.GetUID())
+
+		// Pod Labels
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
+		ch <- newKubePodLabelsMetric("kube_pod_labels", podNS, podName, podUID, labelNames, labelValues)
+
+		// Owner References
+		for _, owner := range pod.OwnerReferences {
+			ch <- newKubePodOwnerMetric("kube_pod_owner", podNS, podName, owner.Name, owner.Kind, owner.Controller != nil)
+		}
+	}
+}

pkg/prom/helpers.go

@@ -0,0 +1,62 @@
+package prom
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	prometheus "github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+
+	"gopkg.in/yaml.v2"
+)
+
+const PrometheusTroubleshootingURL = "http://docs.kubecost.com/custom-prom#troubleshoot"
+
+// ScrapeConfig is the minimalized view of a prometheus scrape configuration
+type ScrapeConfig struct {
+	JobName        string `yaml:"job_name,omitempty"`
+	ScrapeInterval string `yaml:"scrape_interval,omitempty"`
+}
+
+// PrometheusConfig is the minimalized view of a prometheus configuration
+type PrometheusConfig struct {
+	ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs,omitempty"`
+}
+
+// GetPrometheusConfig uses the provided yaml string to parse the minimalized prometheus config
+func GetPrometheusConfig(pcfg string) (PrometheusConfig, error) {
+	var promCfg PrometheusConfig
+	err := yaml.Unmarshal([]byte(pcfg), &promCfg)
+	return promCfg, err
+}
+
+// ScrapeIntervalFor uses the provided prometheus client to locate a scrape interval for a specific job name
+func ScrapeIntervalFor(client prometheus.Client, jobName string) (time.Duration, error) {
+	api := v1.NewAPI(client)
+	promConfig, err := api.Config(context.Background())
+	if err != nil {
+		return 0, err
+	}
+
+	cfg, err := GetPrometheusConfig(promConfig.YAML)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, sc := range cfg.ScrapeConfigs {
+		if sc.JobName == jobName {
+			if sc.ScrapeInterval != "" {
+				si := sc.ScrapeInterval
+				sid, err := time.ParseDuration(si)
+				if err != nil {
+					return 0, fmt.Errorf("Error parsing scrape config for %s", sc.JobName)
+				} else {
+					return sid, nil
+				}
+			}
+		}
+	}
+
+	return 0, fmt.Errorf("Failed to locate scrape config for %s", jobName)
+}

pkg/clustermanager/boltdbstorage.go → pkg/services/clusters/boltdbstorage.go

@@ -1,15 +1,16 @@
-package clustermanager
+package clusters
 
 import (
 	bolt "go.etcd.io/bbolt"
-	_ "k8s.io/klog"
 )
 
+// BoltDBClusterStorage is a boltdb implementation of a database used to store cluster definitions
 type BoltDBClusterStorage struct {
 	bucket []byte
 	db     *bolt.DB
 }
 
+// NewBoltDBClusterStorage creates a new boltdb backed ClusterStorage implementation
 func NewBoltDBClusterStorage(bucket string, db *bolt.DB) (ClusterStorage, error) {
 	bucketKey := []byte(bucket)
 
@@ -32,7 +33,7 @@ func NewBoltDBClusterStorage(bucket string, db *bolt.DB) (ClusterStorage, error)
 	}, nil
 }
 
-// Adds the entry if the key does not exist
+// AddIfNotExists Adds the entry if the key does not exist
 func (cs *BoltDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
 	return cs.db.Update(func(tx *bolt.Tx) error {
 		k := []byte(key)
@@ -45,7 +46,7 @@ func (cs *BoltDBClusterStorage) AddIfNotExists(key string, cluster []byte) error
 	})
 }
 
-// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+// AddOrUpdate Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
 // value with the provided.
 func (cs *BoltDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
 	return cs.db.Update(func(tx *bolt.Tx) error {
@@ -55,7 +56,7 @@ func (cs *BoltDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
 	})
 }
 
-// Removes a key from the cluster storage
+// Remove Removes a key from the cluster storage
 func (cs *BoltDBClusterStorage) Remove(key string) error {
 	return cs.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket(cs.bucket)
@@ -64,7 +65,7 @@ func (cs *BoltDBClusterStorage) Remove(key string) error {
 	})
 }
 
-// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+// Each Iterates through all key/values for the storage and calls the handler func. If a handler returns
 // an error, the iteration stops.
 func (cs *BoltDBClusterStorage) Each(handler func(string, []byte) error) error {
 	return cs.db.View(func(tx *bolt.Tx) error {
@@ -87,7 +88,7 @@ func (cs *BoltDBClusterStorage) Each(handler func(string, []byte) error) error {
 	})
 }
 
-// Closes the backing storage
+// Close Closes the backing storage
 func (cs *BoltDBClusterStorage) Close() error {
 	return cs.db.Close()
 }

pkg/clustermanager/clustermanager.go → pkg/services/clusters/clustermanager.go

@@ -1,4 +1,4 @@
-package clustermanager
+package clusters
 
 import (
 	"encoding/base64"
@@ -71,6 +71,7 @@ type ClusterStorage interface {
 	Close() error
 }
 
+// ClusterManager provides an implementation
 type ClusterManager struct {
 	storage ClusterStorage
 	// cache   map[string]*ClusterDefinition
@@ -133,7 +134,7 @@ func NewConfiguredClusterManager(storage ClusterStorage, config string) *Cluster
 	return clusterManager
 }
 
-// Adds, but will not update an existing entry.
+// Add Adds a cluster definition, but will not update an existing entry.
 func (cm *ClusterManager) Add(cluster ClusterDefinition) (*ClusterDefinition, error) {
 	// First time add
 	if cluster.ID == "" {
@@ -153,6 +154,8 @@ func (cm *ClusterManager) Add(cluster ClusterDefinition) (*ClusterDefinition, er
 	return &cluster, nil
 }
 
+// AddOrUpdate will add the cluster definition if it doesn't exist, or update the existing definition
+// if it does exist.
 func (cm *ClusterManager) AddOrUpdate(cluster ClusterDefinition) (*ClusterDefinition, error) {
 	// First time add
 	if cluster.ID == "" {
@@ -172,10 +175,12 @@ func (cm *ClusterManager) AddOrUpdate(cluster ClusterDefinition) (*ClusterDefini
 	return &cluster, nil
 }
 
+// Remove will remove a cluster definition by id.
 func (cm *ClusterManager) Remove(id string) error {
 	return cm.storage.Remove(id)
 }
 
+// GetAll will return all of the cluster definitions
 func (cm *ClusterManager) GetAll() []*ClusterDefinition {
 	clusters := []*ClusterDefinition{}
 
@@ -198,6 +203,7 @@ func (cm *ClusterManager) GetAll() []*ClusterDefinition {
 	return clusters
 }
 
+// Close will close the backing database
 func (cm *ClusterManager) Close() error {
 	return cm.storage.Close()
 }

pkg/clustermanager/clustersendpoints.go → pkg/services/clusters/clustersendpoints.go

@@ -1,4 +1,4 @@
-package clustermanager
+package clusters
 
 import (
 	"errors"
@@ -18,27 +18,37 @@ type DataEnvelope struct {
 	Data   interface{} `json:"data"`
 }
 
-type ClusterManagerEndpoints struct {
+// ClusterManagerHTTPService is an implementation of HTTPService which provides
+// the frontend with the ability to manage stored cluster definitions.
+type ClusterManagerHTTPService struct {
 	manager *ClusterManager
 }
 
-func NewClusterManagerEndpoints(manager *ClusterManager) *ClusterManagerEndpoints {
-	return &ClusterManagerEndpoints{
+// NewClusterManagerHTTPService creates a new cluster management http service
+func NewClusterManagerHTTPService(manager *ClusterManager) *ClusterManagerHTTPService {
+	return &ClusterManagerHTTPService{
 		manager: manager,
 	}
 }
 
-func (cme *ClusterManagerEndpoints) GetAllClusters(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+// Register assigns the endpoints and returns an error on failure.
+func (cme *ClusterManagerHTTPService) Register(router *httprouter.Router) error {
+	router.GET("/clusters", cme.GetAllClusters)
+	router.PUT("/clusters", cme.PutCluster)
+	router.DELETE("/clusters/:id", cme.DeleteCluster)
+
+	return nil
+}
+
+func (cme *ClusterManagerHTTPService) GetAllClusters(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	clusters := cme.manager.GetAll()
 	w.Write(wrapData(clusters, nil))
 }
 
-func (cme *ClusterManagerEndpoints) PutCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+func (cme *ClusterManagerHTTPService) PutCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
@@ -62,9 +72,8 @@ func (cme *ClusterManagerEndpoints) PutCluster(w http.ResponseWriter, r *http.Re
 	w.Write(wrapData(cd, nil))
 }
 
-func (cme *ClusterManagerEndpoints) DeleteCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+func (cme *ClusterManagerHTTPService) DeleteCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	clusterID := ps.ByName("id")
 	if clusterID == "" {

pkg/clustermanager/mapdbstorage.go → pkg/services/clusters/mapdbstorage.go

@@ -1,20 +1,18 @@
-package clustermanager
-
-import (
-	_ "k8s.io/klog"
-)
+package clusters
 
+// MapDBClusterStorage is a map implementation of a database used to store cluster definitions
 type MapDBClusterStorage struct {
 	store map[string][]byte
 }
 
+// NewMapDBClusterStorage creates a new map backed ClusterStorage implementation
 func NewMapDBClusterStorage() ClusterStorage {
 	return &MapDBClusterStorage{
 		store: make(map[string][]byte),
 	}
 }
 
-// Adds the entry if the key does not exist
+// AddIfNotExists Adds the entry if the key does not exist
 func (cs *MapDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
 	if _, ok := cs.store[key]; !ok {
 		cs.store[key] = cluster
@@ -22,20 +20,20 @@ func (cs *MapDBClusterStorage) AddIfNotExists(key string, cluster []byte) error
 	return nil
 }
 
-// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+// AddOrUpdate Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
 // value with the provided.
 func (cs *MapDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
 	cs.store[key] = cluster
 	return nil
 }
 
-// Removes a key from the cluster storage
+// Remove Removes a key from the cluster storage
 func (cs *MapDBClusterStorage) Remove(key string) error {
 	delete(cs.store, key)
 	return nil
 }
 
-// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+// Each Iterates through all key/values for the storage and calls the handler func. If a handler returns
 // an error, the iteration stops.
 func (cs *MapDBClusterStorage) Each(handler func(string, []byte) error) error {
 	for k, v := range cs.store {
@@ -49,7 +47,7 @@ func (cs *MapDBClusterStorage) Each(handler func(string, []byte) error) error {
 	return nil
 }
 
-// Closes the backing storage
+// Close Closes the backing storage
 func (cs *MapDBClusterStorage) Close() error {
 	return nil
 }

pkg/services/clusterservice.go

@@ -0,0 +1,37 @@
+package services
+
+import "github.com/kubecost/cost-model/pkg/services/clusters"
+
+// NewClusterManagerService creates a new HTTPService implementation driving cluster definition management
+// for the frontend
+func NewClusterManagerService() HTTPService {
+	return clusters.NewClusterManagerHTTPService(newClusterManager())
+}
+
+// newClusterManager creates a new cluster manager instance for use in the service
+func newClusterManager() *clusters.ClusterManager {
+	clustersConfigFile := "/var/configs/clusters/default-clusters.yaml"
+
+	// Return a memory-backed cluster manager populated by configmap
+	return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
+
+	// NOTE: The following should be used with a persistent disk store. Since the
+	// NOTE: configmap approach is currently the "persistent" source (entries are read-only
+	// NOTE: on the backend), we don't currently need to store on disk.
+	/*
+		path := env.GetConfigPath()
+		db, err := bolt.Open(path+"costmodel.db", 0600, nil)
+		if err != nil {
+			klog.V(1).Infof("[Error] Failed to create costmodel.db: %s", err.Error())
+			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
+		}
+
+		store, err := clusters.NewBoltDBClusterStorage("clusters", db)
+		if err != nil {
+			klog.V(1).Infof("[Error] Failed to Create Cluster Storage: %s", err.Error())
+			return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
+		}
+
+		return clusters.NewConfiguredClusterManager(store, clustersConfigFile)
+	*/
+}

pkg/services/services.go

@@ -0,0 +1,65 @@
+package services
+
+import (
+	"sync"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/kubecost/cost-model/pkg/log"
+)
+
+// HTTPService defines an implementation prototype for an object capable of registering
+// endpoints on an http router which provide an service relevant to the cost-model.
+type HTTPService interface {
+	// Register assigns the endpoints and returns an error on failure.
+	Register(*httprouter.Router) error
+}
+
+// HTTPServices defines an implementation prototype for an object capable of managing and registering
+// predefined HTTPService routes
+type HTTPServices interface {
+	// Add a HTTPService implementation for registration
+	Add(service HTTPService)
+
+	// RegisterAll registers all the services added with the provided router
+	RegisterAll(*httprouter.Router) error
+}
+
+type defaultHTTPServices struct {
+	sync.Mutex
+	services []HTTPService
+}
+
+// Add a HTTPService implementation for
+func (dhs *defaultHTTPServices) Add(service HTTPService) {
+	if service == nil {
+		log.Warningf("Attempting to Add nil HTTPService")
+		return
+	}
+
+	dhs.Lock()
+	defer dhs.Unlock()
+
+	dhs.services = append(dhs.services, service)
+}
+
+// RegisterAll registers all the services added with the provided router
+func (dhs *defaultHTTPServices) RegisterAll(router *httprouter.Router) error {
+	dhs.Lock()
+	defer dhs.Unlock()
+
+	for _, svc := range dhs.services {
+		svc.Register(router)
+	}
+
+	return nil
+}
+
+// NewCostModelServices creates an HTTPServices implementation containing any predefined
+// http services used with the cost-model
+func NewCostModelServices() HTTPServices {
+	return &defaultHTTPServices{
+		services: []HTTPService{
+			NewClusterManagerService(),
+		},
+	}
+}

pkg/util/cloudutil/aws.go

@@ -1,10 +1,8 @@
 package cloudutil
 
 import (
-	"regexp"
 	"strings"
-
-	"github.com/kubecost/cost-model/pkg/log"
+	"unicode"
 )
 
 // ConvertToGlueColumnFormat takes a string and runs through various regex
@@ -14,27 +12,33 @@ import (
 // https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/run-athena-sql.html
 // It returns a string containing the column name in proper column name format and length.
 func ConvertToGlueColumnFormat(columnName string) string {
-	log.Debugf("Converting string \"%s\" to proper AWS Glue column name.", columnName)
-
-	// An underscore is added in front of uppercase letters
-	capitalUnderscore := regexp.MustCompile(`[A-Z]`)
-	final := capitalUnderscore.ReplaceAllString(columnName, `_$0`)
-
-	// Any non-alphanumeric characters are replaced with an underscore
-	noSpacePunc := regexp.MustCompile(`[\s]{1,}|[^A-Za-z0-9]`)
-	final = noSpacePunc.ReplaceAllString(final, "_")
-
-	// Duplicate underscores are removed
-	noDupUnderscore := regexp.MustCompile(`_{2,}`)
-	final = noDupUnderscore.ReplaceAllString(final, "_")
-
-	// Any leading and trailing underscores are removed
-	noFrontUnderscore := regexp.MustCompile(`(^\_|\_$)`)
-	final = noFrontUnderscore.ReplaceAllString(final, "")
-
-	// Uppercase to lowercase
-	final = strings.ToLower(final)
+	var sb strings.Builder
+	var prev rune
+	for i, r := range columnName {
+		if unicode.IsUpper(r) && prev != '_' && i != 0 {
+			sb.WriteRune('_')
+		}
+		if !unicode.IsLetter(r) && !unicode.IsNumber(r) {
+			if prev != '_' && i != 0 && i != (len(columnName)-1) {
+				sb.WriteRune('_')
+			}
+			prev = '_'
+			continue
+		}
+		if r == '_' {
+			if prev == '_' || i == 0 || i == len(columnName)-1 {
+				prev = '_'
+				continue
+			}
+		}
+		sb.WriteRune(unicode.ToLower(r))
+		prev = r
+	}
 
+	final := sb.String()
+	if prev == '_' { // string any trailing '_'
+		final = final[:len(final)-1]
+	}
 	// Longer column name than expected - remove _ left to right
 	allowedColLen := 128
 	underscoreToRemove := len(final) - allowedColLen
@@ -48,7 +52,5 @@ func ConvertToGlueColumnFormat(columnName string) string {
 		final = final[:allowedColLen]
 	}
 
-	log.Debugf("Column name being returned: \"%s\". Length: \"%d\".", final, len(final))
-
 	return final
 }

pkg/util/json/json.go

@@ -10,5 +10,8 @@ var Marshal = jsoniter.ConfigCompatibleWithStandardLibrary.Marshal
 var Unmarshal = jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal
 
 type Marshaler json.Marshaler
+type Unmarshaler json.Unmarshaler
+
+type RawMessage json.RawMessage
 
 var NewDecoder = json.NewDecoder

pkg/util/watcher/configwatcher_test.go

@@ -0,0 +1,142 @@
+package watcher
+
+import (
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	TestConfigMapName          = "test-config"
+	AlternateTestConfigMapName = "alternate-test-config"
+	TestDataProperty           = "test-prop"
+)
+
+func newTestWatcher(t *testing.T, configMapName string, instanceName string, didRun *bool) *ConfigMapWatcher {
+	return &ConfigMapWatcher{
+		ConfigMapName: configMapName,
+		WatchFunc: func(cmn string, data map[string]string) error {
+			t.Logf("ConfigMapWatcher[%s] triggered for ConfigMap: %s, data[\"test\"] = %s\n", instanceName, cmn, data[TestDataProperty])
+			*didRun = true
+			return nil
+		},
+	}
+}
+
+func newConfigMap(configMapName string, dataValue string) *v1.ConfigMap {
+	return &v1.ConfigMap{
+		Data: map[string]string{
+			TestDataProperty: dataValue,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: configMapName,
+		},
+	}
+}
+
+func TestConfigWatcherSingleHandler(t *testing.T) {
+	// Test that a single watcher added for the configmap test-config is executed when
+	// triggered
+	var didRun bool = false
+
+	w := NewConfigMapWatchers(newTestWatcher(t, TestConfigMapName, "single", &didRun))
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'test-config' configmap
+	f(newConfigMap(TestConfigMapName, "testing 1 2 3"))
+
+	if !didRun {
+		t.Errorf("Failed to run configmap handler for 'single'\n")
+	}
+}
+
+func TestConfigWatcherMultipleHandlers(t *testing.T) {
+	// Test that adding two different configmap watchers aren't both triggered on a configmap update
+	var firstDidRun bool = false
+	var secondDidRun bool = false
+
+	w := NewConfigMapWatchers(
+		newTestWatcher(t, TestConfigMapName, "single", &firstDidRun),
+		newTestWatcher(t, AlternateTestConfigMapName, "alternate", &secondDidRun))
+
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'alternate-test-config' configmap
+	f(newConfigMap(AlternateTestConfigMapName, "oof!"))
+
+	// Assert that first did not run
+	if firstDidRun {
+		t.Errorf("Executed alternate-test-config map change, but test-config handler, 'single' executed!\n")
+	}
+
+	if !secondDidRun {
+		t.Errorf("Failed to run configmap handler for 'alternate'\n")
+	}
+}
+
+func TestConfigWatcherMultipleHandlersForSameConfig(t *testing.T) {
+	// Test that adding two different configmap watchers for the same configmap are both triggered
+	var firstDidRun bool = false
+	var secondDidRun bool = false
+	var thirdDidRun bool = false
+
+	w := NewConfigMapWatchers(
+		newTestWatcher(t, TestConfigMapName, "first", &firstDidRun),
+		newTestWatcher(t, AlternateTestConfigMapName, "alternate", &secondDidRun),
+		// third watcher watches for the same configmap as "first"
+		newTestWatcher(t, TestConfigMapName, "third", &thirdDidRun),
+	)
+
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'test-config' configmap
+	f(newConfigMap(TestConfigMapName, "double trouble"))
+
+	// Assert that second did not run
+	if secondDidRun {
+		t.Errorf("Executed test-config map change, first handler, 'single', executed!\n")
+	}
+
+	if !firstDidRun {
+		t.Errorf("Failed to run configmap handler for 'first'\n")
+	}
+	if !thirdDidRun {
+		t.Errorf("Failed to run configmap handler for 'third'\n")
+	}
+}
+
+func TestConfigMapWatcherWithAdd(t *testing.T) {
+	// Test that adding two different configmap watchers for the same configmap are both triggered
+	// when using Add() and AddWatcher()
+	var firstDidRun bool = false
+	var secondDidRun bool = false
+	var thirdDidRun bool = false
+
+	a, b, c := newTestWatcher(t, TestConfigMapName, "first", &firstDidRun),
+		newTestWatcher(t, AlternateTestConfigMapName, "alternate", &secondDidRun),
+		// third watcher watches for the same configmap as "first"
+		newTestWatcher(t, TestConfigMapName, "third", &thirdDidRun)
+
+	w := NewConfigMapWatchers()
+	w.AddWatcher(a)
+	w.AddWatcher(b)
+	w.Add(c.ConfigMapName, c.WatchFunc)
+
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'test-config' configmap
+	f(newConfigMap(TestConfigMapName, "double trouble"))
+
+	// Assert that second did not run
+	if secondDidRun {
+		t.Errorf("Executed test-config map change, first handler, 'single', executed!\n")
+	}
+
+	if !firstDidRun {
+		t.Errorf("Failed to run configmap handler for 'first'\n")
+	}
+	if !thirdDidRun {
+		t.Errorf("Failed to run configmap handler for 'third'\n")
+	}
+}

pkg/util/watcher/configwatchers.go

@@ -0,0 +1,74 @@
+package watcher
+
+import (
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog"
+)
+
+// ConfigMapWatcher represents a single configmap watcher
+type ConfigMapWatcher struct {
+	ConfigMapName string
+	WatchFunc     func(string, map[string]string) error
+}
+
+type ConfigMapWatchers struct {
+	watchers map[string][]*ConfigMapWatcher
+}
+
+func NewConfigMapWatchers(watchers ...*ConfigMapWatcher) *ConfigMapWatchers {
+	cmw := &ConfigMapWatchers{
+		watchers: make(map[string][]*ConfigMapWatcher),
+	}
+
+	for _, w := range watchers {
+		cmw.AddWatcher(w)
+	}
+
+	return cmw
+}
+
+func (cmw *ConfigMapWatchers) AddWatcher(watcher *ConfigMapWatcher) {
+	if watcher == nil {
+		return
+	}
+
+	name := watcher.ConfigMapName
+	cmw.watchers[name] = append(cmw.watchers[name], watcher)
+}
+
+func (cmw *ConfigMapWatchers) Add(configMapName string, watchFunc func(string, map[string]string) error) {
+	cmw.AddWatcher(&ConfigMapWatcher{
+		ConfigMapName: configMapName,
+		WatchFunc:     watchFunc,
+	})
+}
+
+func (cmw *ConfigMapWatchers) GetWatchedConfigs() []string {
+	configNames := []string{}
+
+	for k := range cmw.watchers {
+		configNames = append(configNames, k)
+	}
+
+	return configNames
+}
+
+func (cmw *ConfigMapWatchers) ToWatchFunc() func(interface{}) {
+	return func(c interface{}) {
+		conf, ok := c.(*v1.ConfigMap)
+		if !ok {
+			return
+		}
+
+		name := conf.GetName()
+		data := conf.Data
+		if watchers, ok := cmw.watchers[name]; ok {
+			for _, cw := range watchers {
+				err := cw.WatchFunc(name, data)
+				if err != nil {
+					klog.Infof("ERROR UPDATING %s CONFIG: %s", name, err.Error())
+				}
+			}
+		}
+	}
+}

ui/package.json

@@ -3,7 +3,8 @@
   "version": "0.0.1",
   "description": "Open source UI for Kubecost",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "preinstall": "npx npm-force-resolutions"
   },
   "author": "",
   "license": "Apache-2.0",
@@ -32,6 +33,10 @@
     "@babel/plugin-proposal-class-properties": "^7.13.0",
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-react": "^7.12.13",
-    "parcel": "*"
+    "parcel": "*",
+    "set-value": "4.0.1"
+  },
+  "resolutions": {
+    "set-value": "4.0.1"
   }
 }