Merge pull request #837 from kubecost/develop

Merge develop into master

pkg/cloud/awsprovider.go

@@ -5,7 +5,6 @@ import (
 	"compress/gzip"
 	"encoding/csv"
 	"fmt"
-	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -16,6 +15,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/endpoints"
+
 	"k8s.io/klog"
 
 	"github.com/kubecost/cost-model/pkg/clustercache"

pkg/cloud/provider.go

@@ -175,6 +175,7 @@ type CustomPricing struct {
 	SharedLabelNames             string            `json:"sharedLabelNames"`
 	SharedLabelValues            string            `json:"sharedLabelValues"`
 	ReadOnly                     string            `json:"readOnly"`
+	KubecostToken                string            `json:"kubecostToken"`
 }
 
 type ServiceAccountStatus struct {

pkg/costmodel/allocation.go

@@ -19,11 +19,11 @@ import (
 const (
 	queryFmtPods              = `avg(kube_pod_container_status_running{}) by (pod, namespace, %s)[%s:%s]%s`
 	queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s, provider_id)`
-	queryFmtRAMRequests       = `avg(avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtRAMRequests       = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
 	queryFmtRAMUsageAvg       = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
 	queryFmtRAMUsageMax       = `max(max_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
 	queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
-	queryFmtCPURequests       = `avg(avg_over_time(kube_pod_container_resource_requests_cpu_cores{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtCPURequests       = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
 	queryFmtCPUUsageAvg       = `avg(rate(container_cpu_usage_seconds_total{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
 
 	// This query could be written without the recording rule
@@ -64,6 +64,20 @@ const (
 	queryFmtLBActiveMins          = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]%s`
 )
 
+// CanCompute should return true if CostModel can act as a valid source for the
+// given time range. In the case of CostModel we want to attempt to compute as
+// long as the range starts in the past. If the CostModel ends up not having
+// data to match, that's okay, and should be communicated with an error
+// response from ComputeAllocation.
+func (cm *CostModel) CanCompute(start, end time.Time) bool {
+	return start.Before(time.Now())
+}
+
+// Name returns the name of the Source
+func (cm *CostModel) Name() string {
+	return "CostModel"
+}
+
 // ComputeAllocation uses the CostModel instance to compute an AllocationSet
 // for the window defined by the given start and end times. The Allocations
 // returned are unaggregated (i.e. down to the container level).
@@ -1678,13 +1692,13 @@ func applyUnmountedPVs(window kubecost.Window, podMap map[podKey]*Pod, pvMap map
 			Cluster: cluster,
 			Name:    kubecost.UnmountedSuffix,
 		}
-		unmountedBreakDown := kubecost.PVAllocations{
+		unmountedPVs := kubecost.PVAllocations{
 			pvKey: {
 				ByteHours: unmountedPVBytes[cluster] * window.Minutes() / 60.0,
 				Cost:      amount,
 			},
 		}
-		podMap[key].Allocations[container].PVs = podMap[key].Allocations[container].PVs.Add(unmountedBreakDown)
+		podMap[key].Allocations[container].PVs = podMap[key].Allocations[container].PVs.Add(unmountedPVs)
 	}
 }
 
@@ -1730,13 +1744,13 @@ func applyUnmountedPVCs(window kubecost.Window, podMap map[podKey]*Pod, pvcMap m
 			Cluster: cluster,
 			Name:    kubecost.UnmountedSuffix,
 		}
-		unmountedBreakDown := kubecost.PVAllocations{
+		unmountedPVs := kubecost.PVAllocations{
 			pvKey: {
 				ByteHours: unmountedPVCBytes[key] * window.Minutes() / 60.0,
 				Cost:      amount,
 			},
 		}
-		podMap[podKey].Allocations[container].PVs = podMap[podKey].Allocations[container].PVs.Add(unmountedBreakDown)
+		podMap[podKey].Allocations[container].PVs = podMap[podKey].Allocations[container].PVs.Add(unmountedPVs)
 
 	}
 }

pkg/costmodel/costmodel.go

@@ -138,9 +138,9 @@ const (
 		label_replace(
 			label_replace(
 				avg(
-					count_over_time(kube_pod_container_resource_requests_memory_bytes{container!="",container!="POD", node!=""}[%s] %s)
+					count_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="",container!="POD", node!=""}[%s] %s)
 					*
-					avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="",container!="POD", node!=""}[%s] %s)
+					avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="",container!="POD", node!=""}[%s] %s)
 				) by (namespace,container,pod,node,%s) , "container_name","$1","container","(.+)"
 			), "pod_name","$1","pod","(.+)"
 		)
@@ -156,9 +156,9 @@ const (
 		label_replace(
 			label_replace(
 				avg(
-					count_over_time(kube_pod_container_resource_requests_cpu_cores{container!="",container!="POD", node!=""}[%s] %s)
+					count_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="",container!="POD", node!=""}[%s] %s)
 					*
-					avg_over_time(kube_pod_container_resource_requests_cpu_cores{container!="",container!="POD", node!=""}[%s] %s)
+					avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="",container!="POD", node!=""}[%s] %s)
 				) by (namespace,container,pod,node,%s) , "container_name","$1","container","(.+)"
 			), "pod_name","$1","pod","(.+)"
 		)
@@ -227,7 +227,7 @@ const (
 	queryZoneNetworkUsage     = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
 	queryRegionNetworkUsage   = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
 	queryInternetNetworkUsage = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
-	normalizationStr          = `max(count_over_time(kube_pod_container_resource_requests_memory_bytes{}[%s] %s))`
+	normalizationStr          = `max(count_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte"}[%s] %s))`
 )
 
 func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyzerCloud.Provider, window string, offset string, filterNamespace string) (map[string]*CostData, error) {
@@ -899,6 +899,7 @@ func GetPVCost(pv *costAnalyzerCloud.PV, kpv *v1.PersistentVolume, cp costAnalyz
 		return err
 	}
 	key := cp.GetPVKey(kpv, pv.Parameters, defaultRegion)
+	pv.ProviderID = key.ID()
 	pvWithCost, err := cp.PVPricing(key)
 	if err != nil {
 		pv.Cost = cfg.Storage
@@ -909,7 +910,6 @@ func GetPVCost(pv *costAnalyzerCloud.PV, kpv *v1.PersistentVolume, cp costAnalyz
 		return nil // set default cost
 	}
 	pv.Cost = pvWithCost.Cost
-	pv.ProviderID = key.ID()
 	return nil
 }
 

pkg/costmodel/metrics.go

@@ -529,6 +529,398 @@ func (cim ClusterInfoMetric) Write(m *dto.Metric) error {
 	return nil
 }
 
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityMemoryBytesCollector
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityMemoryBytesCollector is a prometheus collector that generates
+// KubeNodeStatusCapacityMemoryBytesMetrics
+type KubeNodeStatusCapacityMemoryBytesCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNodeStatusCapacityMemoryBytesCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_node_status_capacity_memory_bytes", "node capacity memory bytes", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNodeStatusCapacityMemoryBytesCollector) Collect(ch chan<- prometheus.Metric) {
+	nodes := nsac.KubeClusterCache.GetAllNodes()
+	for _, node := range nodes {
+		// k8s.io/apimachinery/pkg/api/resource/amount.go and
+		// k8s.io/apimachinery/pkg/api/resource/quantity.go for
+		// details on the "amount" API. See
+		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-types
+		// for the units of memory and CPU.
+		memoryBytes := node.Status.Capacity.Memory().Value()
+
+		m := newKubeNodeStatusCapacityMemoryBytesMetric(node.GetName(), memoryBytes, "kube_node_status_capacity_memory_bytes", nil, nil)
+		ch <- m
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityMemoryBytesMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityMemoryBytesMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_status_capacity_memory_bytes
+type KubeNodeStatusCapacityMemoryBytesMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	bytes       int64
+	node        string
+}
+
+// Creates a new KubeNodeStatusCapacityMemoryBytesMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityMemoryBytesMetric(node string, bytes int64, fqname string, labelNames []string, labelValues []string) KubeNodeStatusCapacityMemoryBytesMetric {
+	return KubeNodeStatusCapacityMemoryBytesMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_node_status_capacity_memory_bytes Node Capacity Memory Bytes",
+		bytes:       bytes,
+		node:        node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusCapacityMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{"node": nam.node}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusCapacityMemoryBytesMetric) Write(m *dto.Metric) error {
+	h := float64(nam.bytes)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	n := "node"
+	labels = append(labels, &dto.LabelPair{
+		Name:  &n,
+		Value: &nam.node,
+	})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityCPUCoresCollector
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityCPUCoresCollector is a prometheus collector that generates
+// KubeNodeStatusCapacityCPUCoresMetrics
+type KubeNodeStatusCapacityCPUCoresCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNodeStatusCapacityCPUCoresCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_node_status_capacity_cpu_cores", "node capacity cpu cores", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNodeStatusCapacityCPUCoresCollector) Collect(ch chan<- prometheus.Metric) {
+	nodes := nsac.KubeClusterCache.GetAllNodes()
+	for _, node := range nodes {
+		// k8s.io/apimachinery/pkg/api/resource/amount.go and
+		// k8s.io/apimachinery/pkg/api/resource/quantity.go for
+		// details on the "amount" API. See
+		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-types
+		// for the units of memory and CPU.
+		cpuCores := float64(node.Status.Capacity.Cpu().MilliValue()) / 1000
+
+		m := newKubeNodeStatusCapacityCPUCoresMetric(node.GetName(), cpuCores, "kube_node_status_capacity_cpu_cores", nil, nil)
+		ch <- m
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityCPUCoresMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityCPUCoresMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_status_capacity_memory_bytes
+type KubeNodeStatusCapacityCPUCoresMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	cores       float64
+	node        string
+}
+
+// Creates a new KubeNodeStatusCapacityCPUCoresMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityCPUCoresMetric(node string, cores float64, fqname string, labelNames []string, labelValues []string) KubeNodeStatusCapacityCPUCoresMetric {
+	return KubeNodeStatusCapacityCPUCoresMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_node_status_capacity_cpu_cores Node Capacity CPU Cores",
+		cores:       cores,
+		node:        node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusCapacityCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{"node": nam.node}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusCapacityCPUCoresMetric) Write(m *dto.Metric) error {
+	h := nam.cores
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	n := "node"
+	labels = append(labels, &dto.LabelPair{
+		Name:  &n,
+		Value: &nam.node,
+	})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodLabelsCollector
+//--------------------------------------------------------------------------
+//
+// We use this to emit kube_pod_labels with all of a pod's labels, regardless
+// of the whitelist setting introduced in KSM v2. See
+// https://github.com/kubernetes/kube-state-metrics/issues/1270#issuecomment-712986441
+
+// KubePodLabelsCollector is a prometheus collector that generates
+// KubePodLabelsMetrics
+type KubePodLabelsCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubePodLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_labels", "all labels for each pod prefixed with label_", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubePodLabelsCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := nsac.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
+
+		m := newKubePodLabelsMetric(
+			pod.GetName(),
+			pod.GetNamespace(),
+			string(pod.GetUID()),
+			"kube_pod_labels",
+			labelNames,
+			labelValues,
+		)
+		ch <- m
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePodLabelsMetric
+//--------------------------------------------------------------------------
+
+// KubePodLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_pod_labels
+type KubePodLabelsMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	pod         string
+	namespace   string
+	uid         string
+}
+
+// Creates a new KubePodLabelsMetric, implementation of prometheus.Metric
+func newKubePodLabelsMetric(pod string, namespace string, uid string, fqname string, labelNames []string, labelValues []string) KubePodLabelsMetric {
+	return KubePodLabelsMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_pod_labels all labels for each pod prefixed with label_",
+		pod:         pod,
+		namespace:   namespace,
+		uid:         uid,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubePodLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"pod":       nam.pod,
+		"namespace": nam.namespace,
+		"uid":       nam.uid,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubePodLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+
+	podString := "pod"
+	namespaceString := "namespace"
+	uidString := "uid"
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  &podString,
+			Value: &nam.pod,
+		},
+		&dto.LabelPair{
+			Name:  &namespaceString,
+			Value: &nam.namespace,
+		}, &dto.LabelPair{
+			Name:  &uidString,
+			Value: &nam.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeLabelsCollector
+//--------------------------------------------------------------------------
+//
+// We use this to emit kube_node_labels with all of a node's labels, regardless
+// of the whitelist setting introduced in KSM v2. See
+// https://github.com/kubernetes/kube-state-metrics/issues/1270#issuecomment-712986441
+
+// KubeNodeLabelsCollector is a prometheus collector that generates
+// KubeNodeLabelsMetrics
+type KubeNodeLabelsCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNodeLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_node_labels", "all labels for each node prefixed with label_", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNodeLabelsCollector) Collect(ch chan<- prometheus.Metric) {
+	nodes := nsac.KubeClusterCache.GetAllNodes()
+	for _, node := range nodes {
+
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(node.GetLabels(), "label_")
+
+		m := newKubeNodeLabelsMetric(
+			node.GetName(),
+			"kube_node_labels",
+			labelNames,
+			labelValues,
+		)
+		ch <- m
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeLabelsMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_labels
+type KubeNodeLabelsMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	node        string
+}
+
+// Creates a new KubeNodeLabelsMetric, implementation of prometheus.Metric
+func newKubeNodeLabelsMetric(node string, fqname string, labelNames []string, labelValues []string) KubeNodeLabelsMetric {
+	return KubeNodeLabelsMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_node_labels all labels for each node prefixed with label_",
+		node:        node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": nam.node,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+
+	nodeString := "node"
+	labels = append(labels, &dto.LabelPair{Name: &nodeString, Value: &nam.node})
+	m.Label = labels
+	return nil
+}
+
 // toStringPtr is used to create a new string pointer from iteration vars
 func toStringPtr(s string) *string {
 	return &s
@@ -675,6 +1067,21 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider cloud
 				KubeClusterCache: clusterCache,
 			})
 		}
+
+		if env.IsEmitKsmV1Metrics() {
+			prometheus.MustRegister(KubeNodeStatusCapacityMemoryBytesCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeNodeStatusCapacityCPUCoresCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePodLabelsCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeNodeLabelsCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
 	})
 }
 

pkg/env/costmodelenv.go

@@ -38,6 +38,8 @@ const (
 	EmitPodAnnotationsMetricEnvVar       = "EMIT_POD_ANNOTATIONS_METRIC"
 	EmitNamespaceAnnotationsMetricEnvVar = "EMIT_NAMESPACE_ANNOTATIONS_METRIC"
 
+	EmitKsmV1MetricsEnvVar = "EMIT_KSM_V1_METRICS"
+
 	ThanosEnabledEnvVar      = "THANOS_ENABLED"
 	ThanosQueryUrlEnvVar     = "THANOS_QUERY_URL"
 	ThanosOffsetEnvVar       = "THANOS_QUERY_OFFSET"
@@ -74,7 +76,7 @@ const (
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAppVersion() string {
-	return Get(AppVersionEnvVar, "1.81.0")
+	return Get(AppVersionEnvVar, "1.82.0")
 }
 
 // IsEmitNamespaceAnnotationsMetric returns true if cost-model is configured to emit the kube_namespace_annotations metric
@@ -89,6 +91,12 @@ func IsEmitPodAnnotationsMetric() bool {
 	return GetBool(EmitPodAnnotationsMetricEnvVar, false)
 }
 
+// IsEmitKsmV1Metrics returns true if cost-model is configured to emit all necessary KSM v1
+// metrics that were removed in KSM v2
+func IsEmitKsmV1Metrics() bool {
+	return GetBool(EmitKsmV1MetricsEnvVar, true)
+}
+
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAWSAccessKeyID() string {

pkg/kubecost/allocation.go

@@ -75,7 +75,6 @@ type Allocation struct {
 	RAMCost                    float64               `json:"ramCost"`
 	RAMCostAdjustment          float64               `json:"ramCostAdjustment"`
 	SharedCost                 float64               `json:"sharedCost"`
-	SharedCostAdjustment       float64               `json:"sharedCostAdjustment"`
 	ExternalCost               float64               `json:"externalCost"`
 	// RawAllocationOnly is a pointer so if it is not present it will be
 	// marshalled as null rather than as an object with Go default values.
@@ -218,7 +217,6 @@ func (a *Allocation) Clone() *Allocation {
 		RAMCost:                    a.RAMCost,
 		RAMCostAdjustment:          a.RAMCostAdjustment,
 		SharedCost:                 a.SharedCost,
-		SharedCostAdjustment:       a.SharedCostAdjustment,
 		ExternalCost:               a.ExternalCost,
 		RawAllocationOnly:          a.RawAllocationOnly.Clone(),
 	}
@@ -305,9 +303,6 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.SharedCost, that.SharedCost) {
 		return false
 	}
-	if !util.IsApproximately(a.SharedCostAdjustment, that.SharedCostAdjustment) {
-		return false
-	}
 	if !util.IsApproximately(a.ExternalCost, that.ExternalCost) {
 		return false
 	}
@@ -380,7 +375,7 @@ func (a *Allocation) LBTotalCost() float64 {
 
 // SharedTotalCost calculates total shared cost of Allocation including adjustment
 func (a *Allocation) SharedTotalCost() float64 {
-	return a.SharedCost + a.SharedCostAdjustment
+	return a.SharedCost
 }
 
 // PVCost calculate cumulative cost of all PVs that Allocation is attached to
@@ -475,6 +470,16 @@ func (a *Allocation) PVBytes() float64 {
 	return a.PVByteHours() / (a.Minutes() / 60.0)
 }
 
+// ResetAdjustments sets all cost adjustment fields to zero
+func (a *Allocation) ResetAdjustments() {
+	a.CPUCostAdjustment = 0.0
+	a.GPUCostAdjustment = 0.0
+	a.RAMCostAdjustment = 0.0
+	a.PVCostAdjustment = 0.0
+	a.NetworkCostAdjustment = 0.0
+	a.LoadBalancerCostAdjustment = 0.0
+}
+
 // MarshalJSON implements json.Marshaler interface
 func (a *Allocation) MarshalJSON() ([]byte, error) {
 	buffer := bytes.NewBufferString("{")
@@ -512,7 +517,6 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "ramCostAdjustment", a.RAMCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "ramEfficiency", a.RAMEfficiency(), ",")
 	jsonEncodeFloat64(buffer, "sharedCost", a.SharedCost, ",")
-	jsonEncodeFloat64(buffer, "sharedCostAdjustment", a.SharedCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "externalCost", a.ExternalCost, ",")
 	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), ",")
 	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), ",")
@@ -648,7 +652,6 @@ func (a *Allocation) add(that *Allocation) {
 	a.PVCostAdjustment += that.PVCostAdjustment
 	a.NetworkCostAdjustment += that.NetworkCostAdjustment
 	a.LoadBalancerCostAdjustment += that.LoadBalancerCostAdjustment
-	a.SharedCostAdjustment += that.SharedCostAdjustment
 
 	// Any data that is in a "raw allocation only" is not valid in any
 	// sort of cumulative Allocation (like one that is added).
@@ -662,6 +665,7 @@ type AllocationSet struct {
 	allocations  map[string]*Allocation
 	externalKeys map[string]bool
 	idleKeys     map[string]bool
+	FromSource   string // stores the name of the source used to compute the data
 	Window       Window
 	Warnings     []string
 	Errors       []string
@@ -928,10 +932,9 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 
 	// (3-5) Filter, distribute idle cost, and aggregate (in that order)
 	for _, alloc := range as.allocations {
-		idleKey, err := alloc.getIdleKey(options)
+		idleId, err := alloc.getIdleId(options)
 		if err != nil {
-			log.DedupedInfof(5,"AllocationSet.AggregateBy: missing idleKey for allocation: %s", alloc.Name)
-			continue
+			log.DedupedWarningf(3,"AllocationSet.AggregateBy: missing idleId for allocation: %s", alloc.Name)
 		}
 
 		skip := false
@@ -949,7 +952,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 			// entry will result in that proportional amount being removed
 			// from the idle allocation at the end of the process.)
 			if idleFiltrationCoefficients != nil {
-				if ifcc, ok := idleFiltrationCoefficients[idleKey]; ok {
+				if ifcc, ok := idleFiltrationCoefficients[idleId]; ok {
 					delete(ifcc, alloc.Name)
 				}
 			}
@@ -962,35 +965,37 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		// all idle allocations will be in the aggSet at this point, so idleSet
 		// will be empty and we won't enter this block.
 		if idleSet.Length() > 0 {
-			// Distribute idle allocations by coefficient per-idleKey, per-allocation
+			// Distribute idle allocations by coefficient per-idleId, per-allocation
 			for _, idleAlloc := range idleSet.allocations {
-				// Only share idle if the idleKey matches; i.e. the allocation
-				// is from the same idleKey as the idle costs
-				iaIdleKey, err := idleAlloc.getIdleKey(options)
+				// Only share idle if the idleId matches; i.e. the allocation
+				// is from the same idleId as the idle costs
+				iaidleId, err := idleAlloc.getIdleId(options)
 				if err != nil {
+					log.Errorf("AllocationSet.AggregateBy: Idle allocation is missing idleId %s", idleAlloc.Name)
 					return err
 				}
-				if iaIdleKey != idleKey {
+
+				if iaidleId != idleId {
 					continue
 				}
 
 				// Make sure idle coefficients exist
-				if _, ok := idleCoefficients[idleKey]; !ok {
-					log.Warningf("AllocationSet.AggregateBy: error getting idle coefficient: no idleKey '%s' for '%s'", idleKey, alloc.Name)
+				if _, ok := idleCoefficients[idleId]; !ok {
+					log.Warningf("AllocationSet.AggregateBy: error getting idle coefficient: no idleId '%s' for '%s'", idleId, alloc.Name)
 					continue
 				}
-				if _, ok := idleCoefficients[idleKey][alloc.Name]; !ok {
+				if _, ok := idleCoefficients[idleId][alloc.Name]; !ok {
 					log.Warningf("AllocationSet.AggregateBy: error getting idle coefficient for '%s'", alloc.Name)
 					continue
 				}
 
-				alloc.CPUCoreHours += idleAlloc.CPUCoreHours * idleCoefficients[idleKey][alloc.Name]["cpu"]
-				alloc.GPUHours += idleAlloc.GPUHours * idleCoefficients[idleKey][alloc.Name]["gpu"]
-				alloc.RAMByteHours += idleAlloc.RAMByteHours * idleCoefficients[idleKey][alloc.Name]["ram"]
+				alloc.CPUCoreHours += idleAlloc.CPUCoreHours * idleCoefficients[idleId][alloc.Name]["cpu"]
+				alloc.GPUHours += idleAlloc.GPUHours * idleCoefficients[idleId][alloc.Name]["gpu"]
+				alloc.RAMByteHours += idleAlloc.RAMByteHours * idleCoefficients[idleId][alloc.Name]["ram"]
 
-				idleCPUCost := idleAlloc.CPUCost * idleCoefficients[idleKey][alloc.Name]["cpu"]
-				idleGPUCost := idleAlloc.GPUCost * idleCoefficients[idleKey][alloc.Name]["gpu"]
-				idleRAMCost := idleAlloc.RAMCost * idleCoefficients[idleKey][alloc.Name]["ram"]
+				idleCPUCost := idleAlloc.CPUCost * idleCoefficients[idleId][alloc.Name]["cpu"]
+				idleGPUCost := idleAlloc.GPUCost * idleCoefficients[idleId][alloc.Name]["gpu"]
+				idleRAMCost := idleAlloc.RAMCost * idleCoefficients[idleId][alloc.Name]["ram"]
 				alloc.CPUCost += idleCPUCost
 				alloc.GPUCost += idleGPUCost
 				alloc.RAMCost += idleRAMCost
@@ -1016,41 +1021,37 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	// before sharing with the aggregated allocations.
 	if idleSet.Length() > 0 && shareSet.Length() > 0 {
 		for _, alloc := range shareSet.allocations {
-			idleKey, err := alloc.getIdleKey(options)
+			idleId, err := alloc.getIdleId(options)
 			if err != nil {
-				log.DedupedWarningf(5, "AllocationSet.AggregateBy: missing idleKey for allocation: %s", alloc.Name)
-				continue
+				log.DedupedWarningf(3, "AllocationSet.AggregateBy: missing idleId for allocation: %s", alloc.Name)
 			}
-
-			// Distribute idle allocations by coefficient per-idleKey, per-allocation
+			// Distribute idle allocations by coefficient per-idleId, per-allocation
 			for _, idleAlloc := range idleSet.allocations {
-				// Only share idle if the idleKey matches; i.e. the allocation
-				// is from the same idleKey as the idle costs
-				iaIdleKey, err := idleAlloc.getIdleKey(options)
-				if err != nil {
-					return nil
-				}
-				if iaIdleKey != idleKey {
+				// Only share idle if the idleId matches; i.e. the allocation
+				// is from the same idleId as the idle costs
+				iaidleId, _ := idleAlloc.getIdleId(options)
+
+				if iaidleId != idleId {
 					continue
 				}
 
 				// Make sure idle coefficients exist
-				if _, ok := idleCoefficients[idleKey]; !ok {
-					log.Warningf("AllocationSet.AggregateBy: error getting idle coefficient: no idleKey '%s' for '%s'", idleKey, alloc.Name)
+				if _, ok := idleCoefficients[idleId]; !ok {
+					log.Warningf("AllocationSet.AggregateBy: error getting idle coefficient: no idleId '%s' for '%s'", idleId, alloc.Name)
 					continue
 				}
-				if _, ok := idleCoefficients[idleKey][alloc.Name]; !ok {
+				if _, ok := idleCoefficients[idleId][alloc.Name]; !ok {
 					log.Warningf("AllocationSet.AggregateBy: error getting idle coefficient for '%s'", alloc.Name)
 					continue
 				}
 
-				alloc.CPUCoreHours += idleAlloc.CPUCoreHours * idleCoefficients[idleKey][alloc.Name]["cpu"]
-				alloc.GPUHours += idleAlloc.GPUHours * idleCoefficients[idleKey][alloc.Name]["gpu"]
-				alloc.RAMByteHours += idleAlloc.RAMByteHours * idleCoefficients[idleKey][alloc.Name]["ram"]
+				alloc.CPUCoreHours += idleAlloc.CPUCoreHours * idleCoefficients[idleId][alloc.Name]["cpu"]
+				alloc.GPUHours += idleAlloc.GPUHours * idleCoefficients[idleId][alloc.Name]["gpu"]
+				alloc.RAMByteHours += idleAlloc.RAMByteHours * idleCoefficients[idleId][alloc.Name]["ram"]
 
-				idleCPUCost := idleAlloc.CPUCost * idleCoefficients[idleKey][alloc.Name]["cpu"]
-				idleGPUCost := idleAlloc.GPUCost * idleCoefficients[idleKey][alloc.Name]["gpu"]
-				idleRAMCost := idleAlloc.RAMCost * idleCoefficients[idleKey][alloc.Name]["ram"]
+				idleCPUCost := idleAlloc.CPUCost * idleCoefficients[idleId][alloc.Name]["cpu"]
+				idleGPUCost := idleAlloc.GPUCost * idleCoefficients[idleId][alloc.Name]["gpu"]
+				idleRAMCost := idleAlloc.RAMCost * idleCoefficients[idleId][alloc.Name]["ram"]
 				alloc.CPUCost += idleCPUCost
 				alloc.GPUCost += idleGPUCost
 				alloc.RAMCost += idleRAMCost
@@ -1067,9 +1068,9 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	if idleFiltrationCoefficients != nil {
 		groupingIdleFiltrationCoeffs = map[string]map[string]float64{}
 
-		for idleKey, m := range idleFiltrationCoefficients {
-			if _, ok := groupingIdleFiltrationCoeffs[idleKey]; !ok {
-				groupingIdleFiltrationCoeffs[idleKey] = map[string]float64{
+		for idleId, m := range idleFiltrationCoefficients {
+			if _, ok := groupingIdleFiltrationCoeffs[idleId]; !ok {
+				groupingIdleFiltrationCoeffs[idleId] = map[string]float64{
 					"cpu": 0.0,
 					"gpu": 0.0,
 					"ram": 0.0,
@@ -1078,7 +1079,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 
 			for _, n := range m {
 				for resource, val := range n {
-					groupingIdleFiltrationCoeffs[idleKey][resource] += val
+					groupingIdleFiltrationCoeffs[idleId][resource] += val
 				}
 			}
 		}
@@ -1089,14 +1090,13 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	if len(aggSet.idleKeys) > 0 && groupingIdleFiltrationCoeffs != nil {
 		for idleKey := range aggSet.idleKeys {
 			idleAlloc := aggSet.Get(idleKey)
-
-			iaIdleKey, err := idleAlloc.getIdleKey(options)
+			iaidleId, err := idleAlloc.getIdleId(options)
 			if err != nil {
-				log.Warningf("AllocationSet.AggregateBy: idle allocation without IdleKey: %s", idleAlloc)
-				continue
+				log.Errorf("AllocationSet.AggregateBy: Idle allocation is missing idleId %s", idleAlloc.Name)
+				return err
 			}
 
-			if resourceCoeffs, ok := groupingIdleFiltrationCoeffs[iaIdleKey]; ok {
+			if resourceCoeffs, ok := groupingIdleFiltrationCoeffs[iaidleId]; ok {
 				idleAlloc.CPUCost *= resourceCoeffs["cpu"]
 				idleAlloc.CPUCoreHours *= resourceCoeffs["cpu"]
 				idleAlloc.RAMCost *= resourceCoeffs["ram"]
@@ -1238,43 +1238,42 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 			continue
 		}
 
-		idleKey, err := alloc.getIdleKey(options)
+		idleId, err := alloc.getIdleId(options)
 		if err != nil {
-			// skip allocations that are missing idleKey
-			continue
+			log.DedupedWarningf(3, "Missing Idle Key for %s", alloc.Name)
 		}
 
 		// get the name key for the allocation
 		name := alloc.Name
 
 		// Create key based tables if they don't exist
-		if _, ok := coeffs[idleKey]; !ok {
-			coeffs[idleKey] = map[string]map[string]float64{}
+		if _, ok := coeffs[idleId]; !ok {
+			coeffs[idleId] = map[string]map[string]float64{}
 		}
-		if _, ok := totals[idleKey]; !ok {
-			totals[idleKey] = map[string]float64{}
+		if _, ok := totals[idleId]; !ok {
+			totals[idleId] = map[string]float64{}
 		}
 
-		if _, ok := coeffs[idleKey][name]; !ok {
-			coeffs[idleKey][name] = map[string]float64{}
+		if _, ok := coeffs[idleId][name]; !ok {
+			coeffs[idleId][name] = map[string]float64{}
 		}
 
 		if shareType == ShareEven {
 			for _, r := range types {
 				// Not additive - hard set to 1.0
-				coeffs[idleKey][name][r] = 1.0
+				coeffs[idleId][name][r] = 1.0
 
 				// totals are additive
-				totals[idleKey][r] += 1.0
+				totals[idleId][r] += 1.0
 			}
 		} else {
-			coeffs[idleKey][name]["cpu"] += alloc.CPUTotalCost()
-			coeffs[idleKey][name]["gpu"] += alloc.GPUTotalCost()
-			coeffs[idleKey][name]["ram"] += alloc.RAMTotalCost()
+			coeffs[idleId][name]["cpu"] += alloc.CPUTotalCost()
+			coeffs[idleId][name]["gpu"] += alloc.GPUTotalCost()
+			coeffs[idleId][name]["ram"] += alloc.RAMTotalCost()
 
-			totals[idleKey]["cpu"] += alloc.CPUTotalCost()
-			totals[idleKey]["gpu"] += alloc.GPUTotalCost()
-			totals[idleKey]["ram"] += alloc.RAMTotalCost()
+			totals[idleId]["cpu"] += alloc.CPUTotalCost()
+			totals[idleId]["gpu"] += alloc.GPUTotalCost()
+			totals[idleId]["ram"] += alloc.RAMTotalCost()
 		}
 	}
 
@@ -1285,43 +1284,43 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 			continue
 		}
 
-		// idleKey will be providerId or cluster
-		idleKey, err := alloc.getIdleKey(options)
+		// idleId will be providerId or cluster
+		idleId, err := alloc.getIdleId(options)
 		if err != nil {
-			return nil, err
+			log.DedupedWarningf(3, "Missing Idle Key in share set for %s", alloc.Name)
 		}
 
 		// get the name key for the allocation
 		name := alloc.Name
 
-		// Create idleKey based tables if they don't exist
-		if _, ok := coeffs[idleKey]; !ok {
-			coeffs[idleKey] = map[string]map[string]float64{}
+		// Create idleId based tables if they don't exist
+		if _, ok := coeffs[idleId]; !ok {
+			coeffs[idleId] = map[string]map[string]float64{}
 		}
-		if _, ok := totals[idleKey]; !ok {
-			totals[idleKey] = map[string]float64{}
+		if _, ok := totals[idleId]; !ok {
+			totals[idleId] = map[string]float64{}
 		}
 
-		if _, ok := coeffs[idleKey][name]; !ok {
-			coeffs[idleKey][name] = map[string]float64{}
+		if _, ok := coeffs[idleId][name]; !ok {
+			coeffs[idleId][name] = map[string]float64{}
 		}
 
 		if shareType == ShareEven {
 			for _, r := range types {
 				// Not additive - hard set to 1.0
-				coeffs[idleKey][name][r] = 1.0
+				coeffs[idleId][name][r] = 1.0
 
 				// totals are additive
-				totals[idleKey][r] += 1.0
+				totals[idleId][r] += 1.0
 			}
 		} else {
-			coeffs[idleKey][name]["cpu"] += alloc.CPUTotalCost()
-			coeffs[idleKey][name]["gpu"] += alloc.GPUTotalCost()
-			coeffs[idleKey][name]["ram"] += alloc.RAMTotalCost()
+			coeffs[idleId][name]["cpu"] += alloc.CPUTotalCost()
+			coeffs[idleId][name]["gpu"] += alloc.GPUTotalCost()
+			coeffs[idleId][name]["ram"] += alloc.RAMTotalCost()
 
-			totals[idleKey]["cpu"] += alloc.CPUTotalCost()
-			totals[idleKey]["gpu"] += alloc.GPUTotalCost()
-			totals[idleKey]["ram"] += alloc.RAMTotalCost()
+			totals[idleId]["cpu"] += alloc.CPUTotalCost()
+			totals[idleId]["gpu"] += alloc.GPUTotalCost()
+			totals[idleId]["ram"] += alloc.RAMTotalCost()
 		}
 	}
 
@@ -1339,24 +1338,24 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 	return coeffs, nil
 }
 
-// getIdleKey returns the providerId or cluster of an Allocation depending on the IdleByNode
+// getIdleId returns the providerId or cluster of an Allocation depending on the IdleByNode
 // option in the AllocationAggregationOptions and an error if the respective field is missing
-func (a *Allocation) getIdleKey(options *AllocationAggregationOptions) (string, error) {
-	var idleKey string
+func (a *Allocation) getIdleId(options *AllocationAggregationOptions) (string, error) {
+	var idleId string
 	if options.IdleByNode {
 		// Key allocations to ProviderId to match against node
-		idleKey = a.Properties.ProviderID
-		if idleKey == "" {
-			return idleKey, fmt.Errorf("ProviderId is not set")
+		idleId = a.Properties.ProviderID
+		if idleId == "" {
+			return idleId, fmt.Errorf("ProviderId is not set")
 		}
 	} else {
 		// key the allocations by cluster id
-		idleKey = a.Properties.Cluster
-		if idleKey == "" {
-			return idleKey, fmt.Errorf("ClusterProp is not set")
+		idleId = a.Properties.Cluster
+		if idleId == "" {
+			return idleId, fmt.Errorf("ClusterProp is not set")
 		}
 	}
-	return idleKey, nil
+	return idleId, nil
 }
 
 func (a *Allocation) generateKey(aggregateBy []string) string {
@@ -1662,6 +1661,7 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 // allocation with idle reflects the adjusted node costs. One idle allocation
 // per-node will be computed and returned, keyed by cluster_id.
 func (as *AllocationSet) ComputeIdleAllocationsByNode(assetSet *AssetSet) (map[string]*Allocation, error) {
+
 	if as == nil {
 		return nil, fmt.Errorf("cannot compute idle allocation for nil AllocationSet")
 	}
@@ -1683,6 +1683,7 @@ func (as *AllocationSet) ComputeIdleAllocationsByNode(assetSet *AssetSet) (map[s
 	assetSet.Each(func(key string, a Asset) {
 		if node, ok := a.(*Node); ok {
 			if _, ok := assetNodeResourceCosts[node.Properties().ProviderID]; ok || node.Properties().ProviderID == "" {
+				log.DedupedWarningf(5, "Compute Idle Allocations By Node: Node missing providerId: %s", node.properties.Name)
 				return
 			}
 
@@ -1729,6 +1730,7 @@ func (as *AllocationSet) ComputeIdleAllocationsByNode(assetSet *AssetSet) (map[s
 		providerId := a.Properties.ProviderID
 		if providerId == "" {
 			// Failed to find allocation's node
+			log.DedupedWarningf(5, "Compute Idle Allocations By Node: Allocation missing providerId: %s", a.Name)
 			return
 		}
 
@@ -1793,137 +1795,6 @@ func (as *AllocationSet) ComputeIdleAllocationsByNode(assetSet *AssetSet) (map[s
 	return idleAllocs, nil
 }
 
-// Reconcile calculate the exact cost of Allocation by resource(cpu, ram, gpu etc) based on Asset(s) on which
-// the Allocation depends.
-func (as *AllocationSet) Reconcile(assetSet *AssetSet) error {
-	if as == nil {
-		return fmt.Errorf("cannot reconcile allocation for nil AllocationSet")
-	}
-
-	if assetSet == nil {
-		return fmt.Errorf("cannot reconcile allocation with nil AssetSet")
-	}
-
-	if !as.Window.Equal(assetSet.Window) {
-		return fmt.Errorf("cannot reconcile allocation for sets with mismatched windows: %s != %s", as.Window, assetSet.Window)
-	}
-
-	// Build map of Assets with type Node by their ProviderId so that they can be matched to Allocations to determine
-	// proper CPU GPU and RAM prices
-	nodeByProviderID := map[string]*Node{}
-	diskByName := map[string]*Disk{}
-	assetSet.Each(func(key string, a Asset) {
-		if node, ok := a.(*Node); ok && node.properties.ProviderID != "" {
-			nodeByProviderID[node.properties.ProviderID] = node
-		}
-		if disk, ok := a.(*Disk); ok {
-			diskByName[disk.properties.Name] = disk
-		}
-	})
-
-	// Match Assets against allocations and adjust allocation cost based on the proportion of the asset that they used
-	as.Each(func(name string, a *Allocation) {
-		a.reconcileNodes(nodeByProviderID)
-		a.reconcileDisks(diskByName)
-	})
-
-	return nil
-}
-
-func (a *Allocation) reconcileNodes(nodeByProviderID map[string]*Node) {
-	providerId := a.Properties.ProviderID
-
-	// Reconcile with node Assets
-	node, ok := nodeByProviderID[providerId]
-	if !ok {
-		// Failed to find node for allocation
-		return
-	}
-
-	// adjustmentRate is used to scale resource costs proportionally
-	// by the adjustment. This is necessary because we only get one
-	// adjustment per Node, not one per-resource-per-Node.
-	//
-	// e.g. total cost = $90, adjustment = -$10 => 0.9
-	// e.g. total cost = $150, adjustment = -$300 => 0.3333
-	// e.g. total cost = $150, adjustment = $50 => 1.5
-	adjustmentRate := 1.0
-	if node.TotalCost()-node.Adjustment() == 0 {
-		// If (totalCost - adjustment) is 0.0 then adjustment cancels
-		// the entire node cost and we should make everything 0
-		// without dividing by 0.
-		adjustmentRate = 0.0
-	} else if node.Adjustment() != 0.0 {
-		// adjustmentRate is the ratio of cost-with-adjustment (i.e. TotalCost)
-		// to cost-without-adjustment (i.e. TotalCost - Adjustment).
-		adjustmentRate = node.TotalCost() / (node.TotalCost() - node.Adjustment())
-	}
-
-	// Find total cost of each node resource for the window
-	cpuCost := node.CPUCost * (1.0 - node.Discount) * adjustmentRate
-	ramCost := node.RAMCost * (1.0 - node.Discount) * adjustmentRate
-	gpuCost := node.GPUCost * adjustmentRate
-
-	// Find the proportion of resource hours used by the allocation, checking for 0 denominators
-	cpuUsageProportion := 0.0
-	if node.CPUCoreHours != 0 {
-		cpuUsageProportion = a.CPUCoreHours / node.CPUCoreHours
-	} else {
-		log.Warningf("Missing CPU Hours for node Provider ID: %s", providerId)
-	}
-	ramUsageProportion := 0.0
-	if node.RAMByteHours != 0 {
-		ramUsageProportion = a.RAMByteHours / node.RAMByteHours
-	} else {
-		log.Warningf("Missing Ram Byte Hours for node Provider ID: %s", providerId)
-	}
-	gpuUsageProportion := 0.0
-	if node.GPUHours != 0 {
-		gpuUsageProportion = a.GPUHours / node.GPUHours
-	}
-	// No log for GPU because not all nodes have GPU
-
-	// Calculate the allocation's resource costs by the proportion of resources used and total costs
-	allocCPUCost := cpuUsageProportion * cpuCost
-	allocRAMCost := ramUsageProportion * ramCost
-	allocGPUCost := gpuUsageProportion * gpuCost
-
-	a.CPUCostAdjustment = allocCPUCost - a.CPUCost
-	a.RAMCostAdjustment = allocRAMCost - a.RAMCost
-	a.GPUCostAdjustment = allocGPUCost - a.GPUCost
-}
-
-func (a *Allocation) reconcileDisks(diskByName map[string]*Disk) {
-	pvs := a.PVs
-	if pvs == nil {
-		// No PV usage to reconcile
-		return
-	}
-	// Set PV Adjustment for allocation to 0 for idempotency
-	a.PVCostAdjustment = 0.0
-	for pvKey, pvUsage := range pvs {
-		disk, ok := diskByName[pvKey.Name]
-		if !ok {
-			// Failed to find disk in assets
-			continue
-		}
-		// Check the proportion of disk that is being used by
-		pvUsageProportion := 0.0
-		if disk.ByteHours != 0 {
-			pvUsageProportion = pvUsage.ByteHours / disk.ByteHours
-		} else {
-			log.Warningf("Missing Byte Hours for disk: %s", pvKey)
-		}
-
-		// take proportion of disk adjusted cost
-		allocPVCost := pvUsageProportion * disk.TotalCost()
-
-		// PVCostAdjustment is cumulative as there can be many PVs for each Allocation
-		a.PVCostAdjustment += allocPVCost - pvUsage.Cost
-	}
-
-}
-
 // Delete removes the allocation with the given name from the set
 func (as *AllocationSet) Delete(name string) {
 	if as == nil {
@@ -1951,11 +1822,11 @@ func (as *AllocationSet) Each(f func(string, *Allocation)) {
 // End returns the End time of the AllocationSet window
 func (as *AllocationSet) End() time.Time {
 	if as == nil {
-		log.Warningf("Allocation ETL: calling End on nil AllocationSet")
+		log.Warningf("AllocationSet: calling End on nil AllocationSet")
 		return time.Unix(0, 0)
 	}
 	if as.Window.End() == nil {
-		log.Warningf("Allocation ETL: AllocationSet with illegal window: End is nil; len(as.allocations)=%d", len(as.allocations))
+		log.Warningf("AllocationSet: AllocationSet with illegal window: End is nil; len(as.allocations)=%d", len(as.allocations))
 		return time.Unix(0, 0)
 	}
 	return *as.Window.End()
@@ -2156,11 +2027,11 @@ func (as *AllocationSet) Set(alloc *Allocation) error {
 // Start returns the Start time of the AllocationSet window
 func (as *AllocationSet) Start() time.Time {
 	if as == nil {
-		log.Warningf("Allocation ETL: calling Start on nil AllocationSet")
+		log.Warningf("AllocationSet: calling Start on nil AllocationSet")
 		return time.Unix(0, 0)
 	}
 	if as.Window.Start() == nil {
-		log.Warningf("Allocation ETL: AllocationSet with illegal window: Start is nil; len(as.allocations)=%d", len(as.allocations))
+		log.Warningf("AllocationSet: AllocationSet with illegal window: Start is nil; len(as.allocations)=%d", len(as.allocations))
 		return time.Unix(0, 0)
 	}
 	return *as.Window.Start()
@@ -2199,11 +2070,11 @@ func (as *AllocationSet) UTCOffset() time.Duration {
 
 func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error) {
 	if as.IsEmpty() {
-		return that, nil
+		return that.Clone(), nil
 	}
 
 	if that.IsEmpty() {
-		return as, nil
+		return as.Clone(), nil
 	}
 
 	// Set start, end to min(start), max(end)
@@ -2248,6 +2119,7 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 type AllocationSetRange struct {
 	sync.RWMutex
 	allocations []*AllocationSet
+	FromStore   string // stores the name of the store used to retrieve the data
 }
 
 // NewAllocationSetRange instantiates a new range composed of the given
@@ -2277,7 +2149,7 @@ func (asr *AllocationSetRange) Accumulate() (*AllocationSet, error) {
 	return allocSet, nil
 }
 
-// TODO niko/etl accumulate into lower-resolution chunks of the given resolution
+// TODO accumulate into lower-resolution chunks of the given resolution
 // func (asr *AllocationSetRange) AccumulateBy(resolution time.Duration) *AllocationSetRange
 
 // AggregateBy aggregates each AllocationSet in the range by the given

pkg/kubecost/allocation_test.go

@@ -10,82 +10,6 @@ import (
 	"github.com/kubecost/cost-model/pkg/util"
 )
 
-const day = 24 * time.Hour
-
-var disk = PVKey{}
-var disk1 = PVKey{
-	Cluster: "cluster2",
-	Name:    "disk1",
-}
-var disk2 = PVKey{
-	Cluster: "cluster2",
-	Name:    "disk2",
-}
-
-func NewUnitAllocation(name string, start time.Time, resolution time.Duration, props *AllocationProperties) *Allocation {
-	if name == "" {
-		name = "cluster1/namespace1/pod1/container1"
-	}
-
-	properties := &AllocationProperties{}
-	if props == nil {
-		properties.Cluster = "cluster1"
-		properties.Node = "node1"
-		properties.Namespace = "namespace1"
-		properties.ControllerKind = "deployment"
-		properties.Controller = "deployment1"
-		properties.Pod = "pod1"
-		properties.Container = "container1"
-	} else {
-		properties = props
-	}
-
-	end := start.Add(resolution)
-
-	alloc := &Allocation{
-		Name:                  name,
-		Properties:            properties,
-		Window:                NewWindow(&start, &end).Clone(),
-		Start:                 start,
-		End:                   end,
-		CPUCoreHours:          1,
-		CPUCost:               1,
-		CPUCoreRequestAverage: 1,
-		CPUCoreUsageAverage:   1,
-		GPUHours:              1,
-		GPUCost:               1,
-		NetworkCost:           1,
-		LoadBalancerCost:      1,
-		PVs: PVAllocations{
-			disk: {
-				ByteHours: 1,
-				Cost:      1,
-			},
-		},
-		RAMByteHours:           1,
-		RAMCost:                1,
-		RAMBytesRequestAverage: 1,
-		RAMBytesUsageAverage:   1,
-		RawAllocationOnly: &RawAllocationOnlyData{
-			CPUCoreUsageMax:  1,
-			RAMBytesUsageMax: 1,
-		},
-	}
-
-	// If idle allocation, remove non-idle costs, but maintain total cost
-	if alloc.IsIdle() {
-		alloc.PVs = nil
-		alloc.NetworkCost = 0.0
-		alloc.LoadBalancerCost = 0.0
-		alloc.CPUCoreHours += 1.0
-		alloc.CPUCost += 1.0
-		alloc.RAMByteHours += 1.0
-		alloc.RAMCost += 1.0
-	}
-
-	return alloc
-}
-
 func TestAllocation_Add(t *testing.T) {
 	var nilAlloc *Allocation
 	zeroAlloc := &Allocation{}
@@ -559,390 +483,6 @@ func TestNewAllocationSet(t *testing.T) {
 	// TODO niko/etl
 }
 
-func generateAllocationSetClusterIdle(start time.Time) *AllocationSet {
-	// Cluster Idle allocations
-	a1i := NewUnitAllocation(fmt.Sprintf("cluster1/%s", IdleSuffix), start, day, &AllocationProperties{
-		Cluster: "cluster1",
-	})
-	a1i.CPUCost = 5.0
-	a1i.RAMCost = 15.0
-	a1i.GPUCost = 0.0
-
-	a2i := NewUnitAllocation(fmt.Sprintf("cluster2/%s", IdleSuffix), start, day, &AllocationProperties{
-		Cluster: "cluster2",
-	})
-	a2i.CPUCost = 5.0
-	a2i.RAMCost = 5.0
-	a2i.GPUCost = 0.0
-
-	as := generateAllocationSet(start)
-	as.Insert(a1i)
-	as.Insert(a2i)
-	return as
-}
-
-func generateAllocationSetNodeIdle(start time.Time) *AllocationSet {
-	// Node Idle allocations
-	a11i := NewUnitAllocation(fmt.Sprintf("c1nodes/%s", IdleSuffix), start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Node:       "c1nodes",
-		ProviderID: "c1nodes",
-	})
-	a11i.CPUCost = 5.0
-	a11i.RAMCost = 15.0
-	a11i.GPUCost = 0.0
-
-	a21i := NewUnitAllocation(fmt.Sprintf("node1/%s", IdleSuffix), start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Node:       "node1",
-		ProviderID: "node1",
-	})
-	a21i.CPUCost = 1.666667
-	a21i.RAMCost = 1.666667
-	a21i.GPUCost = 0.0
-
-	a22i := NewUnitAllocation(fmt.Sprintf("node2/%s", IdleSuffix), start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Node:       "node2",
-		ProviderID: "node2",
-	})
-	a22i.CPUCost = 1.666667
-	a22i.RAMCost = 1.666667
-	a22i.GPUCost = 0.0
-
-	a23i := NewUnitAllocation(fmt.Sprintf("node3/%s", IdleSuffix), start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Node:       "node3",
-		ProviderID: "node3",
-		Namespace: "",
-	})
-	a23i.CPUCost = 1.666667
-	a23i.RAMCost = 1.666667
-	a23i.GPUCost = 0.0
-
-	as := generateAllocationSet(start)
-	as.Insert(a11i)
-	as.Insert(a21i)
-	as.Insert(a22i)
-	as.Insert(a23i)
-	return as
-}
-
-func generateAllocationSet(start time.Time) *AllocationSet {
-
-	// Active allocations
-	a1111 := NewUnitAllocation("cluster1/namespace1/pod1/container1", start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Namespace:  "namespace1",
-		Pod:        "pod1",
-		Container:  "container1",
-		ProviderID: "c1nodes",
-	})
-	a1111.RAMCost = 11.00
-
-	a11abc2 := NewUnitAllocation("cluster1/namespace1/pod-abc/container2", start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Namespace:  "namespace1",
-		Pod:        "pod-abc",
-		Container:  "container2",
-		ProviderID: "c1nodes",
-	})
-
-	a11def3 := NewUnitAllocation("cluster1/namespace1/pod-def/container3", start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Namespace:  "namespace1",
-		Pod:        "pod-def",
-		Container:  "container3",
-		ProviderID: "c1nodes",
-	})
-
-	a12ghi4 := NewUnitAllocation("cluster1/namespace2/pod-ghi/container4", start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Namespace:  "namespace2",
-		Pod:        "pod-ghi",
-		Container:  "container4",
-		ProviderID: "c1nodes",
-	})
-
-	a12ghi5 := NewUnitAllocation("cluster1/namespace2/pod-ghi/container5", start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Namespace:  "namespace2",
-		Pod:        "pod-ghi",
-		Container:  "container5",
-		ProviderID: "c1nodes",
-	})
-
-	a12jkl6 := NewUnitAllocation("cluster1/namespace2/pod-jkl/container6", start, day, &AllocationProperties{
-		Cluster:    "cluster1",
-		Namespace:  "namespace2",
-		Pod:        "pod-jkl",
-		Container:  "container6",
-		ProviderID: "c1nodes",
-	})
-
-	a22mno4 := NewUnitAllocation("cluster2/namespace2/pod-mno/container4", start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Namespace:  "namespace2",
-		Pod:        "pod-mno",
-		Container:  "container4",
-		ProviderID: "node1",
-	})
-
-	a22mno5 := NewUnitAllocation("cluster2/namespace2/pod-mno/container5", start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Namespace:  "namespace2",
-		Pod:        "pod-mno",
-		Container:  "container5",
-		ProviderID: "node1",
-	})
-
-	a22pqr6 := NewUnitAllocation("cluster2/namespace2/pod-pqr/container6", start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Namespace:  "namespace2",
-		Pod:        "pod-pqr",
-		Container:  "container6",
-		ProviderID: "node2",
-	})
-
-	a23stu7 := NewUnitAllocation("cluster2/namespace3/pod-stu/container7", start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Namespace:  "namespace3",
-		Pod:        "pod-stu",
-		Container:  "container7",
-		ProviderID: "node2",
-	})
-
-	a23vwx8 := NewUnitAllocation("cluster2/namespace3/pod-vwx/container8", start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Namespace:  "namespace3",
-		Pod:        "pod-vwx",
-		Container:  "container8",
-		ProviderID: "node3",
-	})
-
-	a23vwx9 := NewUnitAllocation("cluster2/namespace3/pod-vwx/container9", start, day, &AllocationProperties{
-		Cluster:    "cluster2",
-		Namespace:  "namespace3",
-		Pod:        "pod-vwx",
-		Container:  "container9",
-		ProviderID: "node3",
-	})
-
-	// Controllers
-
-	a11abc2.Properties.ControllerKind = "deployment"
-	a11abc2.Properties.Controller = "deployment1"
-	a11def3.Properties.ControllerKind = "deployment"
-	a11def3.Properties.Controller = "deployment1"
-
-	a12ghi4.Properties.ControllerKind = "deployment"
-	a12ghi4.Properties.Controller = "deployment2"
-	a12ghi5.Properties.ControllerKind = "deployment"
-	a12ghi5.Properties.Controller = "deployment2"
-	a22mno4.Properties.ControllerKind = "deployment"
-	a22mno4.Properties.Controller = "deployment2"
-	a22mno5.Properties.ControllerKind = "deployment"
-	a22mno5.Properties.Controller = "deployment2"
-
-	a23stu7.Properties.ControllerKind = "deployment"
-	a23stu7.Properties.Controller = "deployment3"
-
-	a12jkl6.Properties.ControllerKind = "daemonset"
-	a12jkl6.Properties.Controller = "daemonset1"
-	a22pqr6.Properties.ControllerKind = "daemonset"
-	a22pqr6.Properties.Controller = "daemonset1"
-
-	a23vwx8.Properties.ControllerKind = "statefulset"
-	a23vwx8.Properties.Controller = "statefulset1"
-	a23vwx9.Properties.ControllerKind = "statefulset"
-	a23vwx9.Properties.Controller = "statefulset1"
-
-	// Labels
-
-	a1111.Properties.Labels = map[string]string{"app": "app1", "env": "env1"}
-	a12ghi4.Properties.Labels = map[string]string{"app": "app2", "env": "env2"}
-	a12ghi5.Properties.Labels = map[string]string{"app": "app2", "env": "env2"}
-	a22mno4.Properties.Labels = map[string]string{"app": "app2"}
-	a22mno5.Properties.Labels = map[string]string{"app": "app2"}
-
-	//Annotations
-	a23stu7.Properties.Annotations = map[string]string{"team": "team1"}
-	a23vwx8.Properties.Annotations = map[string]string{"team": "team2"}
-	a23vwx9.Properties.Annotations = map[string]string{"team": "team1"}
-
-	// Services
-	a12jkl6.Properties.Services = []string{"service1"}
-	a22pqr6.Properties.Services = []string{"service1"}
-
-	return NewAllocationSet(start, start.Add(day),
-		// cluster 1, namespace1
-		a1111, a11abc2, a11def3,
-		// cluster 1, namespace 2
-		a12ghi4, a12ghi5, a12jkl6,
-		// cluster 2, namespace 2
-		a22mno4, a22mno5, a22pqr6,
-		// cluster 2, namespace 3
-		a23stu7, a23vwx8, a23vwx9,
-	)
-}
-
-func generateAssetSets(start, end time.Time) []*AssetSet {
-	var assetSets []*AssetSet
-
-	// Create an AssetSet representing cluster costs for two clusters (cluster1
-	// and cluster2). Include Nodes and Disks for both, even though only
-	// Nodes will be counted. Whereas in practice, Assets should be aggregated
-	// by type, here we will provide multiple Nodes for one of the clusters to
-	// make sure the function still holds.
-
-	// NOTE: we're re-using generateAllocationSet so this has to line up with
-	// the allocated node costs from that function. See table above.
-
-	// | Hierarchy                               | Cost |  CPU |  RAM |  GPU | Adjustment |
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1:
-	//     nodes                                  100.00  55.00  44.00  11.00      -10.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1 subtotal (adjusted)             100.00  50.00  40.00  10.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1 allocated                        48.00   6.00  16.00   6.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1 idle                             72.00  44.00  24.00   4.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2:
-	//     node1                                   35.00  20.00  15.00   0.00        0.00
-	//     node2                                   35.00  20.00  15.00   0.00        0.00
-	//     node3                                   30.00  10.00  10.00  10.00        0.00
-	//     (disks should not matter for idle)
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2 subtotal                        100.00  50.00  40.00  10.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2 allocated                        28.00   6.00   6.00   6.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2 idle                             82.00  44.00  34.00   4.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-
-	cluster1Nodes := NewNode("c1nodes", "cluster1", "c1nodes", start, end, NewWindow(&start, &end))
-	cluster1Nodes.CPUCost = 55.0
-	cluster1Nodes.RAMCost = 44.0
-	cluster1Nodes.GPUCost = 11.0
-	cluster1Nodes.adjustment = -10.00
-	cluster1Nodes.CPUCoreHours = 8
-	cluster1Nodes.RAMByteHours = 6
-	cluster1Nodes.GPUHours = 24
-
-	cluster2Node1 := NewNode("node1", "cluster2", "node1", start, end, NewWindow(&start, &end))
-	cluster2Node1.CPUCost = 20.0
-	cluster2Node1.RAMCost = 15.0
-	cluster2Node1.GPUCost = 0.0
-	cluster2Node1.CPUCoreHours = 4
-	cluster2Node1.RAMByteHours = 3
-	cluster2Node1.GPUHours = 0
-
-	cluster2Node2 := NewNode("node2", "cluster2", "node2", start, end, NewWindow(&start, &end))
-	cluster2Node2.CPUCost = 20.0
-	cluster2Node2.RAMCost = 15.0
-	cluster2Node2.GPUCost = 0.0
-	cluster2Node2.CPUCoreHours = 3
-	cluster2Node2.RAMByteHours = 2
-	cluster2Node2.GPUHours = 0
-
-	cluster2Node3 := NewNode("node3", "cluster2", "node3", start, end, NewWindow(&start, &end))
-	cluster2Node3.CPUCost = 10.0
-	cluster2Node3.RAMCost = 10.0
-	cluster2Node3.GPUCost = 10.0
-	cluster2Node3.CPUCoreHours = 2
-	cluster2Node3.RAMByteHours = 2
-	cluster2Node3.GPUHours = 24
-
-	cluster2Disk1 := NewDisk("disk1", "cluster2", "disk1", start, end, NewWindow(&start, &end))
-	cluster2Disk1.Cost = 5.0
-	cluster2Disk1.adjustment = 1.0
-	cluster2Disk1.ByteHours = 5 * gb
-
-	cluster2Disk2 := NewDisk("disk2", "cluster2", "disk2", start, end, NewWindow(&start, &end))
-	cluster2Disk2.Cost = 10.0
-	cluster2Disk2.adjustment = 3.0
-	cluster2Disk2.ByteHours = 10 * gb
-
-	assetSet1 := NewAssetSet(start, end, cluster1Nodes, cluster2Node1, cluster2Node2, cluster2Node3, cluster2Disk1, cluster2Disk2)
-	assetSets = append(assetSets, assetSet1)
-
-	// NOTE: we're re-using generateAllocationSet so this has to line up with
-	// the allocated node costs from that function. See table above.
-
-	// | Hierarchy                               | Cost |  CPU |  RAM |  GPU | Adjustment |
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1:
-	//     nodes                                  100.00   5.00   4.00   1.00       90.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1 subtotal (adjusted)             100.00  50.00  40.00  10.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1 allocated                        48.00   6.00  16.00   6.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster1 idle                             72.00  44.00  24.00   4.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2:
-	//     node1                                   35.00  20.00  15.00   0.00        0.00
-	//     node2                                   35.00  20.00  15.00   0.00        0.00
-	//     node3                                   30.00  10.00  10.00  10.00        0.00
-	//     (disks should not matter for idle)
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2 subtotal                        100.00  50.00  40.00  10.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2 allocated                        28.00   6.00   6.00   6.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-	//   cluster2 idle                             82.00  44.00  34.00   4.00        0.00
-	// +-----------------------------------------+------+------+------+------+------------+
-
-	cluster1Nodes = NewNode("", "cluster1", "c1nodes", start, end, NewWindow(&start, &end))
-	cluster1Nodes.CPUCost = 5.0
-	cluster1Nodes.RAMCost = 4.0
-	cluster1Nodes.GPUCost = 1.0
-	cluster1Nodes.adjustment = 90.00
-	cluster1Nodes.CPUCoreHours = 8
-	cluster1Nodes.RAMByteHours = 6
-	cluster1Nodes.GPUHours = 24
-
-	cluster2Node1 = NewNode("node1", "cluster2", "node1", start, end, NewWindow(&start, &end))
-	cluster2Node1.CPUCost = 20.0
-	cluster2Node1.RAMCost = 15.0
-	cluster2Node1.GPUCost = 0.0
-	cluster2Node1.CPUCoreHours = 4
-	cluster2Node1.RAMByteHours = 3
-	cluster2Node1.GPUHours = 0
-
-	cluster2Node2 = NewNode("node2", "cluster2", "node2", start, end, NewWindow(&start, &end))
-	cluster2Node2.CPUCost = 20.0
-	cluster2Node2.RAMCost = 15.0
-	cluster2Node2.GPUCost = 0.0
-	cluster2Node2.CPUCoreHours = 3
-	cluster2Node2.RAMByteHours = 2
-	cluster2Node2.GPUHours = 0
-
-	cluster2Node3 = NewNode("node3", "cluster2", "node3", start, end, NewWindow(&start, &end))
-	cluster2Node3.CPUCost = 10.0
-	cluster2Node3.RAMCost = 10.0
-	cluster2Node3.GPUCost = 10.0
-	cluster2Node3.CPUCoreHours = 2
-	cluster2Node3.RAMByteHours = 2
-	cluster2Node3.GPUHours = 24
-
-	cluster2Disk1 = NewDisk("disk1", "cluster2", "disk1", start, end, NewWindow(&start, &end))
-	cluster2Disk1.Cost = 5.0
-	cluster2Disk1.adjustment = 1.0
-	cluster2Disk1.ByteHours = 5 * gb
-
-	cluster2Disk2 = NewDisk("disk2", "cluster2", "disk2", start, end, NewWindow(&start, &end))
-	cluster2Disk2.Cost = 12.0
-	cluster2Disk2.adjustment = 4.0
-	cluster2Disk2.ByteHours = 20 * gb
-
-	assetSet2 := NewAssetSet(start, end, cluster1Nodes, cluster2Node1, cluster2Node2, cluster2Node3, cluster2Disk1, cluster2Disk2)
-	assetSets = append(assetSets, assetSet2)
-	return assetSets
-}
-
 func assertAllocationSetTotals(t *testing.T, as *AllocationSet, msg string, err error, length int, totalCost float64) {
 	if err != nil {
 		t.Fatalf("AllocationSet.AggregateBy[%s]: unexpected error: %s", msg, err)
@@ -990,7 +530,7 @@ func printAllocationSet(msg string, as *AllocationSet) {
 
 func TestAllocationSet_AggregateBy(t *testing.T) {
 	// Test AggregateBy against the following workload topology, which is
-	// generated by generateAllocationSet:
+	// generated by GenerateMockAllocationSet:
 
 	// | Hierarchy                              | Cost |  CPU |  RAM |  GPU |   PV |  Net |  LB  |
 	// +----------------------------------------+------+------+------+------+------+------+------+
@@ -1992,7 +1532,6 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			windowEnd:   endYesterday,
 			expMinutes:  1440.0,
 		},
-
 		// 7  Edge cases and errors
 
 		// 7a Empty AggregationProperties
@@ -2004,9 +1543,9 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 	for name, testcase := range cases {
 		t.Run(name, func(t *testing.T) {
 			if testcase.aggOpts != nil && testcase.aggOpts.IdleByNode {
-				as = generateAllocationSetNodeIdle(testcase.start)
+				as = GenerateMockAllocationSetNodeIdle(testcase.start)
 			} else {
-				as = generateAllocationSetClusterIdle(testcase.start)
+				as = GenerateMockAllocationSetClusterIdle(testcase.start)
 			}
 			err = as.AggregateBy(testcase.aggBy, testcase.aggOpts)
 			assertAllocationSetTotals(t, as, name, err, testcase.numResults, testcase.totalCost)
@@ -2028,9 +1567,9 @@ func TestAllocationSet_ComputeIdleAllocations(t *testing.T) {
 	start := end.Add(-day)
 
 	// Generate AllocationSet without idle allocations
-	as = generateAllocationSet(start)
+	as = GenerateMockAllocationSet(start)
 
-	assetSets := generateAssetSets(start, end)
+	assetSets := GenerateMockAssetSets(start, end)
 
 	cases := map[string]struct {
 		allocationSet *AllocationSet
@@ -2105,6 +1644,7 @@ func TestAllocationSet_ComputeIdleAllocations(t *testing.T) {
 }
 
 func TestAllocationSet_ComputeIdleAllocationsPerNode(t *testing.T) {
+
 	var as *AllocationSet
 	var err error
 	var idles map[string]*Allocation
@@ -2113,9 +1653,9 @@ func TestAllocationSet_ComputeIdleAllocationsPerNode(t *testing.T) {
 	start := end.Add(-day)
 
 	// Generate AllocationSet without idle allocations
-	as = generateAllocationSet(start)
+	as = GenerateMockAllocationSet(start)
 
-	assetSets := generateAssetSets(start, end)
+	assetSets := GenerateMockAssetSets(start, end)
 
 	cases := map[string]struct {
 		allocationSet *AllocationSet
@@ -2209,259 +1749,6 @@ func TestAllocationSet_ComputeIdleAllocationsPerNode(t *testing.T) {
 	}
 }
 
-func TestAllocationSet_ReconcileAllocations(t *testing.T) {
-	var as *AllocationSet
-	var err error
-
-	end := time.Now().UTC().Truncate(day)
-	start := end.Add(-day)
-
-	// Generate AllocationSet without idle allocations
-	as = generateAllocationSet(start)
-
-	// add reconcilable pvs to pod-mno
-	for _, a := range as.allocations {
-		if a.Properties.Pod == "pod-mno" {
-			a.PVs = a.PVs.Add(PVAllocations{
-				disk1: {
-					Cost:      2.5,
-					ByteHours: 2.5 * gb,
-				},
-				disk2: {
-					Cost:      5,
-					ByteHours: 5 * gb,
-				},
-			})
-		}
-	}
-
-	assetSets := generateAssetSets(start, end)
-
-	cases := map[string]struct {
-		allocationSet *AllocationSet
-		assetSet      *AssetSet
-		allocations   map[string]Allocation
-	}{
-		"1a": {
-			allocationSet: as,
-			assetSet:      assetSets[0],
-			allocations: map[string]Allocation{
-				// Allocation adjustments are found with the formula:
-				// ADJUSTMENT_RATE * NODE_COST * (ALLOC_HOURS / NODE_HOURS) - ALLOC_COST
-				// ADJUSTMENT_RATE: 0.90909090909
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    55	    |	  8	     |     1 	  |	  1
-				// RAM	|    44	    |	  6	     |     11 	  |	  1
-				// GPU	|    11	    |	 24      |     1 	  |	  1
-				"cluster1/namespace1/pod1/container1": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: -4.333333,
-					GPUCostAdjustment: -0.583333,
-				},
-				// ADJUSTMENT_RATE: 0.90909090909
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    55	    |	  8	     |     1 	  |	  1
-				// RAM	|    44	    |	  6	     |     1 	  |	  1
-				// GPU	|    11	    |	 24      |     1 	  |	  1
-				"cluster1/namespace1/pod-abc/container2": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace1/pod-def/container3": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace2/pod-ghi/container4": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace2/pod-ghi/container5": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace2/pod-jkl/container6": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				// ADJUSTMENT_RATE: 1.0
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    20	    |	  4	     |     1 	  |	  1
-				// RAM	|    15	    |	  3	     |     1 	  |	  1
-				// GPU	|    0	    |	  0      |     1 	  |	  1
-				"cluster2/namespace2/pod-mno/container4": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -1.0,
-					PVCostAdjustment:  2.0,
-				},
-				"cluster2/namespace2/pod-mno/container5": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -1.0,
-					PVCostAdjustment:  2.0,
-				},
-				// ADJUSTMENT_RATE: 1.0
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    20	    |	  3	     |     1 	  |	  1
-				// RAM	|    15	    |	  2	     |     1 	  |	  1
-				// GPU	|    0	    |	  0      |     1 	  |	  1
-				"cluster2/namespace2/pod-pqr/container6": {
-					CPUCostAdjustment: 5.666667,
-					RAMCostAdjustment: 6.5,
-					GPUCostAdjustment: -1.0,
-				},
-				"cluster2/namespace3/pod-stu/container7": {
-					CPUCostAdjustment: 5.666667,
-					RAMCostAdjustment: 6.5,
-					GPUCostAdjustment: -1.0,
-				},
-				// ADJUSTMENT_RATE: 1.0
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    10	    |	  2	     |     1 	  |	  1
-				// RAM	|    10	    |	  2	     |     1 	  |	  1
-				// GPU	|    10	    |	 24      |     1 	  |	  1
-				"cluster2/namespace3/pod-vwx/container8": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster2/namespace3/pod-vwx/container9": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -0.583333,
-				},
-			},
-		},
-		"1b": {
-			allocationSet: as,
-			assetSet:      assetSets[1],
-			allocations: map[string]Allocation{
-				// ADJUSTMENT_RATE: 10
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|     5	    |	  8	     |     1 	  |	  1
-				// RAM	|     4	    |	  6	     |    11 	  |	  1
-				// GPU	|     1	    |	 24      |     1 	  |	  1
-				"cluster1/namespace1/pod1/container1": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: -4.333333,
-					GPUCostAdjustment: -0.583333,
-				},
-				// ADJUSTMENT_RATE: 10
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|     5	    |	  8	     |     1 	  |	  1
-				// RAM	|     4	    |	  6	     |     1 	  |	  1
-				// GPU	|     1	    |	 24      |     1 	  |	  1
-				"cluster1/namespace1/pod-abc/container2": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.6666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace1/pod-def/container3": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.6666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace2/pod-ghi/container4": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.6666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace2/pod-ghi/container5": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.6666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster1/namespace2/pod-jkl/container6": {
-					CPUCostAdjustment: 5.25,
-					RAMCostAdjustment: 5.6666667,
-					GPUCostAdjustment: -0.583333,
-				},
-				// ADJUSTMENT_RATE: 1.0
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    20	    |	  4	     |     1 	  |	  1
-				// RAM	|    15	    |	  3	     |     1 	  |	  1
-				// GPU	|    0	    |	  0      |     1 	  |	  1
-				"cluster2/namespace2/pod-mno/container4": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -1.0,
-					PVCostAdjustment:  -0.5,
-				},
-				"cluster2/namespace2/pod-mno/container5": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -1.0,
-					PVCostAdjustment:  -0.5,
-				},
-				// ADJUSTMENT_RATE: 1.0
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    20	    |	  3	     |     1 	  |	  1
-				// RAM	|    15	    |	  2	     |     1 	  |	  1
-				// GPU	|    0	    |	  0      |     1 	  |	  1
-				"cluster2/namespace2/pod-pqr/container6": {
-					CPUCostAdjustment: 5.666667,
-					RAMCostAdjustment: 6.5,
-					GPUCostAdjustment: -1.0,
-				},
-				"cluster2/namespace3/pod-stu/container7": {
-					CPUCostAdjustment: 5.666667,
-					RAMCostAdjustment: 6.5,
-					GPUCostAdjustment: -1.0,
-				},
-				// ADJUSTMENT_RATE: 1.0
-				// Type | NODE_COST | NODE_HOURs | ALLOC_COST | ALLOC_HOURS
-				// CPU	|    10	    |	  2	     |     1 	  |	  1
-				// RAM	|    10	    |	  2	     |     1 	  |	  1
-				// GPU	|    10	    |	 24      |     1 	  |	  1
-				"cluster2/namespace3/pod-vwx/container8": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -0.583333,
-				},
-				"cluster2/namespace3/pod-vwx/container9": {
-					CPUCostAdjustment: 4.0,
-					RAMCostAdjustment: 4.0,
-					GPUCostAdjustment: -0.583333,
-				},
-			},
-		},
-	}
-
-	for name, testcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			err = as.Reconcile(testcase.assetSet)
-			reconAllocs := as.allocations
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-
-			for allocationName, testAlloc := range testcase.allocations {
-				if _, ok := reconAllocs[allocationName]; !ok {
-					t.Fatalf("expected allocation %s", allocationName)
-				}
-
-				if !util.IsApproximately(reconAllocs[allocationName].CPUCostAdjustment, testAlloc.CPUCostAdjustment) {
-					t.Fatalf("expected CPU Adjustment for %s to be %f; got %f", allocationName, testAlloc.CPUCostAdjustment, reconAllocs[allocationName].CPUCostAdjustment)
-				}
-				if !util.IsApproximately(reconAllocs[allocationName].RAMCostAdjustment, testAlloc.RAMCostAdjustment) {
-					t.Fatalf("expected RAM Adjustment for %s to be %f; got %f", allocationName, testAlloc.RAMCostAdjustment, reconAllocs[allocationName].RAMCostAdjustment)
-				}
-				if !util.IsApproximately(reconAllocs[allocationName].GPUCostAdjustment, testAlloc.GPUCostAdjustment) {
-					t.Fatalf("expected GPU Adjustment for %s to be %f; got %f", allocationName, testAlloc.GPUCostAdjustment, reconAllocs[allocationName].GPUCostAdjustment)
-				}
-				if !util.IsApproximately(reconAllocs[allocationName].PVCostAdjustment, testAlloc.PVCostAdjustment) {
-					t.Fatalf("expected PV Adjustment for %s to be %f; got %f", allocationName, testAlloc.PVCostAdjustment, reconAllocs[allocationName].PVCostAdjustment)
-				}
-			}
-		})
-	}
-}
-
 // TODO niko/etl
 //func TestAllocationSet_Delete(t *testing.T) {}
 
@@ -2544,10 +1831,10 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	}
 
 	todayAS := NewAllocationSet(today, tomorrow)
-	todayAS.Set(NewUnitAllocation("", today, day, nil))
+	todayAS.Set(NewMockUnitAllocation("", today, day, nil))
 
 	yesterdayAS := NewAllocationSet(yesterday, today)
-	yesterdayAS.Set(NewUnitAllocation("", yesterday, day, nil))
+	yesterdayAS.Set(NewMockUnitAllocation("", yesterday, day, nil))
 
 	// Accumulate non-nil with nil should result in copy of non-nil, regardless of order
 	result, err = NewAllocationSetRange(nil, todayAS).Accumulate()
@@ -2677,22 +1964,22 @@ func TestAllocationSetRange_InsertRange(t *testing.T) {
 	today := time.Now().UTC().Truncate(day)
 	tomorrow := time.Now().UTC().Truncate(day).Add(day)
 
-	unit := NewUnitAllocation("", today, day, nil)
+	unit := NewMockUnitAllocation("", today, day, nil)
 
 	ago2dAS := NewAllocationSet(ago2d, yesterday)
-	ago2dAS.Set(NewUnitAllocation("a", ago2d, day, nil))
-	ago2dAS.Set(NewUnitAllocation("b", ago2d, day, nil))
-	ago2dAS.Set(NewUnitAllocation("c", ago2d, day, nil))
+	ago2dAS.Set(NewMockUnitAllocation("a", ago2d, day, nil))
+	ago2dAS.Set(NewMockUnitAllocation("b", ago2d, day, nil))
+	ago2dAS.Set(NewMockUnitAllocation("c", ago2d, day, nil))
 
 	yesterdayAS := NewAllocationSet(yesterday, today)
-	yesterdayAS.Set(NewUnitAllocation("a", yesterday, day, nil))
-	yesterdayAS.Set(NewUnitAllocation("b", yesterday, day, nil))
-	yesterdayAS.Set(NewUnitAllocation("c", yesterday, day, nil))
+	yesterdayAS.Set(NewMockUnitAllocation("a", yesterday, day, nil))
+	yesterdayAS.Set(NewMockUnitAllocation("b", yesterday, day, nil))
+	yesterdayAS.Set(NewMockUnitAllocation("c", yesterday, day, nil))
 
 	todayAS := NewAllocationSet(today, tomorrow)
-	todayAS.Set(NewUnitAllocation("a", today, day, nil))
-	todayAS.Set(NewUnitAllocation("b", today, day, nil))
-	todayAS.Set(NewUnitAllocation("c", today, day, nil))
+	todayAS.Set(NewMockUnitAllocation("a", today, day, nil))
+	todayAS.Set(NewMockUnitAllocation("b", today, day, nil))
+	todayAS.Set(NewMockUnitAllocation("c", today, day, nil))
 
 	var nilASR *AllocationSetRange
 	thisASR := NewAllocationSetRange(yesterdayAS.Clone(), todayAS.Clone())

pkg/kubecost/allocationprops_test.go

@@ -1,61 +0,0 @@
-package kubecost
-
-// TODO niko/etl
-// func TestParseProperty(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperty_String(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_Clone(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_Intersection(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetCluster(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetCluster(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetContainer(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetContainer(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetController(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetController(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetControllerKind(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetControllerKind(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetLabels(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetLabels(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetNamespace(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetNamespace(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetPod(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetPod(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_GetServices(t *testing.T) {}
-
-// TODO niko/etl
-// func TestProperties_SetServices(t *testing.T) {}

pkg/kubecost/asset.go

@@ -12,8 +12,6 @@ import (
 	"github.com/kubecost/cost-model/pkg/util/json"
 )
 
-const timeFmt = "2006-01-02T15:04:05-0700"
-
 // UndefinedKey is used in composing Asset group keys if the group does not have that property defined.
 // E.g. if aggregating on Cluster, Assets in the AssetSet where Asset has no cluster will be grouped under key "__undefined__"
 const UndefinedKey = "__undefined__"
@@ -285,10 +283,10 @@ func key(a Asset, aggregateBy []string) (string, error) {
 				}
 			} else {
 				// Don't allow aggregating on label ""
-				return "", fmt.Errorf("Attempted to aggregate on invalid key: %s", s)
+				return "", fmt.Errorf("attempted to aggregate on invalid key: %s", s)
 			}
 		default:
-			return "", fmt.Errorf("Attempted to aggregate on invalid key: %s", s)
+			return "", fmt.Errorf("attempted to aggregate on invalid key: %s", s)
 		}
 
 		if key != "" {
@@ -596,9 +594,9 @@ func (a *Any) MarshalJSON() ([]byte, error) {
 	buffer := bytes.NewBufferString("{")
 	jsonEncode(buffer, "properties", a.Properties(), ",")
 	jsonEncode(buffer, "labels", a.Labels(), ",")
-	jsonEncodeString(buffer, "window", a.Window().String(), ",")
-	jsonEncodeString(buffer, "start", a.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", a.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", a.Window(), ",")
+	jsonEncodeString(buffer, "start", a.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", a.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", a.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "adjustment", a.Adjustment(), ",")
 	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), "")
@@ -839,9 +837,9 @@ func (ca *Cloud) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "type", ca.Type().String(), ",")
 	jsonEncode(buffer, "properties", ca.Properties(), ",")
 	jsonEncode(buffer, "labels", ca.Labels(), ",")
-	jsonEncodeString(buffer, "window", ca.Window().String(), ",")
-	jsonEncodeString(buffer, "start", ca.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", ca.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", ca.Window(), ",")
+	jsonEncodeString(buffer, "start", ca.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", ca.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", ca.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "adjustment", ca.Adjustment(), ",")
 	jsonEncodeFloat64(buffer, "credit", ca.Credit, ",")
@@ -1039,9 +1037,9 @@ func (cm *ClusterManagement) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "type", cm.Type().String(), ",")
 	jsonEncode(buffer, "properties", cm.Properties(), ",")
 	jsonEncode(buffer, "labels", cm.Labels(), ",")
-	jsonEncodeString(buffer, "window", cm.Window().String(), ",")
-	jsonEncodeString(buffer, "start", cm.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", cm.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", cm.Window(), ",")
+	jsonEncodeString(buffer, "start", cm.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", cm.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", cm.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "totalCost", cm.TotalCost(), "")
 	buffer.WriteString("}")
@@ -1320,9 +1318,9 @@ func (d *Disk) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "type", d.Type().String(), ",")
 	jsonEncode(buffer, "properties", d.Properties(), ",")
 	jsonEncode(buffer, "labels", d.Labels(), ",")
-	jsonEncodeString(buffer, "window", d.Window().String(), ",")
-	jsonEncodeString(buffer, "start", d.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", d.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", d.Window(), ",")
+	jsonEncodeString(buffer, "start", d.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", d.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", d.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "byteHours", d.ByteHours, ",")
 	jsonEncodeFloat64(buffer, "bytes", d.Bytes(), ",")
@@ -1637,9 +1635,9 @@ func (n *Network) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "type", n.Type().String(), ",")
 	jsonEncode(buffer, "properties", n.Properties(), ",")
 	jsonEncode(buffer, "labels", n.Labels(), ",")
-	jsonEncodeString(buffer, "window", n.Window().String(), ",")
-	jsonEncodeString(buffer, "start", n.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", n.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", n.Window(), ",")
+	jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
 	jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
@@ -1997,9 +1995,9 @@ func (n *Node) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "type", n.Type().String(), ",")
 	jsonEncode(buffer, "properties", n.Properties(), ",")
 	jsonEncode(buffer, "labels", n.Labels(), ",")
-	jsonEncodeString(buffer, "window", n.Window().String(), ",")
-	jsonEncodeString(buffer, "start", n.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", n.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", n.Window(), ",")
+	jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
 	jsonEncodeString(buffer, "nodeType", n.NodeType, ",")
 	jsonEncodeFloat64(buffer, "cpuCores", n.CPUCores(), ",")
@@ -2303,9 +2301,9 @@ func (lb *LoadBalancer) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "type", lb.Type().String(), ",")
 	jsonEncode(buffer, "properties", lb.Properties(), ",")
 	jsonEncode(buffer, "labels", lb.Labels(), ",")
-	jsonEncodeString(buffer, "window", lb.Window().String(), ",")
-	jsonEncodeString(buffer, "start", lb.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", lb.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", lb.Window(), ",")
+	jsonEncodeString(buffer, "start", lb.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", lb.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", lb.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "adjustment", lb.Adjustment(), ",")
 	jsonEncodeFloat64(buffer, "totalCost", lb.TotalCost(), "")
@@ -2514,9 +2512,9 @@ func (sa *SharedAsset) MarshalJSON() ([]byte, error) {
 	jsonEncode(buffer, "labels", sa.Labels(), ",")
 	jsonEncode(buffer, "properties", sa.Properties(), ",")
 	jsonEncode(buffer, "labels", sa.Labels(), ",")
-	jsonEncodeString(buffer, "window", sa.Window().String(), ",")
-	jsonEncodeString(buffer, "start", sa.Start().Format(timeFmt), ",")
-	jsonEncodeString(buffer, "end", sa.End().Format(timeFmt), ",")
+	jsonEncode(buffer, "window", sa.Window(), ",")
+	jsonEncodeString(buffer, "start", sa.Start().Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", sa.End().Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", sa.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "totalCost", sa.TotalCost(), "")
 	buffer.WriteString("}")
@@ -2534,6 +2532,7 @@ type AssetSet struct {
 	sync.RWMutex
 	aggregateBy []string
 	assets      map[string]Asset
+	FromSource  string // stores the name of the source used to compute the data
 	Window      Window
 	Warnings    []string
 	Errors      []string
@@ -2575,7 +2574,7 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 	// Compute hours of the given AssetSet, and if it ends in the future,
 	// adjust the hours accordingly
 	hours := as.Window.Minutes() / 60.0
-	diff := time.Now().Sub(as.End())
+	diff := time.Since(as.End())
 	if diff < 0.0 {
 		hours += diff.Hours()
 	}
@@ -2628,10 +2627,7 @@ func (as *AssetSet) Clone() *AssetSet {
 
 	var aggregateBy []string
 	if as.aggregateBy != nil {
-		aggregateBy := []string{}
-		for _, s := range as.aggregateBy {
-			aggregateBy = append(aggregateBy, s)
-		}
+		aggregateBy = append([]string{}, as.aggregateBy...)
 	}
 
 	assets := map[string]Asset{}
@@ -2929,11 +2925,18 @@ func (as *AssetSet) accumulate(that *AssetSet) (*AssetSet, error) {
 	return acc, nil
 }
 
+// AssetSetRange is a thread-safe slice of AssetSets. It is meant to
+// be used such that the AssetSets held are consecutive and coherent with
+// respect to using the same aggregation properties, UTC offset, and
+// resolution. However these rules are not necessarily enforced, so use wisely.
 type AssetSetRange struct {
 	sync.RWMutex
-	assets []*AssetSet
+	assets    []*AssetSet
+	FromStore string // stores the name of the store used to retrieve the data
 }
 
+// NewAssetSetRange instantiates a new range composed of the given
+// AssetSets in the order provided.
 func NewAssetSetRange(assets ...*AssetSet) *AssetSetRange {
 	return &AssetSetRange{
 		assets: assets,
@@ -3023,7 +3026,7 @@ func (asr *AssetSetRange) Length() int {
 
 func (asr *AssetSetRange) MarshalJSON() ([]byte, error) {
 	asr.RLock()
-	asr.RUnlock()
+	defer asr.RUnlock()
 	return json.Marshal(asr.assets)
 }
 

pkg/kubecost/asset_test.go

@@ -21,127 +21,6 @@ var windows = []Window{
 	NewWindow(&start3, &start4),
 }
 
-const gb = 1024 * 1024 * 1024
-
-// generateAssetSet generates the following topology:
-//
-// | Asset                        | Cost |  Adj |
-// +------------------------------+------+------+
-//   cluster1:
-//     node1:                        6.00   1.00
-//     node2:                        4.00   1.50
-//     node3:                        7.00  -0.50
-//     disk1:                        2.50   0.00
-//     disk2:                        1.50   0.00
-//     clusterManagement1:           3.00   0.00
-// +------------------------------+------+------+
-//   cluster1 subtotal              24.00   2.00
-// +------------------------------+------+------+
-//   cluster2:
-//     node4:                       12.00  -1.00
-//     disk3:                        2.50   0.00
-//     disk4:                        1.50   0.00
-//     clusterManagement2:           0.00   0.00
-// +------------------------------+------+------+
-//   cluster2 subtotal              16.00  -1.00
-// +------------------------------+------+------+
-//   cluster3:
-//     node5:                       17.00   2.00
-// +------------------------------+------+------+
-//   cluster3 subtotal              17.00   2.00
-// +------------------------------+------+------+
-//   total                          57.00   3.00
-// +------------------------------+------+------+
-func generateAssetSet(start time.Time) *AssetSet {
-	end := start.Add(day)
-	window := NewWindow(&start, &end)
-
-	hours := window.Duration().Hours()
-
-	node1 := NewNode("node1", "cluster1", "gcp-node1", *window.Clone().start, *window.Clone().end, window.Clone())
-	node1.CPUCost = 4.0
-	node1.RAMCost = 4.0
-	node1.GPUCost = 2.0
-	node1.Discount = 0.5
-	node1.CPUCoreHours = 2.0 * hours
-	node1.RAMByteHours = 4.0 * gb * hours
-	node1.GPUHours = 1.0 * hours
-	node1.SetAdjustment(1.0)
-	node1.SetLabels(map[string]string{"test": "test"})
-
-	node2 := NewNode("node2", "cluster1", "gcp-node2", *window.Clone().start, *window.Clone().end, window.Clone())
-	node2.CPUCost = 4.0
-	node2.RAMCost = 4.0
-	node2.GPUCost = 0.0
-	node2.Discount = 0.5
-	node2.CPUCoreHours = 2.0 * hours
-	node2.RAMByteHours = 4.0 * gb * hours
-	node2.GPUHours = 0.0 * hours
-	node2.SetAdjustment(1.5)
-
-	node3 := NewNode("node3", "cluster1", "gcp-node3", *window.Clone().start, *window.Clone().end, window.Clone())
-	node3.CPUCost = 4.0
-	node3.RAMCost = 4.0
-	node3.GPUCost = 3.0
-	node3.Discount = 0.5
-	node3.CPUCoreHours = 2.0 * hours
-	node3.RAMByteHours = 4.0 * gb * hours
-	node3.GPUHours = 2.0 * hours
-	node3.SetAdjustment(-0.5)
-
-	node4 := NewNode("node4", "cluster2", "gcp-node4", *window.Clone().start, *window.Clone().end, window.Clone())
-	node4.CPUCost = 10.0
-	node4.RAMCost = 6.0
-	node4.GPUCost = 0.0
-	node4.Discount = 0.25
-	node4.CPUCoreHours = 4.0 * hours
-	node4.RAMByteHours = 12.0 * gb * hours
-	node4.GPUHours = 0.0 * hours
-	node4.SetAdjustment(-1.0)
-
-	node5 := NewNode("node5", "cluster3", "aws-node5", *window.Clone().start, *window.Clone().end, window.Clone())
-	node5.CPUCost = 10.0
-	node5.RAMCost = 7.0
-	node5.GPUCost = 0.0
-	node5.Discount = 0.0
-	node5.CPUCoreHours = 8.0 * hours
-	node5.RAMByteHours = 24.0 * gb * hours
-	node5.GPUHours = 0.0 * hours
-	node5.SetAdjustment(2.0)
-
-	disk1 := NewDisk("disk1", "cluster1", "gcp-disk1", *window.Clone().start, *window.Clone().end, window.Clone())
-	disk1.Cost = 2.5
-	disk1.ByteHours = 100 * gb * hours
-
-	disk2 := NewDisk("disk2", "cluster1", "gcp-disk2", *window.Clone().start, *window.Clone().end, window.Clone())
-	disk2.Cost = 1.5
-	disk2.ByteHours = 60 * gb * hours
-
-	disk3 := NewDisk("disk3", "cluster2", "gcp-disk3", *window.Clone().start, *window.Clone().end, window.Clone())
-	disk3.Cost = 2.5
-	disk3.ByteHours = 100 * gb * hours
-
-	disk4 := NewDisk("disk4", "cluster2", "gcp-disk4", *window.Clone().start, *window.Clone().end, window.Clone())
-	disk4.Cost = 1.5
-	disk4.ByteHours = 100 * gb * hours
-
-	cm1 := NewClusterManagement("gcp", "cluster1", window.Clone())
-	cm1.Cost = 3.0
-
-	cm2 := NewClusterManagement("gcp", "cluster2", window.Clone())
-	cm2.Cost = 0.0
-
-	return NewAssetSet(
-		start, end,
-		// cluster 1
-		node1, node2, node3, disk1, disk2, cm1,
-		// cluster 2
-		node4, disk3, disk4, cm2,
-		// cluster 3
-		node5,
-	)
-}
-
 func assertAssetSet(t *testing.T, as *AssetSet, msg string, window Window, exps map[string]float64, err error) {
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy[%s]: unexpected error: %s", msg, err)
@@ -794,7 +673,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	// 1  Single-aggregation
 
 	// 1a []AssetProperty=[Cluster]
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy([]string{string(AssetClusterProp)}, nil)
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy: unexpected error: %s", err)
@@ -806,7 +685,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	}, nil)
 
 	// 1b []AssetProperty=[Type]
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy([]string{string(AssetTypeProp)}, nil)
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy: unexpected error: %s", err)
@@ -818,7 +697,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	}, nil)
 
 	// 1c []AssetProperty=[Nil]
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy([]string{}, nil)
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy: unexpected error: %s", err)
@@ -828,7 +707,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	}, nil)
 
 	// 1d []AssetProperty=nil
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy(nil, nil)
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy: unexpected error: %s", err)
@@ -848,7 +727,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	}, nil)
 
 	// 1e aggregateBy []string=["label:test"]
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy([]string{"label:test"}, nil)
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy: unexpected error: %s", err)
@@ -861,7 +740,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	// 2  Multi-aggregation
 
 	// 2a []AssetProperty=[Cluster,Type]
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy([]string{string(AssetClusterProp), string(AssetTypeProp)}, nil)
 	if err != nil {
 		t.Fatalf("AssetSet.AggregateBy: unexpected error: %s", err)
@@ -879,7 +758,7 @@ func TestAssetSet_AggregateBy(t *testing.T) {
 	// 3  Share resources
 
 	// 3a Shared hourly cost > 0.0
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	err = as.AggregateBy([]string{string(AssetTypeProp)}, &AssetAggregationOptions{
 		SharedHourlyCosts: map[string]float64{"shared1": 0.5},
 	})
@@ -905,7 +784,7 @@ func TestAssetSet_FindMatch(t *testing.T) {
 	var err error
 
 	// Assert success of a simple match of Type and ProviderID
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	query = NewNode("", "", "gcp-node3", s, e, w)
 	match, err = as.FindMatch(query, []string{string(AssetTypeProp), string(AssetProviderIDProp)})
 	if err != nil {
@@ -913,7 +792,7 @@ func TestAssetSet_FindMatch(t *testing.T) {
 	}
 
 	// Assert error of a simple non-match of Type and ProviderID
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	query = NewNode("", "", "aws-node3", s, e, w)
 	match, err = as.FindMatch(query, []string{string(AssetTypeProp), string(AssetProviderIDProp)})
 	if err == nil {
@@ -921,7 +800,7 @@ func TestAssetSet_FindMatch(t *testing.T) {
 	}
 
 	// Assert error of matching ProviderID, but not Type
-	as = generateAssetSet(startYesterday)
+	as = GenerateMockAssetSet(startYesterday)
 	query = NewCloud(ComputeCategory, "gcp-node3", s, e, w)
 	match, err = as.FindMatch(query, []string{string(AssetTypeProp), string(AssetProviderIDProp)})
 	if err == nil {
@@ -944,9 +823,9 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 	var err error
 
 	asr = NewAssetSetRange(
-		generateAssetSet(startD0),
-		generateAssetSet(startD1),
-		generateAssetSet(startD2),
+		GenerateMockAssetSet(startD0),
+		GenerateMockAssetSet(startD1),
+		GenerateMockAssetSet(startD2),
 	)
 	err = asr.AggregateBy(nil, nil)
 	as, err = asr.Accumulate()
@@ -968,9 +847,9 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 	}, nil)
 
 	asr = NewAssetSetRange(
-		generateAssetSet(startD0),
-		generateAssetSet(startD1),
-		generateAssetSet(startD2),
+		GenerateMockAssetSet(startD0),
+		GenerateMockAssetSet(startD1),
+		GenerateMockAssetSet(startD2),
 	)
 	err = asr.AggregateBy([]string{}, nil)
 	as, err = asr.Accumulate()
@@ -982,9 +861,9 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 	}, nil)
 
 	asr = NewAssetSetRange(
-		generateAssetSet(startD0),
-		generateAssetSet(startD1),
-		generateAssetSet(startD2),
+		GenerateMockAssetSet(startD0),
+		GenerateMockAssetSet(startD1),
+		GenerateMockAssetSet(startD2),
 	)
 	err = asr.AggregateBy([]string{string(AssetTypeProp)}, nil)
 	if err != nil {
@@ -1001,9 +880,9 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 	}, nil)
 
 	asr = NewAssetSetRange(
-		generateAssetSet(startD0),
-		generateAssetSet(startD1),
-		generateAssetSet(startD2),
+		GenerateMockAssetSet(startD0),
+		GenerateMockAssetSet(startD1),
+		GenerateMockAssetSet(startD2),
 	)
 	err = asr.AggregateBy([]string{string(AssetClusterProp)}, nil)
 	if err != nil {
@@ -1023,8 +902,8 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 	// is empty (this was previously an issue)
 	asr = NewAssetSetRange(
 		NewAssetSet(startD0, startD1),
-		generateAssetSet(startD1),
-		generateAssetSet(startD2),
+		GenerateMockAssetSet(startD1),
+		GenerateMockAssetSet(startD2),
 	)
 
 	err = asr.AggregateBy([]string{string(AssetTypeProp)}, nil)
@@ -1186,7 +1065,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 // })
 // }
 
-// // generateAssetSet generates the following topology:
+// // GenerateMockAssetSet generates the following topology:
 // //
 // // | Asset                        | Cost |  Adj |
 // // +------------------------------+------+------+
@@ -1215,7 +1094,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 // // +------------------------------+------+------+
 // //   total                          57.00   3.00
 // // +------------------------------+------+------+
-// func generateAssetSet(start time.Time) *AssetSet {
+// func GenerateMockAssetSet(start time.Time) *AssetSet {
 // end := start.Add(day)
 // window := NewWindow(&start, &end)
 

pkg/kubecost/bingen.go

@@ -29,4 +29,4 @@ package kubecost
 // @bingen:generate:PVKey
 // @bingen:generate:PVAllocation
 
-//go:generate bingen -package=kubecost -version=13 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=14 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/kubecost_codecs.go

@@ -14,10 +14,11 @@ package kubecost
 import (
 	"encoding"
 	"fmt"
-	util "github.com/kubecost/cost-model/pkg/util"
 	"reflect"
 	"strings"
 	"time"
+
+	util "github.com/kubecost/cost-model/pkg/util"
 )
 
 const (
@@ -25,7 +26,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 13
+	CodecVersion uint8 = 14
 )
 
 //--------------------------------------------------------------------------
@@ -220,7 +221,6 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteFloat64(target.RAMCost)                // write float64
 	buff.WriteFloat64(target.RAMCostAdjustment)      // write float64
 	buff.WriteFloat64(target.SharedCost)             // write float64
-	buff.WriteFloat64(target.SharedCostAdjustment)   // write float64
 	buff.WriteFloat64(target.ExternalCost)           // write float64
 	if target.RawAllocationOnly == nil {
 		buff.WriteUInt8(uint8(0)) // write nil byte
@@ -418,23 +418,20 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	target.SharedCost = uu
 
 	ww := buff.ReadFloat64() // read float64
-	target.SharedCostAdjustment = ww
-
-	xx := buff.ReadFloat64() // read float64
-	target.ExternalCost = xx
+	target.ExternalCost = ww
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.RawAllocationOnly = nil
 	} else {
 		// --- [begin][read][struct](RawAllocationOnlyData) ---
-		yy := &RawAllocationOnlyData{}
-		aaa := buff.ReadInt()      // byte array length
-		bbb := buff.ReadBytes(aaa) // byte array
-		errG := yy.UnmarshalBinary(bbb)
+		xx := &RawAllocationOnlyData{}
+		yy := buff.ReadInt()      // byte array length
+		aaa := buff.ReadBytes(yy) // byte array
+		errG := xx.UnmarshalBinary(aaa)
 		if errG != nil {
 			return errG
 		}
-		target.RawAllocationOnly = yy
+		target.RawAllocationOnly = xx
 		// --- [end][read][struct](RawAllocationOnlyData) ---
 
 	}
@@ -721,6 +718,7 @@ func (target *AllocationSet) MarshalBinary() (data []byte, err error) {
 		// --- [end][write][map](map[string]bool) ---
 
 	}
+	buff.WriteString(target.FromSource) // write string
 	// --- [begin][write][struct](Window) ---
 	b, errB := target.Window.MarshalBinary()
 	if errB != nil {
@@ -858,31 +856,34 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 		// --- [end][read][map](map[string]bool) ---
 
 	}
+	q := buff.ReadString() // read string
+	target.FromSource = q
+
 	// --- [begin][read][struct](Window) ---
-	q := &Window{}
-	r := buff.ReadInt()    // byte array length
-	s := buff.ReadBytes(r) // byte array
-	errB := q.UnmarshalBinary(s)
+	r := &Window{}
+	s := buff.ReadInt()    // byte array length
+	t := buff.ReadBytes(s) // byte array
+	errB := r.UnmarshalBinary(t)
 	if errB != nil {
 		return errB
 	}
-	target.Window = *q
+	target.Window = *r
 	// --- [end][read][struct](Window) ---
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.Warnings = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		u := buff.ReadInt() // array len
-		t := make([]string, u)
-		for jj := 0; jj < u; jj++ {
-			var w string
-			x := buff.ReadString() // read string
-			w = x
+		w := buff.ReadInt() // array len
+		u := make([]string, w)
+		for jj := 0; jj < w; jj++ {
+			var x string
+			y := buff.ReadString() // read string
+			x = y
 
-			t[jj] = w
+			u[jj] = x
 		}
-		target.Warnings = t
+		target.Warnings = u
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -890,16 +891,16 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 		target.Errors = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		aa := buff.ReadInt() // array len
-		y := make([]string, aa)
-		for iii := 0; iii < aa; iii++ {
-			var bb string
-			cc := buff.ReadString() // read string
-			bb = cc
+		bb := buff.ReadInt() // array len
+		aa := make([]string, bb)
+		for iii := 0; iii < bb; iii++ {
+			var cc string
+			dd := buff.ReadString() // read string
+			cc = dd
 
-			y[iii] = bb
+			aa[iii] = cc
 		}
-		target.Errors = y
+		target.Errors = aa
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -956,6 +957,7 @@ func (target *AllocationSetRange) MarshalBinary() (data []byte, err error) {
 		// --- [end][write][slice]([]*AllocationSet) ---
 
 	}
+	buff.WriteString(target.FromStore) // write string
 	return buff.Bytes(), nil
 }
 
@@ -1012,6 +1014,9 @@ func (target *AllocationSetRange) UnmarshalBinary(data []byte) (err error) {
 		// --- [end][read][slice]([]*AllocationSet) ---
 
 	}
+	g := buff.ReadString() // read string
+	target.FromStore = g
+
 	return nil
 }
 
@@ -1364,6 +1369,7 @@ func (target *AssetSet) MarshalBinary() (data []byte, err error) {
 		// --- [end][write][map](map[string]Asset) ---
 
 	}
+	buff.WriteString(target.FromSource) // write string
 	// --- [begin][write][struct](Window) ---
 	d, errB := target.Window.MarshalBinary()
 	if errB != nil {
@@ -1484,31 +1490,34 @@ func (target *AssetSet) UnmarshalBinary(data []byte) (err error) {
 		// --- [end][read][map](map[string]Asset) ---
 
 	}
+	o := buff.ReadString() // read string
+	target.FromSource = o
+
 	// --- [begin][read][struct](Window) ---
-	o := &Window{}
-	p := buff.ReadInt()    // byte array length
-	q := buff.ReadBytes(p) // byte array
-	errB := o.UnmarshalBinary(q)
+	p := &Window{}
+	q := buff.ReadInt()    // byte array length
+	r := buff.ReadBytes(q) // byte array
+	errB := p.UnmarshalBinary(r)
 	if errB != nil {
 		return errB
 	}
-	target.Window = *o
+	target.Window = *p
 	// --- [end][read][struct](Window) ---
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.Warnings = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		s := buff.ReadInt() // array len
-		r := make([]string, s)
-		for ii := 0; ii < s; ii++ {
-			var t string
-			u := buff.ReadString() // read string
-			t = u
+		t := buff.ReadInt() // array len
+		s := make([]string, t)
+		for ii := 0; ii < t; ii++ {
+			var u string
+			w := buff.ReadString() // read string
+			u = w
 
-			r[ii] = t
+			s[ii] = u
 		}
-		target.Warnings = r
+		target.Warnings = s
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -1516,16 +1525,16 @@ func (target *AssetSet) UnmarshalBinary(data []byte) (err error) {
 		target.Errors = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		x := buff.ReadInt() // array len
-		w := make([]string, x)
-		for jj := 0; jj < x; jj++ {
-			var y string
-			aa := buff.ReadString() // read string
-			y = aa
+		y := buff.ReadInt() // array len
+		x := make([]string, y)
+		for jj := 0; jj < y; jj++ {
+			var aa string
+			bb := buff.ReadString() // read string
+			aa = bb
 
-			w[jj] = y
+			x[jj] = aa
 		}
-		target.Errors = w
+		target.Errors = x
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -1582,6 +1591,7 @@ func (target *AssetSetRange) MarshalBinary() (data []byte, err error) {
 		// --- [end][write][slice]([]*AssetSet) ---
 
 	}
+	buff.WriteString(target.FromStore) // write string
 	return buff.Bytes(), nil
 }
 
@@ -1638,6 +1648,9 @@ func (target *AssetSetRange) UnmarshalBinary(data []byte) (err error) {
 		// --- [end][read][slice]([]*AssetSet) ---
 
 	}
+	g := buff.ReadString() // read string
+	target.FromStore = g
+
 	return nil
 }
 

pkg/kubecost/kubecost_codecs_test.go

@@ -25,9 +25,9 @@ func BenchmarkAllocationSetRange_BinaryEncoding(b *testing.B) {
 	var err error
 
 	asr0 = NewAllocationSetRange(
-		generateAllocationSetClusterIdle(startD0),
-		generateAllocationSetClusterIdle(startD1),
-		generateAllocationSetClusterIdle(startD2),
+		GenerateMockAllocationSetClusterIdle(startD0),
+		GenerateMockAllocationSetClusterIdle(startD1),
+		GenerateMockAllocationSetClusterIdle(startD2),
 	)
 
 	for it := 0; it < b.N; it++ {
@@ -90,9 +90,9 @@ func TestAllocationSetRange_BinaryEncoding(t *testing.T) {
 	var err error
 
 	asr0 = NewAllocationSetRange(
-		generateAllocationSetClusterIdle(startD0),
-		generateAllocationSetClusterIdle(startD1),
-		generateAllocationSetClusterIdle(startD2),
+		GenerateMockAllocationSetClusterIdle(startD0),
+		GenerateMockAllocationSetClusterIdle(startD1),
+		GenerateMockAllocationSetClusterIdle(startD2),
 	)
 
 	bs, err = asr0.MarshalBinary()
@@ -212,9 +212,9 @@ func TestAssetSetRange_BinaryEncoding(t *testing.T) {
 	var err error
 
 	asr0 = NewAssetSetRange(
-		generateAssetSet(startD0),
-		generateAssetSet(startD1),
-		generateAssetSet(startD2),
+		GenerateMockAssetSet(startD0),
+		GenerateMockAssetSet(startD1),
+		GenerateMockAssetSet(startD2),
 	)
 
 	bs, err = asr0.MarshalBinary()

pkg/kubecost/mock.go

@@ -0,0 +1,667 @@
+package kubecost
+
+import (
+	"fmt"
+	"time"
+)
+const gb = 1024 * 1024 * 1024
+const day = 24 * time.Hour
+var disk = PVKey{}
+
+// NewMockUnitAllocation creates an *Allocation with all of its float64 values set to 1 and generic properties if not provided in arg
+func NewMockUnitAllocation(name string, start time.Time, resolution time.Duration, props *AllocationProperties) *Allocation {
+	if name == "" {
+		name = "cluster1/namespace1/pod1/container1"
+	}
+
+	properties := &AllocationProperties{}
+	if props == nil {
+		properties.Cluster = "cluster1"
+		properties.Node = "node1"
+		properties.Namespace = "namespace1"
+		properties.ControllerKind = "deployment"
+		properties.Controller = "deployment1"
+		properties.Pod = "pod1"
+		properties.Container = "container1"
+	} else {
+		properties = props
+	}
+
+	end := start.Add(resolution)
+
+	alloc := &Allocation{
+		Name:                  name,
+		Properties:            properties,
+		Window:                NewWindow(&start, &end).Clone(),
+		Start:                 start,
+		End:                   end,
+		CPUCoreHours:          1,
+		CPUCost:               1,
+		CPUCoreRequestAverage: 1,
+		CPUCoreUsageAverage:   1,
+		GPUHours:              1,
+		GPUCost:               1,
+		NetworkCost:           1,
+		LoadBalancerCost:      1,
+		PVs: PVAllocations{
+			disk: {
+				ByteHours: 1,
+				Cost:      1,
+			},
+		},
+		RAMByteHours:           1,
+		RAMCost:                1,
+		RAMBytesRequestAverage: 1,
+		RAMBytesUsageAverage:   1,
+		RawAllocationOnly: &RawAllocationOnlyData{
+			CPUCoreUsageMax:  1,
+			RAMBytesUsageMax: 1,
+		},
+	}
+
+	// If idle allocation, remove non-idle costs, but maintain total cost
+	if alloc.IsIdle() {
+		alloc.PVs = nil
+		alloc.NetworkCost = 0.0
+		alloc.LoadBalancerCost = 0.0
+		alloc.CPUCoreHours += 1.0
+		alloc.CPUCost += 1.0
+		alloc.RAMByteHours += 1.0
+		alloc.RAMCost += 1.0
+	}
+
+	return alloc
+}
+
+// GenerateMockAllocationSetClusterIdle creates generic allocation set which includes an idle set broken down by cluster
+func GenerateMockAllocationSetClusterIdle(start time.Time) *AllocationSet {
+	// Cluster Idle allocations
+	a1i := NewMockUnitAllocation(fmt.Sprintf("cluster1/%s", IdleSuffix), start, day, &AllocationProperties{
+		Cluster: "cluster1",
+	})
+	a1i.CPUCost = 5.0
+	a1i.RAMCost = 15.0
+	a1i.GPUCost = 0.0
+
+	a2i := NewMockUnitAllocation(fmt.Sprintf("cluster2/%s", IdleSuffix), start, day, &AllocationProperties{
+		Cluster: "cluster2",
+	})
+	a2i.CPUCost = 5.0
+	a2i.RAMCost = 5.0
+	a2i.GPUCost = 0.0
+
+	as := GenerateMockAllocationSet(start)
+	as.Insert(a1i)
+	as.Insert(a2i)
+	return as
+}
+
+// GenerateMockAllocationSetNodeIdle creates generic allocation set which includes an idle set broken down by node
+func GenerateMockAllocationSetNodeIdle(start time.Time) *AllocationSet {
+	// Node Idle allocations
+	a11i := NewMockUnitAllocation(fmt.Sprintf("c1nodes/%s", IdleSuffix), start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Node:       "c1nodes",
+		ProviderID: "c1nodes",
+	})
+	a11i.CPUCost = 5.0
+	a11i.RAMCost = 15.0
+	a11i.GPUCost = 0.0
+
+	a21i := NewMockUnitAllocation(fmt.Sprintf("node1/%s", IdleSuffix), start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Node:       "node1",
+		ProviderID: "node1",
+	})
+	a21i.CPUCost = 1.666667
+	a21i.RAMCost = 1.666667
+	a21i.GPUCost = 0.0
+
+	a22i := NewMockUnitAllocation(fmt.Sprintf("node2/%s", IdleSuffix), start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Node:       "node2",
+		ProviderID: "node2",
+	})
+	a22i.CPUCost = 1.666667
+	a22i.RAMCost = 1.666667
+	a22i.GPUCost = 0.0
+
+	a23i := NewMockUnitAllocation(fmt.Sprintf("node3/%s", IdleSuffix), start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Node:       "node3",
+		ProviderID: "node3",
+		Namespace:  "",
+	})
+	a23i.CPUCost = 1.666667
+	a23i.RAMCost = 1.666667
+	a23i.GPUCost = 0.0
+
+	as := GenerateMockAllocationSet(start)
+	as.Insert(a11i)
+	as.Insert(a21i)
+	as.Insert(a22i)
+	as.Insert(a23i)
+	return as
+}
+
+// GenerateMockAllocationSet creates generic allocation set without idle allocations
+func GenerateMockAllocationSet(start time.Time) *AllocationSet {
+
+	// Active allocations
+	a1111 := NewMockUnitAllocation("cluster1/namespace1/pod1/container1", start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Namespace:  "namespace1",
+		Pod:        "pod1",
+		Container:  "container1",
+		ProviderID: "c1nodes",
+	})
+	a1111.RAMCost = 11.00
+
+	a11abc2 := NewMockUnitAllocation("cluster1/namespace1/pod-abc/container2", start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Namespace:  "namespace1",
+		Pod:        "pod-abc",
+		Container:  "container2",
+		ProviderID: "c1nodes",
+	})
+
+	a11def3 := NewMockUnitAllocation("cluster1/namespace1/pod-def/container3", start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Namespace:  "namespace1",
+		Pod:        "pod-def",
+		Container:  "container3",
+		ProviderID: "c1nodes",
+	})
+
+	a12ghi4 := NewMockUnitAllocation("cluster1/namespace2/pod-ghi/container4", start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Namespace:  "namespace2",
+		Pod:        "pod-ghi",
+		Container:  "container4",
+		ProviderID: "c1nodes",
+	})
+
+	a12ghi5 := NewMockUnitAllocation("cluster1/namespace2/pod-ghi/container5", start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Namespace:  "namespace2",
+		Pod:        "pod-ghi",
+		Container:  "container5",
+		ProviderID: "c1nodes",
+	})
+
+	a12jkl6 := NewMockUnitAllocation("cluster1/namespace2/pod-jkl/container6", start, day, &AllocationProperties{
+		Cluster:    "cluster1",
+		Namespace:  "namespace2",
+		Pod:        "pod-jkl",
+		Container:  "container6",
+		ProviderID: "c1nodes",
+	})
+
+	a22mno4 := NewMockUnitAllocation("cluster2/namespace2/pod-mno/container4", start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Namespace:  "namespace2",
+		Pod:        "pod-mno",
+		Container:  "container4",
+		ProviderID: "node1",
+	})
+
+	a22mno5 := NewMockUnitAllocation("cluster2/namespace2/pod-mno/container5", start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Namespace:  "namespace2",
+		Pod:        "pod-mno",
+		Container:  "container5",
+		ProviderID: "node1",
+	})
+
+	a22pqr6 := NewMockUnitAllocation("cluster2/namespace2/pod-pqr/container6", start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Namespace:  "namespace2",
+		Pod:        "pod-pqr",
+		Container:  "container6",
+		ProviderID: "node2",
+	})
+
+	a23stu7 := NewMockUnitAllocation("cluster2/namespace3/pod-stu/container7", start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Namespace:  "namespace3",
+		Pod:        "pod-stu",
+		Container:  "container7",
+		ProviderID: "node2",
+	})
+
+	a23vwx8 := NewMockUnitAllocation("cluster2/namespace3/pod-vwx/container8", start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Namespace:  "namespace3",
+		Pod:        "pod-vwx",
+		Container:  "container8",
+		ProviderID: "node3",
+	})
+
+	a23vwx9 := NewMockUnitAllocation("cluster2/namespace3/pod-vwx/container9", start, day, &AllocationProperties{
+		Cluster:    "cluster2",
+		Namespace:  "namespace3",
+		Pod:        "pod-vwx",
+		Container:  "container9",
+		ProviderID: "node3",
+	})
+
+	// Controllers
+
+	a11abc2.Properties.ControllerKind = "deployment"
+	a11abc2.Properties.Controller = "deployment1"
+	a11def3.Properties.ControllerKind = "deployment"
+	a11def3.Properties.Controller = "deployment1"
+
+	a12ghi4.Properties.ControllerKind = "deployment"
+	a12ghi4.Properties.Controller = "deployment2"
+	a12ghi5.Properties.ControllerKind = "deployment"
+	a12ghi5.Properties.Controller = "deployment2"
+	a22mno4.Properties.ControllerKind = "deployment"
+	a22mno4.Properties.Controller = "deployment2"
+	a22mno5.Properties.ControllerKind = "deployment"
+	a22mno5.Properties.Controller = "deployment2"
+
+	a23stu7.Properties.ControllerKind = "deployment"
+	a23stu7.Properties.Controller = "deployment3"
+
+	a12jkl6.Properties.ControllerKind = "daemonset"
+	a12jkl6.Properties.Controller = "daemonset1"
+	a22pqr6.Properties.ControllerKind = "daemonset"
+	a22pqr6.Properties.Controller = "daemonset1"
+
+	a23vwx8.Properties.ControllerKind = "statefulset"
+	a23vwx8.Properties.Controller = "statefulset1"
+	a23vwx9.Properties.ControllerKind = "statefulset"
+	a23vwx9.Properties.Controller = "statefulset1"
+
+	// Labels
+
+	a1111.Properties.Labels = map[string]string{"app": "app1", "env": "env1"}
+	a12ghi4.Properties.Labels = map[string]string{"app": "app2", "env": "env2"}
+	a12ghi5.Properties.Labels = map[string]string{"app": "app2", "env": "env2"}
+	a22mno4.Properties.Labels = map[string]string{"app": "app2"}
+	a22mno5.Properties.Labels = map[string]string{"app": "app2"}
+
+	//Annotations
+	a23stu7.Properties.Annotations = map[string]string{"team": "team1"}
+	a23vwx8.Properties.Annotations = map[string]string{"team": "team2"}
+	a23vwx9.Properties.Annotations = map[string]string{"team": "team1"}
+
+	// Services
+	a12jkl6.Properties.Services = []string{"service1"}
+	a22pqr6.Properties.Services = []string{"service1"}
+
+	return NewAllocationSet(start, start.Add(day),
+		// cluster 1, namespace1
+		a1111, a11abc2, a11def3,
+		// cluster 1, namespace 2
+		a12ghi4, a12ghi5, a12jkl6,
+		// cluster 2, namespace 2
+		a22mno4, a22mno5, a22pqr6,
+		// cluster 2, namespace 3
+		a23stu7, a23vwx8, a23vwx9,
+	)
+}
+
+// GenerateMockAllocationSetWithAssetProperties with no idle and connections to Assets in properties
+func GenerateMockAllocationSetWithAssetProperties(start time.Time)  *AllocationSet {
+	as := GenerateMockAllocationSet(start)
+	disk1 := PVKey{
+		Cluster: "cluster2",
+		Name:    "disk1",
+	}
+	disk2 := PVKey{
+		Cluster: "cluster2",
+		Name:    "disk2",
+	}
+	for _, a := range as.allocations {
+		// add reconcilable pvs to pod-mno
+		if a.Properties.Pod == "pod-mno" {
+			a.PVs = a.PVs.Add(PVAllocations{
+				disk1: {
+					Cost:      2.5,
+					ByteHours: 2.5 * gb,
+				},
+				disk2: {
+					Cost:      5,
+					ByteHours: 5 * gb,
+				},
+			})
+		}
+		// add loadBalancer service to allocations
+		if a.Name == "cluster2/namespace2/pod-mno/container4" {
+			a.Properties.Services = append(a.Properties.Services, "loadBalancer1")
+		}
+		if a.Name == "cluster2/namespace2/pod-mno/container5" {
+			a.Properties.Services = append(a.Properties.Services, "loadBalancer2")
+		}
+		if a.Name == "cluster2/namespace2/pod-pqr/container6" {
+			a.Properties.Services = append(a.Properties.Services, "loadBalancer1")
+			a.Properties.Services = append(a.Properties.Services, "loadBalancer2")
+		}
+	}
+	return as
+}
+
+// GenerateMockAssetSets creates generic AssetSets
+func GenerateMockAssetSets(start, end time.Time) []*AssetSet {
+	var assetSets []*AssetSet
+
+	// Create an AssetSet representing cluster costs for two clusters (cluster1
+	// and cluster2). Include Nodes and Disks for both, even though only
+	// Nodes will be counted. Whereas in practice, Assets should be aggregated
+	// by type, here we will provide multiple Nodes for one of the clusters to
+	// make sure the function still holds.
+
+	// NOTE: we're re-using GenerateMockAllocationSet so this has to line up with
+	// the allocated node costs from that function. See table above.
+
+	// | Hierarchy                               | Cost |  CPU |  RAM |  GPU | Adjustment |
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1:
+	//     nodes                                  100.00  55.00  44.00  11.00      -10.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1 subtotal (adjusted)             100.00  50.00  40.00  10.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1 allocated                        48.00   6.00  16.00   6.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1 idle                             72.00  44.00  24.00   4.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2:
+	//     node1                                   35.00  20.00  15.00   0.00        0.00
+	//     node2                                   35.00  20.00  15.00   0.00        0.00
+	//     node3                                   30.00  10.00  10.00  10.00        0.00
+	//     (disks should not matter for idle)
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2 subtotal                        100.00  50.00  40.00  10.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2 allocated                        28.00   6.00   6.00   6.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2 idle                             82.00  44.00  34.00   4.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+
+	cluster1Nodes := NewNode("c1nodes", "cluster1", "c1nodes", start, end, NewWindow(&start, &end))
+	cluster1Nodes.CPUCost = 55.0
+	cluster1Nodes.RAMCost = 44.0
+	cluster1Nodes.GPUCost = 11.0
+	cluster1Nodes.adjustment = -10.00
+	cluster1Nodes.CPUCoreHours = 8
+	cluster1Nodes.RAMByteHours = 6
+	cluster1Nodes.GPUHours = 24
+
+	cluster2Node1 := NewNode("node1", "cluster2", "node1", start, end, NewWindow(&start, &end))
+	cluster2Node1.CPUCost = 20.0
+	cluster2Node1.RAMCost = 15.0
+	cluster2Node1.GPUCost = 0.0
+	cluster2Node1.CPUCoreHours = 4
+	cluster2Node1.RAMByteHours = 3
+	cluster2Node1.GPUHours = 0
+
+	cluster2Node2 := NewNode("node2", "cluster2", "node2", start, end, NewWindow(&start, &end))
+	cluster2Node2.CPUCost = 20.0
+	cluster2Node2.RAMCost = 15.0
+	cluster2Node2.GPUCost = 0.0
+	cluster2Node2.CPUCoreHours = 3
+	cluster2Node2.RAMByteHours = 2
+	cluster2Node2.GPUHours = 0
+
+	cluster2Node3 := NewNode("node3", "cluster2", "node3", start, end, NewWindow(&start, &end))
+	cluster2Node3.CPUCost = 10.0
+	cluster2Node3.RAMCost = 10.0
+	cluster2Node3.GPUCost = 10.0
+	cluster2Node3.CPUCoreHours = 2
+	cluster2Node3.RAMByteHours = 2
+	cluster2Node3.GPUHours = 24
+
+	// Add PVs
+	cluster2Disk1 := NewDisk("disk1", "cluster2", "disk1", start, end, NewWindow(&start, &end))
+	cluster2Disk1.Cost = 5.0
+	cluster2Disk1.adjustment = 1.0
+	cluster2Disk1.ByteHours = 5 * gb
+
+	cluster2Disk2 := NewDisk("disk2", "cluster2", "disk2", start, end, NewWindow(&start, &end))
+	cluster2Disk2.Cost = 10.0
+	cluster2Disk2.adjustment = 3.0
+	cluster2Disk2.ByteHours = 10 * gb
+
+	cluster2Node1Disk := NewDisk("node1", "cluster2", "node1", start, end, NewWindow(&start, &end))
+	cluster2Node1Disk.Cost = 1.0
+	cluster2Node1Disk.ByteHours = 5 * gb
+
+	// Add Attached Disks
+	cluster2Node2Disk := NewDisk("node2", "cluster2", "node2", start, end, NewWindow(&start, &end))
+	cluster2Node2Disk.Cost = 2.0
+	cluster2Node2Disk.ByteHours = 5 * gb
+
+	cluster2Node3Disk := NewDisk("node3", "cluster2", "node3", start, end, NewWindow(&start, &end))
+	cluster2Node3Disk.Cost = 3.0
+	cluster2Node3Disk.ByteHours = 5 * gb
+
+	// Add Cluster Management
+	cluster1ClusterManagement := NewClusterManagement("", "cluster1", NewWindow(&start, &end))
+	cluster1ClusterManagement.Cost = 2.0
+
+	cluster2ClusterManagement := NewClusterManagement("", "cluster2", NewWindow(&start, &end))
+	cluster2ClusterManagement.Cost = 2.0
+
+	// Add Networks
+	c1Network := NewNetwork("", "cluster1", "c1nodes", start, end, NewWindow(&start, &end))
+	c1Network.Cost = 3.0
+
+	node1Network := NewNetwork("node1", "cluster2", "node1", start, end, NewWindow(&start, &end))
+	node1Network.Cost = 4.0
+
+	node2Network := NewNetwork("node2", "cluster2", "node2", start, end, NewWindow(&start, &end))
+	node2Network.Cost = 5.0
+
+	node3Network := NewNetwork("node3", "cluster2", "node3", start, end, NewWindow(&start, &end))
+	node3Network.Cost = 2.0
+
+	// Add LoadBalancers
+	cluster2LoadBalancer1 := NewLoadBalancer("namespace2/loadBalancer1", "cluster2", "lb1", start, end, NewWindow(&start, &end))
+	cluster2LoadBalancer1.Cost = 10.0
+
+	cluster2LoadBalancer2 := NewLoadBalancer("namespace2/loadBalancer2", "cluster2", "lb2", start, end, NewWindow(&start, &end))
+	cluster2LoadBalancer2.Cost = 15.0
+
+	assetSet1 := NewAssetSet(start, end, cluster1Nodes, cluster2Node1, cluster2Node2, cluster2Node3, cluster2Disk1,
+		cluster2Disk2, cluster2Node1Disk, cluster2Node2Disk, cluster2Node3Disk, cluster1ClusterManagement,
+		cluster2ClusterManagement, c1Network, node1Network, node2Network, node3Network, cluster2LoadBalancer1, cluster2LoadBalancer2)
+	assetSets = append(assetSets, assetSet1)
+
+	// NOTE: we're re-using GenerateMockAllocationSet so this has to line up with
+	// the allocated node costs from that function. See table above.
+
+	// | Hierarchy                               | Cost |  CPU |  RAM |  GPU | Adjustment |
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1:
+	//     nodes                                  100.00   5.00   4.00   1.00       90.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1 subtotal (adjusted)             100.00  50.00  40.00  10.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1 allocated                        48.00   6.00  16.00   6.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster1 idle                             72.00  44.00  24.00   4.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2:
+	//     node1                                   35.00  20.00  15.00   0.00        0.00
+	//     node2                                   35.00  20.00  15.00   0.00        0.00
+	//     node3                                   30.00  10.00  10.00  10.00        0.00
+	//     (disks should not matter for idle)
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2 subtotal                        100.00  50.00  40.00  10.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2 allocated                        28.00   6.00   6.00   6.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+	//   cluster2 idle                             82.00  44.00  34.00   4.00        0.00
+	// +-----------------------------------------+------+------+------+------+------------+
+
+	cluster1Nodes = NewNode("", "cluster1", "c1nodes", start, end, NewWindow(&start, &end))
+	cluster1Nodes.CPUCost = 5.0
+	cluster1Nodes.RAMCost = 4.0
+	cluster1Nodes.GPUCost = 1.0
+	cluster1Nodes.adjustment = 90.00
+	cluster1Nodes.CPUCoreHours = 8
+	cluster1Nodes.RAMByteHours = 6
+	cluster1Nodes.GPUHours = 24
+
+	cluster2Node1 = NewNode("node1", "cluster2", "node1", start, end, NewWindow(&start, &end))
+	cluster2Node1.CPUCost = 20.0
+	cluster2Node1.RAMCost = 15.0
+	cluster2Node1.GPUCost = 0.0
+	cluster2Node1.CPUCoreHours = 4
+	cluster2Node1.RAMByteHours = 3
+	cluster2Node1.GPUHours = 0
+
+	cluster2Node2 = NewNode("node2", "cluster2", "node2", start, end, NewWindow(&start, &end))
+	cluster2Node2.CPUCost = 20.0
+	cluster2Node2.RAMCost = 15.0
+	cluster2Node2.GPUCost = 0.0
+	cluster2Node2.CPUCoreHours = 3
+	cluster2Node2.RAMByteHours = 2
+	cluster2Node2.GPUHours = 0
+
+	cluster2Node3 = NewNode("node3", "cluster2", "node3", start, end, NewWindow(&start, &end))
+	cluster2Node3.CPUCost = 10.0
+	cluster2Node3.RAMCost = 10.0
+	cluster2Node3.GPUCost = 10.0
+	cluster2Node3.CPUCoreHours = 2
+	cluster2Node3.RAMByteHours = 2
+	cluster2Node3.GPUHours = 24
+
+	// Add PVs
+	cluster2Disk1 = NewDisk("disk1", "cluster2", "disk1", start, end, NewWindow(&start, &end))
+	cluster2Disk1.Cost = 5.0
+	cluster2Disk1.adjustment = 1.0
+	cluster2Disk1.ByteHours = 5 * gb
+
+	cluster2Disk2 = NewDisk("disk2", "cluster2", "disk2", start, end, NewWindow(&start, &end))
+	cluster2Disk2.Cost = 12.0
+	cluster2Disk2.adjustment = 4.0
+	cluster2Disk2.ByteHours = 20 * gb
+
+	assetSet2 := NewAssetSet(start, end, cluster1Nodes, cluster2Node1, cluster2Node2, cluster2Node3, cluster2Disk1,
+		cluster2Disk2, cluster2Node1Disk, cluster2Node2Disk, cluster2Node3Disk, cluster1ClusterManagement,
+		cluster2ClusterManagement, c1Network, node1Network, node2Network, node3Network, cluster2LoadBalancer1, cluster2LoadBalancer2)
+	assetSets = append(assetSets, assetSet2)
+	return assetSets
+}
+
+// GenerateMockAssetSet generates the following topology:
+//
+// | Asset                        | Cost |  Adj |
+// +------------------------------+------+------+
+//   cluster1:
+//     node1:                        6.00   1.00
+//     node2:                        4.00   1.50
+//     node3:                        7.00  -0.50
+//     disk1:                        2.50   0.00
+//     disk2:                        1.50   0.00
+//     clusterManagement1:           3.00   0.00
+// +------------------------------+------+------+
+//   cluster1 subtotal              24.00   2.00
+// +------------------------------+------+------+
+//   cluster2:
+//     node4:                       12.00  -1.00
+//     disk3:                        2.50   0.00
+//     disk4:                        1.50   0.00
+//     clusterManagement2:           0.00   0.00
+// +------------------------------+------+------+
+//   cluster2 subtotal              16.00  -1.00
+// +------------------------------+------+------+
+//   cluster3:
+//     node5:                       17.00   2.00
+// +------------------------------+------+------+
+//   cluster3 subtotal              17.00   2.00
+// +------------------------------+------+------+
+//   total                          57.00   3.00
+// +------------------------------+------+------+
+func GenerateMockAssetSet(start time.Time) *AssetSet {
+	end := start.Add(day)
+	window := NewWindow(&start, &end)
+
+	hours := window.Duration().Hours()
+
+	node1 := NewNode("node1", "cluster1", "gcp-node1", *window.Clone().start, *window.Clone().end, window.Clone())
+	node1.CPUCost = 4.0
+	node1.RAMCost = 4.0
+	node1.GPUCost = 2.0
+	node1.Discount = 0.5
+	node1.CPUCoreHours = 2.0 * hours
+	node1.RAMByteHours = 4.0 * gb * hours
+	node1.GPUHours = 1.0 * hours
+	node1.SetAdjustment(1.0)
+	node1.SetLabels(map[string]string{"test": "test"})
+
+	node2 := NewNode("node2", "cluster1", "gcp-node2", *window.Clone().start, *window.Clone().end, window.Clone())
+	node2.CPUCost = 4.0
+	node2.RAMCost = 4.0
+	node2.GPUCost = 0.0
+	node2.Discount = 0.5
+	node2.CPUCoreHours = 2.0 * hours
+	node2.RAMByteHours = 4.0 * gb * hours
+	node2.GPUHours = 0.0 * hours
+	node2.SetAdjustment(1.5)
+
+	node3 := NewNode("node3", "cluster1", "gcp-node3", *window.Clone().start, *window.Clone().end, window.Clone())
+	node3.CPUCost = 4.0
+	node3.RAMCost = 4.0
+	node3.GPUCost = 3.0
+	node3.Discount = 0.5
+	node3.CPUCoreHours = 2.0 * hours
+	node3.RAMByteHours = 4.0 * gb * hours
+	node3.GPUHours = 2.0 * hours
+	node3.SetAdjustment(-0.5)
+
+	node4 := NewNode("node4", "cluster2", "gcp-node4", *window.Clone().start, *window.Clone().end, window.Clone())
+	node4.CPUCost = 10.0
+	node4.RAMCost = 6.0
+	node4.GPUCost = 0.0
+	node4.Discount = 0.25
+	node4.CPUCoreHours = 4.0 * hours
+	node4.RAMByteHours = 12.0 * gb * hours
+	node4.GPUHours = 0.0 * hours
+	node4.SetAdjustment(-1.0)
+
+	node5 := NewNode("node5", "cluster3", "aws-node5", *window.Clone().start, *window.Clone().end, window.Clone())
+	node5.CPUCost = 10.0
+	node5.RAMCost = 7.0
+	node5.GPUCost = 0.0
+	node5.Discount = 0.0
+	node5.CPUCoreHours = 8.0 * hours
+	node5.RAMByteHours = 24.0 * gb * hours
+	node5.GPUHours = 0.0 * hours
+	node5.SetAdjustment(2.0)
+
+	disk1 := NewDisk("disk1", "cluster1", "gcp-disk1", *window.Clone().start, *window.Clone().end, window.Clone())
+	disk1.Cost = 2.5
+	disk1.ByteHours = 100 * gb * hours
+
+	disk2 := NewDisk("disk2", "cluster1", "gcp-disk2", *window.Clone().start, *window.Clone().end, window.Clone())
+	disk2.Cost = 1.5
+	disk2.ByteHours = 60 * gb * hours
+
+	disk3 := NewDisk("disk3", "cluster2", "gcp-disk3", *window.Clone().start, *window.Clone().end, window.Clone())
+	disk3.Cost = 2.5
+	disk3.ByteHours = 100 * gb * hours
+
+	disk4 := NewDisk("disk4", "cluster2", "gcp-disk4", *window.Clone().start, *window.Clone().end, window.Clone())
+	disk4.Cost = 1.5
+	disk4.ByteHours = 100 * gb * hours
+
+	cm1 := NewClusterManagement("gcp", "cluster1", window.Clone())
+	cm1.Cost = 3.0
+
+	cm2 := NewClusterManagement("gcp", "cluster2", window.Clone())
+	cm2.Cost = 0.0
+
+	return NewAssetSet(
+		start, end,
+		// cluster 1
+		node1, node2, node3, disk1, disk2, cm1,
+		// cluster 2
+		node4, disk3, disk4, cm2,
+		// cluster 3
+		node5,
+	)
+}

pkg/prom/error.go

@@ -260,6 +260,11 @@ func NewCommError(messages ...string) CommError {
 	return CommError{messages: messages}
 }
 
+// CommErrorf creates a new CommError using a string formatter
+func CommErrorf(format string, args ...interface{}) CommError {
+	return NewCommError(fmt.Sprintf(format, args...))
+}
+
 // IsCommError returns true if the given error is a CommError
 func IsCommError(err error) bool {
 	_, ok := err.(CommError)

pkg/prom/query.go

@@ -186,6 +186,12 @@ func (ctx *Context) query(query string) (interface{}, prometheus.Warnings, error
 
 		return nil, warnings, fmt.Errorf("query error %d: '%s' fetching query '%s'", resp.StatusCode, err.Error(), query)
 	}
+	// Unsuccessful Status Code, log body and status
+	statusCode := resp.StatusCode
+	statusText := http.StatusText(statusCode)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, warnings, CommErrorf("%d (%s) URL: '%s' Headers: '%s', Body: '%s' Query: '%s'", statusCode, statusText, req.URL, util.HeaderString(resp.Header), body, query)
+	}
 
 	var toReturn interface{}
 	err = json.Unmarshal(body, &toReturn)
@@ -287,7 +293,7 @@ func (ctx *Context) queryRange(query string, start, end time.Time, step time.Dur
 	statusCode := resp.StatusCode
 	statusText := http.StatusText(statusCode)
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, warnings, fmt.Errorf("%d (%s) Headers: %s, Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), body, query)
+		return nil, warnings, CommErrorf("%d (%s) Headers: %s, Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), body, query)
 	}
 
 	var toReturn interface{}

pkg/util/time.go

@@ -3,6 +3,7 @@ package util
 import (
 	"fmt"
 	"strconv"
+	"sync"
 	"time"
 )
 
@@ -136,3 +137,89 @@ func normalizeTimeParam(param string) (string, error) {
 
 	return param, nil
 }
+
+// FormatStoreResolution provides a clean notation for ETL store resolutions.
+// e.g. daily => 1d; hourly => 1h
+func FormatStoreResolution(dur time.Duration) string {
+	if dur >= 24*time.Hour {
+		return fmt.Sprintf("%dd", int(dur.Hours()/24.0))
+	} else if dur >= time.Hour {
+		return fmt.Sprintf("%dh", int(dur.Hours()))
+	}
+	return fmt.Sprint(dur)
+}
+
+// JobTicker is a ticker used to synchronize the next run of a repeating
+// process. The designated use-case is for infinitely-looping selects,
+// where a timeout or an exit channel might cancel the process, but otherwise
+// the intent is to wait at the select for some amount of time until the
+// next run. This differs from a standard ticker, which ticks without
+// waiting and drops any missed ticks; rather, this ticker must be kicked
+// off manually for each tick, so that after the current run of the job
+// completes, the timer starts again.
+type JobTicker struct {
+	Ch     <-chan time.Time
+	ch     chan time.Time
+	closed bool
+	mx     sync.Mutex
+}
+
+// NewJobTicker instantiates a new JobTicker.
+func NewJobTicker() *JobTicker {
+	c := make(chan time.Time)
+
+	return &JobTicker{
+		Ch:     c,
+		ch:     c,
+		closed: false,
+	}
+}
+
+// Close closes the JobTicker channels
+func (jt *JobTicker) Close() {
+	jt.mx.Lock()
+	defer jt.mx.Unlock()
+
+	if jt.closed {
+		return
+	}
+
+	jt.closed = true
+	close(jt.ch)
+}
+
+// TickAt schedules the next tick of the ticker for the given time in the
+// future. If the time is not in the future, the ticker will tick immediately.
+func (jt *JobTicker) TickAt(t time.Time) {
+	go func(t time.Time) {
+		n := time.Now()
+		if t.After(n) {
+			time.Sleep(t.Sub(n))
+		}
+
+		jt.mx.Lock()
+		defer jt.mx.Unlock()
+
+		if !jt.closed {
+			jt.ch <- time.Now()
+		}
+	}(t)
+}
+
+// TickIn schedules the next tick of the ticker for the given duration into
+// the future. If the duration is less than or equal to zero, the ticker will
+// tick immediately.
+func (jt *JobTicker) TickIn(d time.Duration) {
+	go func(d time.Duration) {
+		if d > 0 {
+			time.Sleep(d)
+		}
+
+		jt.mx.Lock()
+		defer jt.mx.Unlock()
+
+		if !jt.closed {
+			jt.ch <- time.Now()
+		}
+	}(d)
+}