Merge pull request #719 from kubecost/niko/allocation

CostModel.ComputeAllocation: improved Allocation Source

pkg/cloud/provider.go

@@ -20,7 +20,6 @@ import (
 const authSecretPath = "/var/secrets/service-key.json"
 const storageConfigSecretPath = "/var/azure-storage-config/azure-storage-config.json"
 
-
 var createTableStatements = []string{
 	`CREATE TABLE IF NOT EXISTS names (
 		cluster_id VARCHAR(255) NOT NULL,
@@ -266,6 +265,7 @@ func CustomPricesEnabled(p Provider) bool {
 	if err != nil {
 		return false
 	}
+	// TODO:CLEANUP what is going on with this?
 	if config.NegotiatedDiscount == "" {
 		config.NegotiatedDiscount = "0%"
 	}

pkg/costmodel/aggregation.go

@@ -47,6 +47,8 @@ type Aggregation struct {
 	Environment                string               `json:"environment"`
 	Cluster                    string               `json:"cluster,omitempty"`
 	Properties                 *kubecost.Properties `json:"-"`
+	Start                      time.Time            `json:"-"`
+	End                        time.Time            `json:"-"`
 	CPUAllocationHourlyAverage float64              `json:"cpuAllocationAverage"`
 	CPUAllocationVectors       []*util.Vector       `json:"-"`
 	CPUAllocationTotal         float64              `json:"-"`

pkg/costmodel/allocation.go

@@ -0,0 +1,1803 @@
+package costmodel
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/kubecost"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/kubecost/cost-model/pkg/util"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+const (
+	queryFmtPods                  = `avg(kube_pod_container_status_running{}) by (pod, namespace, cluster_id)[%s:%s]%s`
+	queryFmtRAMBytesAllocated     = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMRequests           = `avg(avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMUsage              = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtCPUCoresAllocated     = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtCPURequests           = `avg(avg_over_time(kube_pod_container_resource_requests_cpu_cores{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtCPUUsage              = `avg(rate(container_cpu_usage_seconds_total{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtGPUsRequested         = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeCostPerGPUHr      = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeIsSpot            = `avg_over_time(kubecost_node_is_spot[%s]%s)`
+	queryFmtPVCInfo               = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, cluster_id)[%s:%s]%s`
+	queryFmtPVBytes               = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, cluster_id)`
+	queryFmtPodPVCAllocation      = `avg(avg_over_time(pod_pvc_allocation[%s]%s)) by (persistentvolume, persistentvolumeclaim, pod, namespace, cluster_id)`
+	queryFmtPVCBytesRequested     = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s]%s)) by (persistentvolumeclaim, namespace, cluster_id)`
+	queryFmtPVCostPerGiBHour      = `avg(avg_over_time(pv_hourly_cost[%s]%s)) by (volumename, cluster_id)`
+	queryFmtNetZoneGiB            = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s]%s)) by (pod_name, namespace, cluster_id) / 1024 / 1024 / 1024`
+	queryFmtNetZoneCostPerGiB     = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s]%s)) by (cluster_id)`
+	queryFmtNetRegionGiB          = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s]%s)) by (pod_name, namespace, cluster_id) / 1024 / 1024 / 1024`
+	queryFmtNetRegionCostPerGiB   = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s]%s)) by (cluster_id)`
+	queryFmtNetInternetGiB        = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s]%s)) by (pod_name, namespace, cluster_id) / 1024 / 1024 / 1024`
+	queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s]%s)) by (cluster_id)`
+	queryFmtNamespaceLabels       = `avg_over_time(kube_namespace_labels[%s]%s)`
+	queryFmtNamespaceAnnotations  = `avg_over_time(kube_namespace_annotations[%s]%s)`
+	queryFmtPodLabels             = `avg_over_time(kube_pod_labels[%s]%s)`
+	queryFmtPodAnnotations        = `avg_over_time(kube_pod_annotations[%s]%s)`
+	queryFmtServiceLabels         = `avg_over_time(service_selector_labels[%s]%s)`
+	queryFmtDeploymentLabels      = `avg_over_time(deployment_match_labels[%s]%s)`
+	queryFmtStatefulSetLabels     = `avg_over_time(statefulSet_match_labels[%s]%s)`
+	queryFmtDaemonSetLabels       = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s]%s)) by (pod, owner_name, namespace, cluster_id)`
+	queryFmtJobLabels             = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s]%s)) by (pod, owner_name, namespace ,cluster_id)`
+)
+
+// ComputeAllocation uses the CostModel instance to compute an AllocationSet
+// for the window defined by the given start and end times. The Allocations
+// returned are unaggregated (i.e. down to the container level).
+func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
+	// 1. Build out Pod map from resolution-tuned, batched Pod start/end query
+	// 2. Run and apply the results of the remaining queries to
+	// 3. Build out AllocationSet from completed Pod map
+
+	// Create a window spanning the requested query
+	window := kubecost.NewWindow(&start, &end)
+
+	// Create an empty AllocationSet. For safety, in the case of an error, we
+	// should prefer to return this empty set with the error. (In the case of
+	// no error, of course we populate the set and return it.)
+	allocSet := kubecost.NewAllocationSet(start, end)
+
+	// (1) Build out Pod map
+
+	// Build out a map of Allocations as a mapping from pod-to-container-to-
+	// underlying-Allocation instance, starting with (start, end) so that we
+	// begin with minutes, from which we compute resource allocation and cost
+	// totals from measured rate data.
+	podMap := map[podKey]*Pod{}
+
+	// clusterStarts and clusterEnds record the earliest start and latest end
+	// times, respectively, on a cluster-basis. These are used for unmounted
+	// PVs and other "virtual" Allocations so that minutes are maximally
+	// accurate during start-up or spin-down of a cluster
+	clusterStart := map[string]time.Time{}
+	clusterEnd := map[string]time.Time{}
+
+	cm.buildPodMap(window, resolution, env.GetETLMaxBatchDuration(), podMap, clusterStart, clusterEnd)
+
+	// (2) Run and apply remaining queries
+
+	// Convert window (start, end) to (duration, offset) for querying Prometheus,
+	// including handling Thanos offset
+	durStr, offStr, err := window.DurationOffsetForPrometheus()
+	if err != nil {
+		// Negative duration, so return empty set
+		return allocSet, nil
+	}
+
+	// Convert resolution duration to a query-ready string
+	resStr := util.DurationString(resolution)
+
+	ctx := prom.NewContext(cm.PrometheusClient)
+
+	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, durStr, offStr)
+	resChRAMBytesAllocated := ctx.Query(queryRAMBytesAllocated)
+
+	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, durStr, offStr)
+	resChRAMRequests := ctx.Query(queryRAMRequests)
+
+	queryRAMUsage := fmt.Sprintf(queryFmtRAMUsage, durStr, offStr)
+	resChRAMUsage := ctx.Query(queryRAMUsage)
+
+	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, durStr, offStr)
+	resChCPUCoresAllocated := ctx.Query(queryCPUCoresAllocated)
+
+	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, durStr, offStr)
+	resChCPURequests := ctx.Query(queryCPURequests)
+
+	queryCPUUsage := fmt.Sprintf(queryFmtCPUUsage, durStr, offStr)
+	resChCPUUsage := ctx.Query(queryCPUUsage)
+
+	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, durStr, offStr)
+	resChGPUsRequested := ctx.Query(queryGPUsRequested)
+
+	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, durStr, offStr)
+	resChNodeCostPerCPUHr := ctx.Query(queryNodeCostPerCPUHr)
+
+	queryNodeCostPerRAMGiBHr := fmt.Sprintf(queryFmtNodeCostPerRAMGiBHr, durStr, offStr)
+	resChNodeCostPerRAMGiBHr := ctx.Query(queryNodeCostPerRAMGiBHr)
+
+	queryNodeCostPerGPUHr := fmt.Sprintf(queryFmtNodeCostPerGPUHr, durStr, offStr)
+	resChNodeCostPerGPUHr := ctx.Query(queryNodeCostPerGPUHr)
+
+	queryNodeIsSpot := fmt.Sprintf(queryFmtNodeIsSpot, durStr, offStr)
+	resChNodeIsSpot := ctx.Query(queryNodeIsSpot)
+
+	queryPVCInfo := fmt.Sprintf(queryFmtPVCInfo, durStr, resStr, offStr)
+	resChPVCInfo := ctx.Query(queryPVCInfo)
+
+	queryPVBytes := fmt.Sprintf(queryFmtPVBytes, durStr, offStr)
+	resChPVBytes := ctx.Query(queryPVBytes)
+
+	queryPodPVCAllocation := fmt.Sprintf(queryFmtPodPVCAllocation, durStr, offStr)
+	resChPodPVCAllocation := ctx.Query(queryPodPVCAllocation)
+
+	queryPVCBytesRequested := fmt.Sprintf(queryFmtPVCBytesRequested, durStr, offStr)
+	resChPVCBytesRequested := ctx.Query(queryPVCBytesRequested)
+
+	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, durStr, offStr)
+	resChPVCostPerGiBHour := ctx.Query(queryPVCostPerGiBHour)
+
+	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, durStr, offStr)
+	resChNetZoneGiB := ctx.Query(queryNetZoneGiB)
+
+	queryNetZoneCostPerGiB := fmt.Sprintf(queryFmtNetZoneCostPerGiB, durStr, offStr)
+	resChNetZoneCostPerGiB := ctx.Query(queryNetZoneCostPerGiB)
+
+	queryNetRegionGiB := fmt.Sprintf(queryFmtNetRegionGiB, durStr, offStr)
+	resChNetRegionGiB := ctx.Query(queryNetRegionGiB)
+
+	queryNetRegionCostPerGiB := fmt.Sprintf(queryFmtNetRegionCostPerGiB, durStr, offStr)
+	resChNetRegionCostPerGiB := ctx.Query(queryNetRegionCostPerGiB)
+
+	queryNetInternetGiB := fmt.Sprintf(queryFmtNetInternetGiB, durStr, offStr)
+	resChNetInternetGiB := ctx.Query(queryNetInternetGiB)
+
+	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, durStr, offStr)
+	resChNetInternetCostPerGiB := ctx.Query(queryNetInternetCostPerGiB)
+
+	queryNamespaceLabels := fmt.Sprintf(queryFmtNamespaceLabels, durStr, offStr)
+	resChNamespaceLabels := ctx.Query(queryNamespaceLabels)
+
+	queryNamespaceAnnotations := fmt.Sprintf(queryFmtNamespaceAnnotations, durStr, offStr)
+	resChNamespaceAnnotations := ctx.Query(queryNamespaceAnnotations)
+
+	queryPodLabels := fmt.Sprintf(queryFmtPodLabels, durStr, offStr)
+	resChPodLabels := ctx.Query(queryPodLabels)
+
+	queryPodAnnotations := fmt.Sprintf(queryFmtPodAnnotations, durStr, offStr)
+	resChPodAnnotations := ctx.Query(queryPodAnnotations)
+
+	queryServiceLabels := fmt.Sprintf(queryFmtServiceLabels, durStr, offStr)
+	resChServiceLabels := ctx.Query(queryServiceLabels)
+
+	queryDeploymentLabels := fmt.Sprintf(queryFmtDeploymentLabels, durStr, offStr)
+	resChDeploymentLabels := ctx.Query(queryDeploymentLabels)
+
+	queryStatefulSetLabels := fmt.Sprintf(queryFmtStatefulSetLabels, durStr, offStr)
+	resChStatefulSetLabels := ctx.Query(queryStatefulSetLabels)
+
+	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, durStr, offStr)
+	resChDaemonSetLabels := ctx.Query(queryDaemonSetLabels)
+
+	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, durStr, offStr)
+	resChJobLabels := ctx.Query(queryJobLabels)
+
+	resCPUCoresAllocated, _ := resChCPUCoresAllocated.Await()
+	resCPURequests, _ := resChCPURequests.Await()
+	resCPUUsage, _ := resChCPUUsage.Await()
+	resRAMBytesAllocated, _ := resChRAMBytesAllocated.Await()
+	resRAMRequests, _ := resChRAMRequests.Await()
+	resRAMUsage, _ := resChRAMUsage.Await()
+	resGPUsRequested, _ := resChGPUsRequested.Await()
+
+	resNodeCostPerCPUHr, _ := resChNodeCostPerCPUHr.Await()
+	resNodeCostPerRAMGiBHr, _ := resChNodeCostPerRAMGiBHr.Await()
+	resNodeCostPerGPUHr, _ := resChNodeCostPerGPUHr.Await()
+	resNodeIsSpot, _ := resChNodeIsSpot.Await()
+
+	resPVBytes, _ := resChPVBytes.Await()
+	resPVCostPerGiBHour, _ := resChPVCostPerGiBHour.Await()
+
+	resPVCInfo, _ := resChPVCInfo.Await()
+	resPVCBytesRequested, _ := resChPVCBytesRequested.Await()
+	resPodPVCAllocation, _ := resChPodPVCAllocation.Await()
+
+	resNetZoneGiB, _ := resChNetZoneGiB.Await()
+	resNetZoneCostPerGiB, _ := resChNetZoneCostPerGiB.Await()
+	resNetRegionGiB, _ := resChNetRegionGiB.Await()
+	resNetRegionCostPerGiB, _ := resChNetRegionCostPerGiB.Await()
+	resNetInternetGiB, _ := resChNetInternetGiB.Await()
+	resNetInternetCostPerGiB, _ := resChNetInternetCostPerGiB.Await()
+
+	resNamespaceLabels, _ := resChNamespaceLabels.Await()
+	resNamespaceAnnotations, _ := resChNamespaceAnnotations.Await()
+	resPodLabels, _ := resChPodLabels.Await()
+	resPodAnnotations, _ := resChPodAnnotations.Await()
+	resServiceLabels, _ := resChServiceLabels.Await()
+	resDeploymentLabels, _ := resChDeploymentLabels.Await()
+	resStatefulSetLabels, _ := resChStatefulSetLabels.Await()
+	resDaemonSetLabels, _ := resChDaemonSetLabels.Await()
+	resJobLabels, _ := resChJobLabels.Await()
+
+	if ctx.HasErrors() {
+		for _, err := range ctx.Errors() {
+			log.Errorf("CostModel.ComputeAllocation: %s", err)
+		}
+
+		return allocSet, ctx.ErrorCollection()
+	}
+
+	// We choose to apply allocation before requests in the cases of RAM and
+	// CPU so that we can assert that allocation should always be greater than
+	// or equal to request.
+	applyCPUCoresAllocated(podMap, resCPUCoresAllocated)
+	applyCPUCoresRequested(podMap, resCPURequests)
+	applyCPUCoresUsed(podMap, resCPUUsage)
+	applyRAMBytesAllocated(podMap, resRAMBytesAllocated)
+	applyRAMBytesRequested(podMap, resRAMRequests)
+	applyRAMBytesUsed(podMap, resRAMUsage)
+	applyGPUsRequested(podMap, resGPUsRequested)
+	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB)
+	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB)
+	applyNetworkAllocation(podMap, resNetInternetGiB, resNetInternetCostPerGiB)
+
+	namespaceLabels := resToNamespaceLabels(resNamespaceLabels)
+	podLabels := resToPodLabels(resPodLabels)
+	namespaceAnnotations := resToNamespaceAnnotations(resNamespaceAnnotations)
+	podAnnotations := resToPodAnnotations(resPodAnnotations)
+	applyLabels(podMap, namespaceLabels, podLabels)
+	applyAnnotations(podMap, namespaceAnnotations, podAnnotations)
+
+	serviceLabels := getServiceLabels(resServiceLabels)
+	applyServicesToPods(podMap, podLabels, serviceLabels)
+
+	podDeploymentMap := labelsToPodControllerMap(podLabels, resToDeploymentLabels(resDeploymentLabels))
+	podStatefulSetMap := labelsToPodControllerMap(podLabels, resToStatefulSetLabels(resStatefulSetLabels))
+	podDaemonSetMap := resToPodDaemonSetMap(resDaemonSetLabels)
+	podJobMap := resToPodJobMap(resJobLabels)
+	applyControllersToPods(podMap, podDeploymentMap)
+	applyControllersToPods(podMap, podStatefulSetMap)
+	applyControllersToPods(podMap, podDaemonSetMap)
+	applyControllersToPods(podMap, podJobMap)
+
+	// TODO breakdown network costs?
+
+	// Build out a map of Nodes with resource costs, discounts, and node types
+	// for converting resource allocation data to cumulative costs.
+	nodeMap := map[nodeKey]*NodePricing{}
+
+	applyNodeCostPerCPUHr(nodeMap, resNodeCostPerCPUHr)
+	applyNodeCostPerRAMGiBHr(nodeMap, resNodeCostPerRAMGiBHr)
+	applyNodeCostPerGPUHr(nodeMap, resNodeCostPerGPUHr)
+	applyNodeSpot(nodeMap, resNodeIsSpot)
+	applyNodeDiscount(nodeMap, cm)
+
+	// Build out the map of all PVs with class, size and cost-per-hour.
+	// Note: this does not record time running, which we may want to
+	// include later for increased PV precision. (As long as the PV has
+	// a PVC, we get time running there, so this is only inaccurate
+	// for short-lived, unmounted PVs.)
+	pvMap := map[pvKey]*PV{}
+	buildPVMap(pvMap, resPVCostPerGiBHour)
+	applyPVBytes(pvMap, resPVBytes)
+
+	// Build out the map of all PVCs with time running, bytes requested,
+	// and connect to the correct PV from pvMap. (If no PV exists, that
+	// is noted, but does not result in any allocation/cost.)
+	pvcMap := map[pvcKey]*PVC{}
+	buildPVCMap(window, pvcMap, pvMap, resPVCInfo)
+	applyPVCBytesRequested(pvcMap, resPVCBytesRequested)
+
+	// Build out the relationships of pods to their PVCs. This step
+	// populates the PVC.Count field so that PVC allocation can be
+	// split appropriately among each pod's container allocation.
+	podPVCMap := map[podKey][]*PVC{}
+	buildPodPVCMap(podPVCMap, pvMap, pvcMap, podMap, resPodPVCAllocation)
+
+	// Identify unmounted PVs (PVs without PVCs) and add one Allocation per
+	// cluster representing each cluster's unmounted PVs (if necessary).
+	applyUnmountedPVs(window, podMap, pvMap, pvcMap)
+
+	// (3) Build out AllocationSet from Pod map
+
+	for _, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			cluster, _ := alloc.Properties.GetCluster()
+			nodeName, _ := alloc.Properties.GetNode()
+			namespace, _ := alloc.Properties.GetNamespace()
+			pod, _ := alloc.Properties.GetPod()
+			container, _ := alloc.Properties.GetContainer()
+
+			podKey := newPodKey(cluster, namespace, pod)
+			nodeKey := newNodeKey(cluster, nodeName)
+
+			node := cm.getNodePricing(nodeMap, nodeKey)
+			alloc.CPUCost = alloc.CPUCoreHours * node.CostPerCPUHr
+			alloc.RAMCost = (alloc.RAMByteHours / 1024 / 1024 / 1024) * node.CostPerRAMGiBHr
+			alloc.GPUCost = alloc.GPUHours * node.CostPerGPUHr
+
+			if pvcs, ok := podPVCMap[podKey]; ok {
+				for _, pvc := range pvcs {
+					// Determine the (start, end) of the relationship between the
+					// given PVC and the associated Allocation so that a precise
+					// number of hours can be used to compute cumulative cost.
+					s, e := alloc.Start, alloc.End
+					if pvc.Start.After(alloc.Start) {
+						s = pvc.Start
+					}
+					if pvc.End.Before(alloc.End) {
+						e = pvc.End
+					}
+					minutes := e.Sub(s).Minutes()
+					hrs := minutes / 60.0
+
+					count := float64(pvc.Count)
+					if pvc.Count < 1 {
+						count = 1
+					}
+
+					gib := pvc.Bytes / 1024 / 1024 / 1024
+					cost := pvc.Volume.CostPerGiBHour * gib * hrs
+
+					// Apply the size and cost of the PV to the allocation, each
+					// weighted by count (i.e. the number of containers in the pod)
+					alloc.PVByteHours += pvc.Bytes * hrs / count
+					alloc.PVCost += cost / count
+				}
+			}
+
+			// Make sure that the name is correct (node may not be present at this
+			// point due to it missing from queryMinutes) then insert.
+			alloc.Name = fmt.Sprintf("%s/%s/%s/%s/%s", cluster, nodeName, namespace, pod, container)
+			allocSet.Set(alloc)
+		}
+	}
+
+	return allocSet, nil
+}
+
+func (cm *CostModel) buildPodMap(window kubecost.Window, resolution, maxBatchSize time.Duration, podMap map[podKey]*Pod, clusterStart, clusterEnd map[string]time.Time) error {
+	// Assumes that window is positive and closed
+	start, end := *window.Start(), *window.End()
+
+	// Convert resolution duration to a query-ready string
+	resStr := util.DurationString(resolution)
+
+	ctx := prom.NewContext(cm.PrometheusClient)
+
+	// Query for (start, end) by (pod, namespace, cluster) over the given
+	// window, using the given resolution, and if necessary in batches no
+	// larger than the given maximum batch size. If working in batches, track
+	// overall progress by starting with (window.start, window.start) and
+	// querying in batches no larger than maxBatchSize from start-to-end,
+	// folding each result set into podMap as the results come back.
+	coverage := kubecost.NewWindow(&start, &start)
+
+	numQuery := 1
+	for coverage.End().Before(end) {
+		// Determine the (start, end) of the current batch
+		batchStart := *coverage.End()
+		batchEnd := coverage.End().Add(maxBatchSize)
+		if batchEnd.After(end) {
+			batchEnd = end
+		}
+		batchWindow := kubecost.NewWindow(&batchStart, &batchEnd)
+
+		var resPods []*prom.QueryResult
+		var err error
+		maxTries := 3
+		numTries := 0
+		for resPods == nil && numTries < maxTries {
+			numTries++
+
+			// Convert window (start, end) to (duration, offset) for querying Prometheus,
+			// including handling Thanos offset
+			durStr, offStr, err := batchWindow.DurationOffsetForPrometheus()
+			if err != nil || durStr == "" {
+				// Negative duration, so set empty results and don't query
+				resPods = []*prom.QueryResult{}
+				err = nil
+				break
+			}
+
+			// Submit and profile query
+			queryPods := fmt.Sprintf(queryFmtPods, durStr, resStr, offStr)
+			queryProfile := time.Now()
+			resPods, err = ctx.Query(queryPods).Await()
+			if err != nil {
+				log.Profile(queryProfile, fmt.Sprintf("CostModel.ComputeAllocation: pod query %d try %d failed: %s", numQuery, numTries, queryPods))
+				resPods = nil
+			}
+		}
+
+		if err != nil {
+			return err
+		}
+
+		applyPodResults(window, resolution, podMap, clusterStart, clusterEnd, resPods)
+
+		coverage = coverage.ExpandEnd(batchEnd)
+		numQuery++
+	}
+
+	return nil
+}
+
+func applyPodResults(window kubecost.Window, resolution time.Duration, podMap map[podKey]*Pod, clusterStart, clusterEnd map[string]time.Time, resPods []*prom.QueryResult) {
+	for _, res := range resPods {
+		if len(res.Values) == 0 {
+			log.Warningf("CostModel.ComputeAllocation: empty minutes result")
+			continue
+		}
+
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		labels, err := res.GetStrings("namespace", "pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: minutes query result missing field: %s", err)
+			continue
+		}
+
+		namespace := labels["namespace"]
+		pod := labels["pod"]
+		key := newPodKey(cluster, namespace, pod)
+
+		// allocStart and allocEnd are the timestamps of the first and last
+		// minutes the pod was running, respectively. We subtract one resolution
+		// from allocStart because this point will actually represent the end
+		// of the first minute. We don't subtract from allocEnd because it
+		// already represents the end of the last minute.
+		var allocStart, allocEnd time.Time
+		startAdjustmentCoeff, endAdjustmentCoeff := 1.0, 1.0
+		for _, datum := range res.Values {
+			t := time.Unix(int64(datum.Timestamp), 0)
+
+			if allocStart.IsZero() && datum.Value > 0 && window.Contains(t) {
+				// Set the start timestamp to the earliest non-zero timestamp
+				allocStart = t
+
+				// Record adjustment coefficient, i.e. the portion of the start
+				// timestamp to "ignore". That is, sometimes the value will be
+				// 0.5, meaning that we should discount the time running by
+				// half of the resolution the timestamp stands for.
+				startAdjustmentCoeff = (1.0 - datum.Value)
+			}
+
+			if datum.Value > 0 && window.Contains(t) {
+				// Set the end timestamp to the latest non-zero timestamp
+				allocEnd = t
+
+				// Record adjustment coefficient, i.e. the portion of the end
+				// timestamp to "ignore". (See explanation above for start.)
+				endAdjustmentCoeff = (1.0 - datum.Value)
+			}
+		}
+
+		if allocStart.IsZero() || allocEnd.IsZero() {
+			continue
+		}
+
+		// Adjust timestamps according to the resolution and the adjustment
+		// coefficients, as described above. That is, count the start timestamp
+		// from the beginning of the resolution, not the end. Then "reduce" the
+		// start and end by the correct amount, in the case that the "running"
+		// value of the first or last timestamp was not a full 1.0.
+		allocStart = allocStart.Add(-resolution)
+		// Note: the *100 and /100 are necessary because Duration is an int, so
+		// 0.5, for instance, will be truncated, resulting in no adjustment.
+		allocStart = allocStart.Add(time.Duration(startAdjustmentCoeff*100) * resolution / time.Duration(100))
+		allocEnd = allocEnd.Add(-time.Duration(endAdjustmentCoeff*100) * resolution / time.Duration(100))
+
+		// If there is only one point with a value <= 0.5 that the start and
+		// end timestamps both share, then we will enter this case because at
+		// least half of a resolution will be subtracted from both the start
+		// and the end. If that is the case, then add back half of each side
+		// so that the pod is said to run for half a resolution total.
+		// e.g. For resolution 1m and a value of 0.5 at one timestamp, we'll
+		//      end up with allocEnd == allocStart and each coeff == 0.5. In
+		//      that case, add 0.25m to each side, resulting in 0.5m duration.
+		if !allocEnd.After(allocStart) {
+			allocStart = allocStart.Add(-time.Duration(50*startAdjustmentCoeff) * resolution / time.Duration(100))
+			allocEnd = allocEnd.Add(time.Duration(50*endAdjustmentCoeff) * resolution / time.Duration(100))
+		}
+
+		// Set start if unset or this datum's start time is earlier than the
+		// current earliest time.
+		if _, ok := clusterStart[cluster]; !ok || allocStart.Before(clusterStart[cluster]) {
+			clusterStart[cluster] = allocStart
+		}
+
+		// Set end if unset or this datum's end time is later than the
+		// current latest time.
+		if _, ok := clusterEnd[cluster]; !ok || allocEnd.After(clusterEnd[cluster]) {
+			clusterEnd[cluster] = allocEnd
+		}
+
+		if pod, ok := podMap[key]; ok {
+			// Pod has already been recorded, so update it accordingly
+			if allocStart.Before(pod.Start) {
+				pod.Start = allocStart
+			}
+			if allocEnd.After(pod.End) {
+				pod.End = allocEnd
+			}
+		} else {
+			// Pod has not been recorded yet, so insert it
+			podMap[key] = &Pod{
+				Window:      window.Clone(),
+				Start:       allocStart,
+				End:         allocEnd,
+				Key:         key,
+				Allocations: map[string]*kubecost.Allocation{},
+			}
+		}
+	}
+}
+
+func applyCPUCoresAllocated(podMap map[podKey]*Pod, resCPUCoresAllocated []*prom.QueryResult) {
+	for _, res := range resCPUCoresAllocated {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU allocation result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU allocation query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		cpuCores := res.Values[0].Value
+		hours := pod.Allocations[container].Minutes() / 60.0
+		pod.Allocations[container].CPUCoreHours = cpuCores * hours
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: CPU allocation query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyCPUCoresRequested(podMap map[podKey]*Pod, resCPUCoresRequested []*prom.QueryResult) {
+	for _, res := range resCPUCoresRequested {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU request result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU request query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].CPUCoreRequestAverage = res.Values[0].Value
+
+		// If CPU allocation is less than requests, set CPUCoreHours to
+		// request level.
+		if pod.Allocations[container].CPUCores() < res.Values[0].Value {
+			pod.Allocations[container].CPUCoreHours = res.Values[0].Value * (pod.Allocations[container].Minutes() / 60.0)
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: CPU request query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyCPUCoresUsed(podMap map[podKey]*Pod, resCPUCoresUsed []*prom.QueryResult) {
+	for _, res := range resCPUCoresUsed {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].CPUCoreUsageAverage = res.Values[0].Value
+	}
+}
+
+func applyRAMBytesAllocated(podMap map[podKey]*Pod, resRAMBytesAllocated []*prom.QueryResult) {
+	for _, res := range resRAMBytesAllocated {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM allocation result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM allocation query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		ramBytes := res.Values[0].Value
+		hours := pod.Allocations[container].Minutes() / 60.0
+		pod.Allocations[container].RAMByteHours = ramBytes * hours
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: RAM allocation query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyRAMBytesRequested(podMap map[podKey]*Pod, resRAMBytesRequested []*prom.QueryResult) {
+	for _, res := range resRAMBytesRequested {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM request result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM request query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].RAMBytesRequestAverage = res.Values[0].Value
+
+		// If RAM allocation is less than requests, set RAMByteHours to
+		// request level.
+		if pod.Allocations[container].RAMBytes() < res.Values[0].Value {
+			pod.Allocations[container].RAMByteHours = res.Values[0].Value * (pod.Allocations[container].Minutes() / 60.0)
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: RAM request query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyRAMBytesUsed(podMap map[podKey]*Pod, resRAMBytesUsed []*prom.QueryResult) {
+	for _, res := range resRAMBytesUsed {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].RAMBytesUsageAverage = res.Values[0].Value
+	}
+}
+
+func applyGPUsRequested(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult) {
+	for _, res := range resGPUsRequested {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU request result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU request query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		hrs := pod.Allocations[container].Minutes() / 60.0
+		pod.Allocations[container].GPUHours = res.Values[0].Value * hrs
+	}
+}
+
+func applyNetworkAllocation(podMap map[podKey]*Pod, resNetworkGiB []*prom.QueryResult, resNetworkCostPerGiB []*prom.QueryResult) {
+	costPerGiBByCluster := map[string]float64{}
+
+	for _, res := range resNetworkCostPerGiB {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		costPerGiBByCluster[cluster] = res.Values[0].Value
+	}
+
+	for _, res := range resNetworkGiB {
+		podKey, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: Network allocation query result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[podKey]
+		if !ok {
+			continue
+		}
+
+		for _, alloc := range pod.Allocations {
+			gib := res.Values[0].Value / float64(len(pod.Allocations))
+			costPerGiB := costPerGiBByCluster[podKey.Cluster]
+			alloc.NetworkCost = gib * costPerGiB
+		}
+	}
+}
+
+func resToNamespaceLabels(resNamespaceLabels []*prom.QueryResult) map[string]map[string]string {
+	namespaceLabels := map[string]map[string]string{}
+
+	for _, res := range resNamespaceLabels {
+		namespace, err := res.GetString("namespace")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := namespaceLabels[namespace]; !ok {
+			namespaceLabels[namespace] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			namespaceLabels[namespace][k] = l
+		}
+	}
+
+	return namespaceLabels
+}
+
+func resToPodLabels(resPodLabels []*prom.QueryResult) map[podKey]map[string]string {
+	podLabels := map[podKey]map[string]string{}
+
+	for _, res := range resPodLabels {
+		podKey, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := podLabels[podKey]; !ok {
+			podLabels[podKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			podLabels[podKey][k] = l
+		}
+	}
+
+	return podLabels
+}
+
+func resToNamespaceAnnotations(resNamespaceAnnotations []*prom.QueryResult) map[string]map[string]string {
+	namespaceAnnotations := map[string]map[string]string{}
+
+	for _, res := range resNamespaceAnnotations {
+		namespace, err := res.GetString("namespace")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := namespaceAnnotations[namespace]; !ok {
+			namespaceAnnotations[namespace] = map[string]string{}
+		}
+
+		for k, l := range res.GetAnnotations() {
+			namespaceAnnotations[namespace][k] = l
+		}
+	}
+
+	return namespaceAnnotations
+}
+
+func resToPodAnnotations(resPodAnnotations []*prom.QueryResult) map[podKey]map[string]string {
+	podAnnotations := map[podKey]map[string]string{}
+
+	for _, res := range resPodAnnotations {
+		podKey, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := podAnnotations[podKey]; !ok {
+			podAnnotations[podKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetAnnotations() {
+			podAnnotations[podKey][k] = l
+		}
+	}
+
+	return podAnnotations
+}
+
+func applyLabels(podMap map[podKey]*Pod, namespaceLabels map[string]map[string]string, podLabels map[podKey]map[string]string) {
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			allocLabels, err := alloc.Properties.GetLabels()
+			if err != nil {
+				allocLabels = map[string]string{}
+			}
+
+			// Apply namespace labels first, then pod labels so that pod labels
+			// overwrite namespace labels.
+			if labels, ok := namespaceLabels[key.Namespace]; ok {
+				for k, v := range labels {
+					allocLabels[k] = v
+				}
+			}
+			if labels, ok := podLabels[key]; ok {
+				for k, v := range labels {
+					allocLabels[k] = v
+				}
+			}
+
+			alloc.Properties.SetLabels(allocLabels)
+		}
+	}
+}
+
+func applyAnnotations(podMap map[podKey]*Pod, namespaceAnnotations map[string]map[string]string, podAnnotations map[podKey]map[string]string) {
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			allocAnnotations, err := alloc.Properties.GetAnnotations()
+			if err != nil {
+				allocAnnotations = map[string]string{}
+			}
+
+			// Apply namespace annotations first, then pod annotations so that
+			// pod labels overwrite namespace labels.
+			if labels, ok := namespaceAnnotations[key.Namespace]; ok {
+				for k, v := range labels {
+					allocAnnotations[k] = v
+				}
+			}
+			if labels, ok := podAnnotations[key]; ok {
+				for k, v := range labels {
+					allocAnnotations[k] = v
+				}
+			}
+
+			alloc.Properties.SetAnnotations(allocAnnotations)
+		}
+	}
+}
+
+func getServiceLabels(resServiceLabels []*prom.QueryResult) map[serviceKey]map[string]string {
+	serviceLabels := map[serviceKey]map[string]string{}
+
+	for _, res := range resServiceLabels {
+		serviceKey, err := resultServiceKey(res, "cluster_id", "namespace", "service")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := serviceLabels[serviceKey]; !ok {
+			serviceLabels[serviceKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			serviceLabels[serviceKey][k] = l
+		}
+	}
+
+	// Prune duplicate services. That is, if the same service exists with
+	// hyphens instead of underscores, keep the one that uses hyphens.
+	for key := range serviceLabels {
+		duplicateService := strings.Replace(key.Service, "_", "-", -1)
+		duplicateKey := newServiceKey(key.Cluster, key.Namespace, duplicateService)
+		if _, ok := serviceLabels[duplicateKey]; ok {
+			delete(serviceLabels, key)
+		}
+	}
+
+	return serviceLabels
+}
+
+func resToDeploymentLabels(resDeploymentLabels []*prom.QueryResult) map[controllerKey]map[string]string {
+	deploymentLabels := map[controllerKey]map[string]string{}
+
+	for _, res := range resDeploymentLabels {
+		controllerKey, err := resultDeploymentKey(res, "cluster_id", "namespace", "deployment")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := deploymentLabels[controllerKey]; !ok {
+			deploymentLabels[controllerKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			deploymentLabels[controllerKey][k] = l
+		}
+	}
+
+	// Prune duplicate deployments. That is, if the same deployment exists with
+	// hyphens instead of underscores, keep the one that uses hyphens.
+	for key := range deploymentLabels {
+		duplicateController := strings.Replace(key.Controller, "_", "-", -1)
+		duplicateKey := newControllerKey(key.Cluster, key.Namespace, key.ControllerKind, duplicateController)
+		if _, ok := deploymentLabels[duplicateKey]; ok {
+			delete(deploymentLabels, key)
+		}
+	}
+
+	return deploymentLabels
+}
+
+func resToStatefulSetLabels(resStatefulSetLabels []*prom.QueryResult) map[controllerKey]map[string]string {
+	statefulSetLabels := map[controllerKey]map[string]string{}
+
+	for _, res := range resStatefulSetLabels {
+		controllerKey, err := resultStatefulSetKey(res, "cluster_id", "namespace", "statefulSet")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := statefulSetLabels[controllerKey]; !ok {
+			statefulSetLabels[controllerKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			statefulSetLabels[controllerKey][k] = l
+		}
+	}
+
+	// Prune duplicate stateful sets. That is, if the same stateful set exists
+	// with hyphens instead of underscores, keep the one that uses hyphens.
+	for key := range statefulSetLabels {
+		duplicateController := strings.Replace(key.Controller, "_", "-", -1)
+		duplicateKey := newControllerKey(key.Cluster, key.Namespace, key.ControllerKind, duplicateController)
+		if _, ok := statefulSetLabels[duplicateKey]; ok {
+			delete(statefulSetLabels, key)
+		}
+	}
+
+	return statefulSetLabels
+}
+
+func labelsToPodControllerMap(podLabels map[podKey]map[string]string, controllerLabels map[controllerKey]map[string]string) map[podKey]controllerKey {
+	podControllerMap := map[podKey]controllerKey{}
+
+	// For each controller, turn the labels into a selector and attempt to
+	// match it with each set of pod labels. A match indicates that the pod
+	// belongs to the controller.
+	for cKey, cLabels := range controllerLabels {
+		selector := labels.Set(cLabels).AsSelectorPreValidated()
+
+		for pKey, pLabels := range podLabels {
+			// If the pod is in a different cluster or namespace, there is
+			// no need to compare the labels.
+			if cKey.Cluster != pKey.Cluster || cKey.Namespace != pKey.Namespace {
+				continue
+			}
+
+			podLabelSet := labels.Set(pLabels)
+			if selector.Matches(podLabelSet) {
+				if _, ok := podControllerMap[pKey]; ok {
+					log.Warningf("CostModel.ComputeAllocation: PodControllerMap match already exists: %s matches %s and %s", pKey, podControllerMap[pKey], cKey)
+				}
+				podControllerMap[pKey] = cKey
+			}
+		}
+	}
+
+	return podControllerMap
+}
+
+func resToPodDaemonSetMap(resDaemonSetLabels []*prom.QueryResult) map[podKey]controllerKey {
+	daemonSetLabels := map[podKey]controllerKey{}
+
+	for _, res := range resDaemonSetLabels {
+		controllerKey, err := resultDaemonSetKey(res, "cluster_id", "namespace", "owner_name")
+		if err != nil {
+			continue
+		}
+
+		pod, err := res.GetString("pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: DaemonSetLabel result without pod: %s", controllerKey)
+		}
+
+		podKey := newPodKey(controllerKey.Cluster, controllerKey.Namespace, pod)
+
+		daemonSetLabels[podKey] = controllerKey
+	}
+
+	return daemonSetLabels
+}
+
+func resToPodJobMap(resJobLabels []*prom.QueryResult) map[podKey]controllerKey {
+	jobLabels := map[podKey]controllerKey{}
+
+	for _, res := range resJobLabels {
+		controllerKey, err := resultJobKey(res, "cluster_id", "namespace", "owner_name")
+		if err != nil {
+			continue
+		}
+
+		// Convert the name of Jobs generated by CronJobs to the name of the
+		// CronJob by stripping the timestamp off the end.
+		match := isCron.FindStringSubmatch(controllerKey.Controller)
+		if match != nil {
+			controllerKey.Controller = match[1]
+		}
+
+		pod, err := res.GetString("pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: JobLabel result without pod: %s", controllerKey)
+		}
+
+		podKey := newPodKey(controllerKey.Cluster, controllerKey.Namespace, pod)
+
+		jobLabels[podKey] = controllerKey
+	}
+
+	return jobLabels
+}
+
+func applyServicesToPods(podMap map[podKey]*Pod, podLabels map[podKey]map[string]string, serviceLabels map[serviceKey]map[string]string) {
+	podServicesMap := map[podKey][]serviceKey{}
+
+	// For each service, turn the labels into a selector and attempt to
+	// match it with each set of pod labels. A match indicates that the pod
+	// belongs to the service.
+	for sKey, sLabels := range serviceLabels {
+		selector := labels.Set(sLabels).AsSelectorPreValidated()
+
+		for pKey, pLabels := range podLabels {
+			// If the pod is in a different cluster or namespace, there is
+			// no need to compare the labels.
+			if sKey.Cluster != pKey.Cluster || sKey.Namespace != pKey.Namespace {
+				continue
+			}
+
+			podLabelSet := labels.Set(pLabels)
+			if selector.Matches(podLabelSet) {
+				if _, ok := podServicesMap[pKey]; !ok {
+					podServicesMap[pKey] = []serviceKey{}
+				}
+				podServicesMap[pKey] = append(podServicesMap[pKey], sKey)
+			}
+		}
+	}
+
+	// For each allocation in each pod, attempt to find and apply the list of
+	// services associated with the allocation's pod.
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			if sKeys, ok := podServicesMap[key]; ok {
+				services := []string{}
+				for _, sKey := range sKeys {
+					services = append(services, sKey.Service)
+				}
+				alloc.Properties.SetServices(services)
+			}
+		}
+	}
+}
+
+func applyControllersToPods(podMap map[podKey]*Pod, podControllerMap map[podKey]controllerKey) {
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			if controllerKey, ok := podControllerMap[key]; ok {
+				alloc.Properties.SetControllerKind(controllerKey.ControllerKind)
+				alloc.Properties.SetController(controllerKey.Controller)
+			}
+		}
+	}
+}
+
+func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr []*prom.QueryResult) {
+	for _, res := range resNodeCostPerCPUHr {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node CPU cost query result missing field: %s", err)
+			continue
+		}
+
+		instanceType, err := res.GetString("instance_type")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node CPU cost query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			nodeMap[key] = &NodePricing{
+				Name:     node,
+				NodeType: instanceType,
+			}
+		}
+
+		nodeMap[key].CostPerCPUHr = res.Values[0].Value
+	}
+}
+
+func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRAMGiBHr []*prom.QueryResult) {
+	for _, res := range resNodeCostPerRAMGiBHr {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node RAM cost query result missing field: %s", err)
+			continue
+		}
+
+		instanceType, err := res.GetString("instance_type")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node RAM cost query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			nodeMap[key] = &NodePricing{
+				Name:     node,
+				NodeType: instanceType,
+			}
+		}
+
+		nodeMap[key].CostPerRAMGiBHr = res.Values[0].Value
+	}
+}
+
+func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr []*prom.QueryResult) {
+	for _, res := range resNodeCostPerGPUHr {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node GPU cost query result missing field: %s", err)
+			continue
+		}
+
+		instanceType, err := res.GetString("instance_type")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node GPU cost query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			nodeMap[key] = &NodePricing{
+				Name:     node,
+				NodeType: instanceType,
+			}
+		}
+
+		nodeMap[key].CostPerGPUHr = res.Values[0].Value
+	}
+}
+
+func applyNodeSpot(nodeMap map[nodeKey]*NodePricing, resNodeIsSpot []*prom.QueryResult) {
+	for _, res := range resNodeIsSpot {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node spot query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: Node spot  query result for missing node: %s", key)
+			continue
+		}
+
+		nodeMap[key].Preemptible = res.Values[0].Value > 0
+	}
+}
+
+func applyNodeDiscount(nodeMap map[nodeKey]*NodePricing, cm *CostModel) {
+	if cm == nil {
+		return
+	}
+
+	c, err := cm.Provider.GetConfig()
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: applyNodeDiscount: %s", err)
+		return
+	}
+
+	discount, err := ParsePercentString(c.Discount)
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: applyNodeDiscount: %s", err)
+		return
+	}
+
+	negotiatedDiscount, err := ParsePercentString(c.NegotiatedDiscount)
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: applyNodeDiscount: %s", err)
+		return
+	}
+
+	for _, node := range nodeMap {
+		// TODO GKE Reserved Instances into account
+		node.Discount = cm.Provider.CombinedDiscountForNode(node.NodeType, node.Preemptible, discount, negotiatedDiscount)
+		node.CostPerCPUHr *= (1.0 - node.Discount)
+		node.CostPerRAMGiBHr *= (1.0 - node.Discount)
+	}
+}
+
+func buildPVMap(pvMap map[pvKey]*PV, resPVCostPerGiBHour []*prom.QueryResult) {
+	for _, res := range resPVCostPerGiBHour {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		name, err := res.GetString("volumename")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PV cost without volumename")
+			continue
+		}
+
+		key := newPVKey(cluster, name)
+
+		pvMap[key] = &PV{
+			Cluster:        cluster,
+			Name:           name,
+			CostPerGiBHour: res.Values[0].Value,
+		}
+	}
+}
+
+func applyPVBytes(pvMap map[pvKey]*PV, resPVBytes []*prom.QueryResult) {
+	for _, res := range resPVBytes {
+		key, err := resultPVKey(res, "cluster_id", "persistentvolume")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PV bytes query result missing field: %s", err)
+			continue
+		}
+
+		if _, ok := pvMap[key]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: PV bytes result for missing PV: %s", err)
+			continue
+		}
+
+		pvMap[key].Bytes = res.Values[0].Value
+	}
+}
+
+func buildPVCMap(window kubecost.Window, pvcMap map[pvcKey]*PVC, pvMap map[pvKey]*PV, resPVCInfo []*prom.QueryResult) {
+	for _, res := range resPVCInfo {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		values, err := res.GetStrings("persistentvolumeclaim", "storageclass", "volumename", "namespace")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: PVC info query result missing field: %s", err)
+			continue
+		}
+
+		namespace := values["namespace"]
+		name := values["persistentvolumeclaim"]
+		volume := values["volumename"]
+		storageClass := values["storageclass"]
+
+		pvKey := newPVKey(cluster, volume)
+		pvcKey := newPVCKey(cluster, namespace, name)
+
+		// pvcStart and pvcEnd are the timestamps of the first and last minutes
+		// the PVC was running, respectively. We subtract 1m from pvcStart
+		// because this point will actually represent the end of the first
+		// minute. We don't subtract from pvcEnd because it already represents
+		// the end of the last minute.
+		var pvcStart, pvcEnd time.Time
+		for _, datum := range res.Values {
+			t := time.Unix(int64(datum.Timestamp), 0)
+			if pvcStart.IsZero() && datum.Value > 0 && window.Contains(t) {
+				pvcStart = t
+			}
+			if datum.Value > 0 && window.Contains(t) {
+				pvcEnd = t
+			}
+		}
+		if pvcStart.IsZero() || pvcEnd.IsZero() {
+			log.Warningf("CostModel.ComputeAllocation: PVC %s has no running time", pvcKey)
+		}
+		pvcStart = pvcStart.Add(-time.Minute)
+
+		if _, ok := pvMap[pvKey]; !ok {
+			continue
+		}
+
+		pvMap[pvKey].StorageClass = storageClass
+
+		if _, ok := pvcMap[pvcKey]; !ok {
+			pvcMap[pvcKey] = &PVC{}
+		}
+
+		pvcMap[pvcKey].Name = name
+		pvcMap[pvcKey].Namespace = namespace
+		pvcMap[pvcKey].Volume = pvMap[pvKey]
+		pvcMap[pvcKey].Start = pvcStart
+		pvcMap[pvcKey].End = pvcEnd
+	}
+}
+
+func applyPVCBytesRequested(pvcMap map[pvcKey]*PVC, resPVCBytesRequested []*prom.QueryResult) {
+	for _, res := range resPVCBytesRequested {
+		key, err := resultPVCKey(res, "cluster_id", "namespace", "persistentvolumeclaim")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := pvcMap[key]; !ok {
+			continue
+		}
+
+		pvcMap[key].Bytes = res.Values[0].Value
+	}
+}
+
+func buildPodPVCMap(podPVCMap map[podKey][]*PVC, pvMap map[pvKey]*PV, pvcMap map[pvcKey]*PVC, podMap map[podKey]*Pod, resPodPVCAllocation []*prom.QueryResult) {
+	for _, res := range resPodPVCAllocation {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		values, err := res.GetStrings("persistentvolume", "persistentvolumeclaim", "pod", "namespace")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PVC allocation query result missing field: %s", err)
+			continue
+		}
+
+		namespace := values["namespace"]
+		pod := values["pod"]
+		name := values["persistentvolumeclaim"]
+		volume := values["persistentvolume"]
+
+		podKey := newPodKey(cluster, namespace, pod)
+		pvKey := newPVKey(cluster, volume)
+		pvcKey := newPVCKey(cluster, namespace, name)
+
+		if _, ok := pvMap[pvKey]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: PV missing for PVC allocation query result: %s", pvKey)
+			continue
+		}
+
+		if _, ok := podPVCMap[podKey]; !ok {
+			podPVCMap[podKey] = []*PVC{}
+		}
+
+		pvc, ok := pvcMap[pvcKey]
+		if !ok {
+			log.Warningf("CostModel.ComputeAllocation: PVC missing for PVC allocation query: %s", pvcKey)
+			continue
+		}
+
+		count := 1
+		if pod, ok := podMap[podKey]; ok && len(pod.Allocations) > 0 {
+			count = len(pod.Allocations)
+		} else {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: PVC %s for missing pod %s", pvcKey, podKey)
+		}
+
+		pvc.Count = count
+		pvc.Mounted = true
+
+		podPVCMap[podKey] = append(podPVCMap[podKey], pvc)
+	}
+}
+
+func applyUnmountedPVs(window kubecost.Window, podMap map[podKey]*Pod, pvMap map[pvKey]*PV, pvcMap map[pvcKey]*PVC) {
+	unmountedPVBytes := map[string]float64{}
+	unmountedPVCost := map[string]float64{}
+
+	for _, pv := range pvMap {
+		mounted := false
+		for _, pvc := range pvcMap {
+			if pvc.Volume == nil {
+				continue
+			}
+			if pvc.Volume == pv {
+				mounted = true
+				break
+			}
+		}
+
+		if !mounted {
+			gib := pv.Bytes / 1024 / 1024 / 1024
+			hrs := window.Minutes() / 60.0 // TODO improve with PV hours, not window hours
+			cost := pv.CostPerGiBHour * gib * hrs
+			unmountedPVCost[pv.Cluster] += cost
+			unmountedPVBytes[pv.Cluster] += pv.Bytes
+		}
+	}
+
+	for cluster, amount := range unmountedPVCost {
+		container := kubecost.UnmountedSuffix
+		pod := kubecost.UnmountedSuffix
+		namespace := kubecost.UnmountedSuffix
+		node := ""
+
+		key := newPodKey(cluster, namespace, pod)
+		podMap[key] = &Pod{
+			Window:      window.Clone(),
+			Start:       *window.Start(),
+			End:         *window.End(),
+			Key:         key,
+			Allocations: map[string]*kubecost.Allocation{},
+		}
+
+		podMap[key].AppendContainer(container)
+		podMap[key].Allocations[container].Properties.SetCluster(cluster)
+		podMap[key].Allocations[container].Properties.SetNode(node)
+		podMap[key].Allocations[container].Properties.SetNamespace(namespace)
+		podMap[key].Allocations[container].Properties.SetPod(pod)
+		podMap[key].Allocations[container].Properties.SetContainer(container)
+		podMap[key].Allocations[container].PVByteHours = unmountedPVBytes[cluster] * window.Minutes() / 60.0
+		podMap[key].Allocations[container].PVCost = amount
+	}
+}
+
+func applyUnmountedPVCs(window kubecost.Window, podMap map[podKey]*Pod, pvcMap map[pvcKey]*PVC) {
+	unmountedPVCBytes := map[namespaceKey]float64{}
+	unmountedPVCCost := map[namespaceKey]float64{}
+
+	for _, pvc := range pvcMap {
+		if !pvc.Mounted && pvc.Volume != nil {
+			key := newNamespaceKey(pvc.Cluster, pvc.Namespace)
+
+			gib := pvc.Volume.Bytes / 1024 / 1024 / 1024
+			hrs := pvc.Minutes() / 60.0
+			cost := pvc.Volume.CostPerGiBHour * gib * hrs
+			unmountedPVCCost[key] += cost
+			unmountedPVCBytes[key] += pvc.Volume.Bytes
+		}
+	}
+
+	for key, amount := range unmountedPVCCost {
+		container := kubecost.UnmountedSuffix
+		pod := kubecost.UnmountedSuffix
+		namespace := key.Namespace
+		node := ""
+		cluster := key.Cluster
+
+		podKey := newPodKey(cluster, namespace, pod)
+		podMap[podKey] = &Pod{
+			Window:      window.Clone(),
+			Start:       *window.Start(),
+			End:         *window.End(),
+			Key:         podKey,
+			Allocations: map[string]*kubecost.Allocation{},
+		}
+
+		podMap[podKey].AppendContainer(container)
+		podMap[podKey].Allocations[container].Properties.SetCluster(cluster)
+		podMap[podKey].Allocations[container].Properties.SetNode(node)
+		podMap[podKey].Allocations[container].Properties.SetNamespace(namespace)
+		podMap[podKey].Allocations[container].Properties.SetPod(pod)
+		podMap[podKey].Allocations[container].Properties.SetContainer(container)
+		podMap[podKey].Allocations[container].PVByteHours = unmountedPVCBytes[key] * window.Minutes() / 60.0
+		podMap[podKey].Allocations[container].PVCost = amount
+	}
+}
+
+// getNodePricing determines node pricing, given a key and a mapping from keys
+// to their NodePricing instances, as well as the custom pricing configuration
+// inherent to the CostModel instance. If custom pricing is set, use that. If
+// not, use the pricing defined by the given key. If that doesn't exist, fall
+// back on custom pricing as a default.
+func (cm *CostModel) getNodePricing(nodeMap map[nodeKey]*NodePricing, nodeKey nodeKey) *NodePricing {
+	// Find the relevant NodePricing, if it exists. If not, substitute the
+	// custom NodePricing as a default.
+	node, ok := nodeMap[nodeKey]
+	if !ok || node == nil {
+		if nodeKey.Node != "" {
+			log.Warningf("CostModel: failed to find node for %s", nodeKey)
+		}
+		return cm.getCustomNodePricing(false)
+	}
+
+	// If custom pricing is enabled and can be retrieved, override detected
+	// node pricing with the custom values.
+	customPricingConfig, err := cm.Provider.GetConfig()
+	if err != nil {
+		log.Warningf("CostModel: failed to load custom pricing: %s", err)
+	}
+	if cloud.CustomPricesEnabled(cm.Provider) && customPricingConfig != nil {
+		return cm.getCustomNodePricing(node.Preemptible)
+	}
+
+	node.Source = "prometheus"
+
+	// If any of the values are NaN or zero, replace them with the custom
+	// values as default.
+	// TODO:CLEANUP can't we parse these custom prices once? why do we store
+	// them as strings like this?
+
+	if node.CostPerCPUHr == 0 || math.IsNaN(node.CostPerCPUHr) {
+		log.Warningf("CostModel: node pricing has illegal CostPerCPUHr; replacing with custom pricing: %s", nodeKey)
+		cpuCostStr := customPricingConfig.CPU
+		if node.Preemptible {
+			cpuCostStr = customPricingConfig.SpotCPU
+		}
+		costPerCPUHr, err := strconv.ParseFloat(cpuCostStr, 64)
+		if err != nil {
+			log.Warningf("CostModel: custom pricing has illegal CPU cost: %s", cpuCostStr)
+		}
+		node.CostPerCPUHr = costPerCPUHr
+		node.Source += "/customCPU"
+	}
+
+	if math.IsNaN(node.CostPerGPUHr) {
+		log.Warningf("CostModel: node pricing has illegal CostPerGPUHr; replacing with custom pricing: %s", nodeKey)
+		gpuCostStr := customPricingConfig.GPU
+		if node.Preemptible {
+			gpuCostStr = customPricingConfig.SpotGPU
+		}
+		costPerGPUHr, err := strconv.ParseFloat(gpuCostStr, 64)
+		if err != nil {
+			log.Warningf("CostModel: custom pricing has illegal GPU cost: %s", gpuCostStr)
+		}
+		node.CostPerGPUHr = costPerGPUHr
+		node.Source += "/customGPU"
+	}
+
+	if node.CostPerRAMGiBHr == 0 || math.IsNaN(node.CostPerRAMGiBHr) {
+		log.Warningf("CostModel: node pricing has illegal CostPerRAMHr; replacing with custom pricing: %s", nodeKey)
+		ramCostStr := customPricingConfig.RAM
+		if node.Preemptible {
+			ramCostStr = customPricingConfig.SpotRAM
+		}
+		costPerRAMHr, err := strconv.ParseFloat(ramCostStr, 64)
+		if err != nil {
+			log.Warningf("CostModel: custom pricing has illegal RAM cost: %s", ramCostStr)
+		}
+		node.CostPerRAMGiBHr = costPerRAMHr
+		node.Source += "/customRAM"
+	}
+
+	return node
+}
+
+// getCustomNodePricing converts the CostModel's configured custom pricing
+// values into a NodePricing instance.
+func (cm *CostModel) getCustomNodePricing(spot bool) *NodePricing {
+	customPricingConfig, err := cm.Provider.GetConfig()
+	if err != nil {
+		return nil
+	}
+
+	cpuCostStr := customPricingConfig.CPU
+	gpuCostStr := customPricingConfig.GPU
+	ramCostStr := customPricingConfig.RAM
+	if spot {
+		cpuCostStr = customPricingConfig.SpotCPU
+		gpuCostStr = customPricingConfig.SpotGPU
+		ramCostStr = customPricingConfig.SpotRAM
+	}
+
+	node := &NodePricing{Source: "custom"}
+
+	costPerCPUHr, err := strconv.ParseFloat(cpuCostStr, 64)
+	if err != nil {
+		log.Warningf("CostModel: custom pricing has illegal CPU cost: %s", cpuCostStr)
+	}
+	node.CostPerCPUHr = costPerCPUHr
+
+	costPerGPUHr, err := strconv.ParseFloat(gpuCostStr, 64)
+	if err != nil {
+		log.Warningf("CostModel: custom pricing has illegal GPU cost: %s", gpuCostStr)
+	}
+	node.CostPerGPUHr = costPerGPUHr
+
+	costPerRAMHr, err := strconv.ParseFloat(ramCostStr, 64)
+	if err != nil {
+		log.Warningf("CostModel: custom pricing has illegal RAM cost: %s", ramCostStr)
+	}
+	node.CostPerRAMGiBHr = costPerRAMHr
+
+	return node
+}
+
+// NodePricing describes the resource costs associated with a given node, as
+// well as the source of the information (e.g. prometheus, custom)
+type NodePricing struct {
+	Name            string
+	NodeType        string
+	Preemptible     bool
+	CostPerCPUHr    float64
+	CostPerRAMGiBHr float64
+	CostPerGPUHr    float64
+	Discount        float64
+	Source          string
+}
+
+// Pod describes a running pod's start and end time within a Window and
+// all the Allocations (i.e. containers) contained within it.
+type Pod struct {
+	Window      kubecost.Window
+	Start       time.Time
+	End         time.Time
+	Key         podKey
+	Allocations map[string]*kubecost.Allocation
+}
+
+// AppendContainer adds an entry for the given container name to the Pod.
+func (p Pod) AppendContainer(container string) {
+	name := fmt.Sprintf("%s/%s/%s/%s", p.Key.Cluster, p.Key.Namespace, p.Key.Pod, container)
+
+	alloc := &kubecost.Allocation{
+		Name:       name,
+		Properties: kubecost.Properties{},
+		Window:     p.Window.Clone(),
+		Start:      p.Start,
+		End:        p.End,
+	}
+	alloc.Properties.SetContainer(container)
+	alloc.Properties.SetPod(p.Key.Pod)
+	alloc.Properties.SetNamespace(p.Key.Namespace)
+	alloc.Properties.SetCluster(p.Key.Cluster)
+
+	p.Allocations[container] = alloc
+}
+
+// PVC describes a PersistentVolumeClaim
+// TODO:CLEANUP move to pkg/kubecost?
+// TODO:CLEANUP add PersistentVolumeClaims field to type Allocation?
+type PVC struct {
+	Bytes     float64   `json:"bytes"`
+	Count     int       `json:"count"`
+	Name      string    `json:"name"`
+	Cluster   string    `json:"cluster"`
+	Namespace string    `json:"namespace"`
+	Volume    *PV       `json:"persistentVolume"`
+	Mounted   bool      `json:"mounted"`
+	Start     time.Time `json:"start"`
+	End       time.Time `json:"end"`
+}
+
+// Cost computes the cumulative cost of the PVC
+func (pvc *PVC) Cost() float64 {
+	if pvc == nil || pvc.Volume == nil {
+		return 0.0
+	}
+
+	gib := pvc.Bytes / 1024 / 1024 / 1024
+	hrs := pvc.Minutes() / 60.0
+
+	return pvc.Volume.CostPerGiBHour * gib * hrs
+}
+
+// Minutes computes the number of minutes over which the PVC is defined
+func (pvc *PVC) Minutes() float64 {
+	if pvc == nil {
+		return 0.0
+	}
+
+	return pvc.End.Sub(pvc.Start).Minutes()
+}
+
+// String returns a string representation of the PVC
+func (pvc *PVC) String() string {
+	if pvc == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("%s/%s/%s{Bytes:%.2f, Cost:%.6f, Start,End:%s}", pvc.Cluster, pvc.Namespace, pvc.Name, pvc.Bytes, pvc.Cost(), kubecost.NewWindow(&pvc.Start, &pvc.End))
+}
+
+// PV describes a PersistentVolume
+// TODO:CLEANUP move to pkg/kubecost?
+type PV struct {
+	Bytes          float64 `json:"bytes"`
+	CostPerGiBHour float64 `json:"costPerGiBHour"`
+	Cluster        string  `json:"cluster"`
+	Name           string  `json:"name"`
+	StorageClass   string  `json:"storageClass"`
+}
+
+// String returns a string representation of the PV
+func (pv *PV) String() string {
+	if pv == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("%s/%s{Bytes:%.2f, Cost/GiB*Hr:%.6f, StorageClass:%s}", pv.Cluster, pv.Name, pv.Bytes, pv.CostPerGiBHour, pv.StorageClass)
+}

pkg/costmodel/cluster.go

@@ -382,23 +382,26 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, duration, o
 }
 
 type Node struct {
-	Cluster      string
-	Name         string
-	ProviderID   string
-	NodeType     string
-	CPUCost      float64
-	CPUCores     float64
-	GPUCost      float64
-	RAMCost      float64
-	RAMBytes     float64
-	Discount     float64
-	Preemptible  bool
-	CPUBreakdown *ClusterCostsBreakdown
-	RAMBreakdown *ClusterCostsBreakdown
-	Start        time.Time
-	End          time.Time
-	Minutes      float64
-	Labels       map[string]string
+	Cluster         string
+	Name            string
+	ProviderID      string
+	NodeType        string
+	CPUCost         float64
+	CPUCores        float64
+	GPUCost         float64
+	RAMCost         float64
+	RAMBytes        float64
+	Discount        float64
+	Preemptible     bool
+	CPUBreakdown    *ClusterCostsBreakdown
+	RAMBreakdown    *ClusterCostsBreakdown
+	Start           time.Time
+	End             time.Time
+	Minutes         float64
+	Labels          map[string]string
+	CostPerCPUHr    float64
+	CostPerRAMGiBHr float64
+	CostPerGPUHr    float64
 }
 
 // GKE lies about the number of cores e2 nodes have. This table
@@ -540,7 +543,7 @@ func ClusterNodes(cp cloud.Provider, client prometheus.Client, duration, offset
 	}
 
 	for _, node := range nodeMap {
-		// TODO take RI into account
+		// TODO take GKE Reserved Instances into account
 		node.Discount = cp.CombinedDiscountForNode(node.NodeType, node.Preemptible, discount, negotiatedDiscount)
 
 		// Apply all remaining resources to Idle

pkg/costmodel/costmodel.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/util"
+	prometheus "github.com/prometheus/client_golang/api"
 	prometheusClient "github.com/prometheus/client_golang/api"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,22 +47,26 @@ const (
 var isCron = regexp.MustCompile(`^(.+)-\d{10}$`)
 
 type CostModel struct {
-	Cache           clustercache.ClusterCache
-	ClusterMap      clusters.ClusterMap
-	ScrapeInterval  time.Duration
-	RequestGroup    *singleflight.Group
-	pricingMetadata *costAnalyzerCloud.PricingMatchMetadata
+	Cache            clustercache.ClusterCache
+	ClusterMap       clusters.ClusterMap
+	RequestGroup     *singleflight.Group
+	ScrapeInterval   time.Duration
+	PrometheusClient prometheus.Client
+	Provider         costAnalyzerCloud.Provider
+	pricingMetadata  *costAnalyzerCloud.PricingMatchMetadata
 }
 
-func NewCostModel(cache clustercache.ClusterCache, clusterMap clusters.ClusterMap, scrapeInterval time.Duration) *CostModel {
+func NewCostModel(client prometheus.Client, provider costAnalyzerCloud.Provider, cache clustercache.ClusterCache, clusterMap clusters.ClusterMap, scrapeInterval time.Duration) *CostModel {
 	// request grouping to prevent over-requesting the same data prior to caching
 	requestGroup := new(singleflight.Group)
 
 	return &CostModel{
-		Cache:          cache,
-		ClusterMap:     clusterMap,
-		RequestGroup:   requestGroup,
-		ScrapeInterval: scrapeInterval,
+		Cache:            cache,
+		ClusterMap:       clusterMap,
+		PrometheusClient: client,
+		Provider:         provider,
+		RequestGroup:     requestGroup,
+		ScrapeInterval:   scrapeInterval,
 	}
 }
 
@@ -1494,10 +1499,6 @@ func requestKeyFor(window kubecost.Window, resolution time.Duration, filterNames
 	return fmt.Sprintf("%s,%s,%s,%s,%s,%t", startKey, endKey, resolution.String(), filterNamespace, filterCluster, remoteEnabled)
 }
 
-// func (cm *CostModel) ComputeCostDataRange(cli prometheusClient.Client, cp costAnalyzerCloud.Provider,
-// 	startString, endString, windowString string, resolutionHours float64, filterNamespace string,
-// 	filterCluster string, remoteEnabled bool, offset string) (map[string]*CostData, error)
-
 // ComputeCostDataRange executes a range query for cost data.
 // Note that "offset" represents the time between the function call and "endString", and is also passed for convenience
 func (cm *CostModel) ComputeCostDataRange(cli prometheusClient.Client, cp costAnalyzerCloud.Provider, window kubecost.Window, resolution time.Duration, filterNamespace string, filterCluster string, remoteEnabled bool) (map[string]*CostData, error) {

pkg/costmodel/key.go

@@ -0,0 +1,404 @@
+package costmodel
+
+import (
+	"fmt"
+
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/prom"
+)
+
+type containerKey struct {
+	Cluster   string
+	Namespace string
+	Pod       string
+	Container string
+}
+
+func (k containerKey) String() string {
+	return fmt.Sprintf("%s/%s/%s/%s", k.Cluster, k.Namespace, k.Pod, k.Container)
+}
+
+func newContainerKey(cluster, namespace, pod, container string) containerKey {
+	return containerKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+		Pod:       pod,
+		Container: container,
+	}
+}
+
+// resultContainerKey converts a Prometheus query result to a containerKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the containerKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultContainerKey(res *prom.QueryResult, clusterLabel, namespaceLabel, podLabel, containerLabel string) (containerKey, error) {
+	key := containerKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	pod, err := res.GetString(podLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Pod = pod
+
+	container, err := res.GetString(containerLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Container = container
+
+	return key, nil
+}
+
+type podKey struct {
+	Cluster   string
+	Namespace string
+	Pod       string
+}
+
+func (k podKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.Cluster, k.Namespace, k.Pod)
+}
+
+func newPodKey(cluster, namespace, pod string) podKey {
+	return podKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+		Pod:       pod,
+	}
+}
+
+// resultPodKey converts a Prometheus query result to a podKey by looking
+// up values associated with the given label names. For example, passing
+// "cluster_id" for clusterLabel will use the value of the label "cluster_id"
+// as the podKey's Cluster field. If a given field does not exist on the
+// result, an error is returned. (The only exception to that is clusterLabel,
+// which we expect may not exist, but has a default value.)
+func resultPodKey(res *prom.QueryResult, clusterLabel, namespaceLabel, podLabel string) (podKey, error) {
+	key := podKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	pod, err := res.GetString(podLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Pod = pod
+
+	return key, nil
+}
+
+type namespaceKey struct {
+	Cluster   string
+	Namespace string
+}
+
+func (k namespaceKey) String() string {
+	return fmt.Sprintf("%s/%s", k.Cluster, k.Namespace)
+}
+
+func newNamespaceKey(cluster, namespace string) namespaceKey {
+	return namespaceKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+	}
+}
+
+// resultNamespaceKey converts a Prometheus query result to a namespaceKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the namespaceKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultNamespaceKey(res *prom.QueryResult, clusterLabel, namespaceLabel string) (namespaceKey, error) {
+	key := namespaceKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	return key, nil
+}
+
+type controllerKey struct {
+	Cluster        string
+	Namespace      string
+	ControllerKind string
+	Controller     string
+}
+
+func (k controllerKey) String() string {
+	return fmt.Sprintf("%s/%s/%s/%s", k.Cluster, k.Namespace, k.ControllerKind, k.Controller)
+}
+
+func newControllerKey(cluster, namespace, controllerKind, controller string) controllerKey {
+	return controllerKey{
+		Cluster:        cluster,
+		Namespace:      namespace,
+		ControllerKind: controllerKind,
+		Controller:     controller,
+	}
+}
+
+// resultControllerKey converts a Prometheus query result to a controllerKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the controllerKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultControllerKey(controllerKind string, res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	key := controllerKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	controller, err := res.GetString(controllerLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Controller = controller
+
+	key.ControllerKind = controllerKind
+
+	return key, nil
+}
+
+// resultDeploymentKey creates a controllerKey for a Deployment.
+// (See resultControllerKey for more.)
+func resultDeploymentKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("deployment", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+// resultDeploymentKey creates a controllerKey for a StatefulSet.
+// (See resultControllerKey for more.)
+func resultStatefulSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("statefulset", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+// resultDeploymentKey creates a controllerKey for a DaemonSet.
+// (See resultControllerKey for more.)
+func resultDaemonSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("daemonset", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+// resultDeploymentKey creates a controllerKey for a Job.
+// (See resultControllerKey for more.)
+func resultJobKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("job", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+type serviceKey struct {
+	Cluster   string
+	Namespace string
+	Service   string
+}
+
+func (k serviceKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.Cluster, k.Namespace, k.Service)
+}
+
+func newServiceKey(cluster, namespace, service string) serviceKey {
+	return serviceKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+		Service:   service,
+	}
+}
+
+// resultServiceKey converts a Prometheus query result to a serviceKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the serviceKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultServiceKey(res *prom.QueryResult, clusterLabel, namespaceLabel, serviceLabel string) (serviceKey, error) {
+	key := serviceKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	service, err := res.GetString(serviceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Service = service
+
+	return key, nil
+}
+
+type nodeKey struct {
+	Cluster string
+	Node    string
+}
+
+func (k nodeKey) String() string {
+	return fmt.Sprintf("%s/%s", k.Cluster, k.Node)
+}
+
+func newNodeKey(cluster, node string) nodeKey {
+	return nodeKey{
+		Cluster: cluster,
+		Node:    node,
+	}
+}
+
+// resultNodeKey converts a Prometheus query result to a nodeKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the nodeKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultNodeKey(res *prom.QueryResult, clusterLabel, nodeLabel string) (nodeKey, error) {
+	key := nodeKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	node, err := res.GetString(nodeLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Node = node
+
+	return key, nil
+}
+
+type pvcKey struct {
+	Cluster               string
+	Namespace             string
+	PersistentVolumeClaim string
+}
+
+func (k pvcKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.Cluster, k.Namespace, k.PersistentVolumeClaim)
+}
+
+func newPVCKey(cluster, namespace, persistentVolumeClaim string) pvcKey {
+	return pvcKey{
+		Cluster:               cluster,
+		Namespace:             namespace,
+		PersistentVolumeClaim: persistentVolumeClaim,
+	}
+}
+
+// resultPVCKey converts a Prometheus query result to a pvcKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the pvcKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultPVCKey(res *prom.QueryResult, clusterLabel, namespaceLabel, pvcLabel string) (pvcKey, error) {
+	key := pvcKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	pvc, err := res.GetString(pvcLabel)
+	if err != nil {
+		return key, err
+	}
+	key.PersistentVolumeClaim = pvc
+
+	return key, nil
+}
+
+type pvKey struct {
+	Cluster          string
+	PersistentVolume string
+}
+
+func (k pvKey) String() string {
+	return fmt.Sprintf("%s/%s", k.Cluster, k.PersistentVolume)
+}
+
+func newPVKey(cluster, persistentVolume string) pvKey {
+	return pvKey{
+		Cluster:          cluster,
+		PersistentVolume: persistentVolume,
+	}
+}
+
+// resultPVKey converts a Prometheus query result to a pvKey by
+// looking up values associated with the given label names. For example,
+// passing "cluster_id" for clusterLabel will use the value of the label
+// "cluster_id" as the pvKey's Cluster field. If a given field does not
+// exist on the result, an error is returned. (The only exception to that is
+// clusterLabel, which we expect may not exist, but has a default value.)
+func resultPVKey(res *prom.QueryResult, clusterLabel, persistentVolumeLabel string) (pvKey, error) {
+	key := pvKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	persistentVolume, err := res.GetString(persistentVolumeLabel)
+	if err != nil {
+		return key, err
+	}
+	key.PersistentVolume = persistentVolume
+
+	return key, nil
+}

pkg/costmodel/router.go

@@ -28,6 +28,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/thanos"
+	prometheus "github.com/prometheus/client_golang/api"
 	prometheusClient "github.com/prometheus/client_golang/api"
 	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
 	v1 "k8s.io/api/core/v1"
@@ -243,6 +244,7 @@ func ParsePercentString(percentStr string) (float64, error) {
 }
 
 // parseDuration converts a Prometheus-style duration string into a Duration
+// TODO:CLEANUP delete this. do it now.
 func ParseDuration(duration string) (*time.Duration, error) {
 	unitStr := duration[len(duration)-1:]
 	var unit time.Duration
@@ -1013,7 +1015,13 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		30 * day: maxCacheMinutes30d * time.Minute,
 	}
 
-	costModel := NewCostModel(k8sCache, clusterMap, scrapeInterval)
+	var pc prometheus.Client
+	if thanosClient != nil {
+		pc = thanosClient
+	} else {
+		pc = promCli
+	}
+	costModel := NewCostModel(pc, cloudProvider, k8sCache, clusterMap, scrapeInterval)
 	metricsEmitter := NewCostModelMetricsEmitter(promCli, k8sCache, cloudProvider, costModel)
 
 	a := &Accesses{

pkg/env/costmodelenv.go

@@ -63,6 +63,7 @@ const (
 
 	CacheWarmingEnabledEnvVar    = "CACHE_WARMING_ENABLED"
 	ETLEnabledEnvVar             = "ETL_ENABLED"
+	ETLMaxBatchHours             = "ETL_MAX_BATCH_HOURS"
 	LegacyExternalAPIDisabledVar = "LEGACY_EXTERNAL_API_DISABLED"
 )
 
@@ -338,6 +339,16 @@ func IsETLEnabled() bool {
 	return GetBool(ETLEnabledEnvVar, true)
 }
 
+// GetETLMaxBatchDuration limits the window duration of the most expensive ETL
+// queries to a maximum batch size, such that queries can be tuned to avoid
+// timeout for large windows; e.g. if a 24h query is expected to timeout, but
+// a 6h query is expected to complete in 1m, then 6h could be a good value.
+func GetETLMaxBatchDuration() time.Duration {
+	// Default to 6h
+	hrs := time.Duration(GetInt64(ETLMaxBatchHours, 6))
+	return hrs * time.Hour
+}
+
 func LegacyExternalCostsAPIDisabled() bool {
 	return GetBool(LegacyExternalAPIDisabledVar, false)
 }

pkg/kubecost/allocation.go

@@ -1,6 +1,7 @@
 package kubecost
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"sort"
@@ -9,6 +10,7 @@ import (
 	"time"
 
 	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 // TODO Clean-up use of IsEmpty; nil checks should be separated for safety.
@@ -28,6 +30,9 @@ const SharedSuffix = "__shared__"
 // UnallocatedSuffix indicates an unallocated allocation property
 const UnallocatedSuffix = "__unallocated__"
 
+// UnmountedSuffix indicated allocation to an unmounted PV
+const UnmountedSuffix = "__unmounted__"
+
 // ShareWeighted indicates that a shared resource should be shared as a
 // proportion of the cost of the remaining allocations.
 const ShareWeighted = "__weighted__"
@@ -42,28 +47,29 @@ const ShareNone = "__none__"
 // Allocation is a unit of resource allocation and cost for a given window
 // of time and for a given kubernetes construct with its associated set of
 // properties.
+// TODO:CLEANUP consider dropping name in favor of just Properties and an
+// Assets-style key() function for AllocationSet.
 type Allocation struct {
-	Name            string     `json:"name"`
-	Properties      Properties `json:"properties,omitempty"`
-	Start           time.Time  `json:"start"`
-	End             time.Time  `json:"end"`
-	Minutes         float64    `json:"minutes"`
-	ActiveStart     time.Time  `json:"-"`
-	CPUCoreHours    float64    `json:"cpuCoreHours"`
-	CPUCost         float64    `json:"cpuCost"`
-	CPUEfficiency   float64    `json:"cpuEfficiency"`
-	GPUHours        float64    `json:"gpuHours"`
-	GPUCost         float64    `json:"gpuCost"`
-	NetworkCost     float64    `json:"networkCost"`
-	PVByteHours     float64    `json:"pvByteHours"`
-	PVCost          float64    `json:"pvCost"`
-	RAMByteHours    float64    `json:"ramByteHours"`
-	RAMCost         float64    `json:"ramCost"`
-	RAMEfficiency   float64    `json:"ramEfficiency"`
-	SharedCost      float64    `json:"sharedCost"`
-	ExternalCost    float64    `json:"externalCost"`
-	TotalCost       float64    `json:"totalCost"`
-	TotalEfficiency float64    `json:"totalEfficiency"`
+	Name                   string     `json:"name"`
+	Properties             Properties `json:"properties,omitempty"`
+	Window                 Window     `json:"window"`
+	Start                  time.Time  `json:"start"`
+	End                    time.Time  `json:"end"`
+	CPUCoreHours           float64    `json:"cpuCoreHours"`
+	CPUCoreRequestAverage  float64    `json:"cpuCoreRequestAverage"`
+	CPUCoreUsageAverage    float64    `json:"cpuCoreUsageAverage"`
+	CPUCost                float64    `json:"cpuCost"`
+	GPUHours               float64    `json:"gpuHours"`
+	GPUCost                float64    `json:"gpuCost"`
+	NetworkCost            float64    `json:"networkCost"`
+	PVByteHours            float64    `json:"pvByteHours"`
+	PVCost                 float64    `json:"pvCost"`
+	RAMByteHours           float64    `json:"ramByteHours"`
+	RAMBytesRequestAverage float64    `json:"ramByteRequestAverage"`
+	RAMBytesUsageAverage   float64    `json:"ramByteUsageAverage"`
+	RAMCost                float64    `json:"ramCost"`
+	SharedCost             float64    `json:"sharedCost"`
+	ExternalCost           float64    `json:"externalCost"`
 }
 
 // AllocationMatchFunc is a function that can be used to match Allocations by
@@ -78,12 +84,13 @@ func (a *Allocation) Add(that *Allocation) (*Allocation, error) {
 		return that.Clone(), nil
 	}
 
-	if !a.Start.Equal(that.Start) || !a.End.Equal(that.End) {
-		return nil, fmt.Errorf("error adding Allocations: mismatched windows")
+	if that == nil {
+		return a.Clone(), nil
 	}
 
+	// Note: no need to clone "that", as add only mutates the receiver
 	agg := a.Clone()
-	agg.add(that, false, false)
+	agg.add(that)
 
 	return agg, nil
 }
@@ -95,32 +102,33 @@ func (a *Allocation) Clone() *Allocation {
 	}
 
 	return &Allocation{
-		Name:            a.Name,
-		Properties:      a.Properties.Clone(),
-		Start:           a.Start,
-		End:             a.End,
-		Minutes:         a.Minutes,
-		ActiveStart:     a.ActiveStart,
-		CPUCoreHours:    a.CPUCoreHours,
-		CPUCost:         a.CPUCost,
-		CPUEfficiency:   a.CPUEfficiency,
-		GPUHours:        a.GPUHours,
-		GPUCost:         a.GPUCost,
-		NetworkCost:     a.NetworkCost,
-		PVByteHours:     a.PVByteHours,
-		PVCost:          a.PVCost,
-		RAMByteHours:    a.RAMByteHours,
-		RAMCost:         a.RAMCost,
-		RAMEfficiency:   a.RAMEfficiency,
-		SharedCost:      a.SharedCost,
-		ExternalCost:    a.ExternalCost,
-		TotalCost:       a.TotalCost,
-		TotalEfficiency: a.TotalEfficiency,
+		Name:                   a.Name,
+		Properties:             a.Properties.Clone(),
+		Window:                 a.Window.Clone(),
+		Start:                  a.Start,
+		End:                    a.End,
+		CPUCoreHours:           a.CPUCoreHours,
+		CPUCoreRequestAverage:  a.CPUCoreRequestAverage,
+		CPUCoreUsageAverage:    a.CPUCoreUsageAverage,
+		CPUCost:                a.CPUCost,
+		GPUHours:               a.GPUHours,
+		GPUCost:                a.GPUCost,
+		NetworkCost:            a.NetworkCost,
+		PVByteHours:            a.PVByteHours,
+		PVCost:                 a.PVCost,
+		RAMByteHours:           a.RAMByteHours,
+		RAMBytesRequestAverage: a.RAMBytesRequestAverage,
+		RAMBytesUsageAverage:   a.RAMBytesUsageAverage,
+		RAMCost:                a.RAMCost,
+		SharedCost:             a.SharedCost,
+		ExternalCost:           a.ExternalCost,
 	}
 }
 
 // Equal returns true if the values held in the given Allocation precisely
-// match those of the receiving Allocation. nil does not match nil.
+// match those of the receiving Allocation. nil does not match nil. Floating
+// point values need to match according to util.IsApproximately, which accounts
+// for small, reasonable floating point error margins.
 func (a *Allocation) Equal(that *Allocation) bool {
 	if a == nil || that == nil {
 		return false
@@ -129,68 +137,159 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.Name != that.Name {
 		return false
 	}
-	if !a.Start.Equal(that.Start) {
+	if !a.Properties.Equal(&that.Properties) {
 		return false
 	}
-	if !a.End.Equal(that.End) {
+	if !a.Window.Equal(that.Window) {
 		return false
 	}
-	if a.Minutes != that.Minutes {
+	if !a.Start.Equal(that.Start) {
 		return false
 	}
-	if !a.ActiveStart.Equal(that.ActiveStart) {
+	if !a.End.Equal(that.End) {
 		return false
 	}
-	if a.CPUCoreHours != that.CPUCoreHours {
+	if !util.IsApproximately(a.CPUCoreHours, that.CPUCoreHours) {
 		return false
 	}
-	if a.CPUCost != that.CPUCost {
+	if !util.IsApproximately(a.CPUCost, that.CPUCost) {
 		return false
 	}
-	if a.CPUEfficiency != that.CPUEfficiency {
+	if !util.IsApproximately(a.GPUHours, that.GPUHours) {
 		return false
 	}
-	if a.GPUHours != that.GPUHours {
+	if !util.IsApproximately(a.GPUCost, that.GPUCost) {
 		return false
 	}
-	if a.GPUCost != that.GPUCost {
+	if !util.IsApproximately(a.NetworkCost, that.NetworkCost) {
 		return false
 	}
-	if a.NetworkCost != that.NetworkCost {
+	if !util.IsApproximately(a.PVByteHours, that.PVByteHours) {
 		return false
 	}
-	if a.PVByteHours != that.PVByteHours {
+	if !util.IsApproximately(a.PVCost, that.PVCost) {
 		return false
 	}
-	if a.PVCost != that.PVCost {
+	if !util.IsApproximately(a.RAMByteHours, that.RAMByteHours) {
 		return false
 	}
-	if a.RAMByteHours != that.RAMByteHours {
+	if !util.IsApproximately(a.RAMCost, that.RAMCost) {
 		return false
 	}
-	if a.RAMCost != that.RAMCost {
+	if !util.IsApproximately(a.SharedCost, that.SharedCost) {
 		return false
 	}
-	if a.RAMEfficiency != that.RAMEfficiency {
+	if !util.IsApproximately(a.ExternalCost, that.ExternalCost) {
 		return false
 	}
-	if a.SharedCost != that.SharedCost {
-		return false
+
+	return true
+}
+
+// TotalCost is the total cost of the Allocation
+func (a *Allocation) TotalCost() float64 {
+	return a.CPUCost + a.GPUCost + a.RAMCost + a.PVCost + a.NetworkCost + a.SharedCost + a.ExternalCost
+}
+
+// CPUEfficiency is the ratio of usage to request. If there is no request and
+// no usage or cost, then efficiency is zero. If there is no request, but there
+// is usage or cost, then efficiency is 100%.
+func (a *Allocation) CPUEfficiency() float64 {
+	if a.CPUCoreRequestAverage > 0 {
+		return a.CPUCoreUsageAverage / a.CPUCoreRequestAverage
 	}
-	if a.ExternalCost != that.ExternalCost {
-		return false
+
+	if a.CPUCoreUsageAverage == 0.0 || a.CPUCost == 0.0 {
+		return 0.0
 	}
-	if a.TotalCost != that.TotalCost {
-		return false
+
+	return 1.0
+}
+
+// RAMEfficiency is the ratio of usage to request. If there is no request and
+// no usage or cost, then efficiency is zero. If there is no request, but there
+// is usage or cost, then efficiency is 100%.
+func (a *Allocation) RAMEfficiency() float64 {
+	if a.RAMBytesRequestAverage > 0 {
+		return a.RAMBytesUsageAverage / a.RAMBytesRequestAverage
 	}
-	if a.TotalEfficiency != that.TotalEfficiency {
-		return false
+
+	if a.RAMBytesUsageAverage == 0.0 || a.RAMCost == 0.0 {
+		return 0.0
 	}
-	if !a.Properties.Equal(&that.Properties) {
-		return false
+
+	return 1.0
+}
+
+// TotalEfficiency is the cost-weighted average of CPU and RAM efficiency. If
+// there is no cost at all, then efficiency is zero.
+func (a *Allocation) TotalEfficiency() float64 {
+	if a.CPUCost+a.RAMCost > 0 {
+		ramCostEff := a.RAMEfficiency() * a.RAMCost
+		cpuCostEff := a.CPUEfficiency() * a.CPUCost
+		return (ramCostEff + cpuCostEff) / (a.CPUCost + a.RAMCost)
 	}
 
-	return true
+	return 0.0
+}
+
+// CPUCores converts the Allocation's CPUCoreHours into average CPUCores
+func (a *Allocation) CPUCores() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.CPUCoreHours / (a.Minutes() / 60.0)
+}
+
+// RAMBytes converts the Allocation's RAMByteHours into average RAMBytes
+func (a *Allocation) RAMBytes() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.RAMByteHours / (a.Minutes() / 60.0)
+}
+
+// PVBytes converts the Allocation's PVByteHours into average PVBytes
+func (a *Allocation) PVBytes() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.PVByteHours / (a.Minutes() / 60.0)
+}
+
+// MarshalJSON implements json.Marshal interface
+func (a *Allocation) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "name", a.Name, ",")
+	jsonEncode(buffer, "properties", a.Properties, ",")
+	jsonEncode(buffer, "window", a.Window, ",")
+	jsonEncodeString(buffer, "start", a.Start.Format(time.RFC3339), ",")
+	jsonEncodeString(buffer, "end", a.End.Format(time.RFC3339), ",")
+	jsonEncodeFloat64(buffer, "minutes", a.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "cpuCores", a.CPUCores(), ",")
+	jsonEncodeFloat64(buffer, "cpuCoreRequestAverage", a.CPUCoreRequestAverage, ",")
+	jsonEncodeFloat64(buffer, "cpuCoreUsageAverage", a.CPUCoreUsageAverage, ",")
+	jsonEncodeFloat64(buffer, "cpuCoreHours", a.CPUCoreHours, ",")
+	jsonEncodeFloat64(buffer, "cpuCost", a.CPUCost, ",")
+	jsonEncodeFloat64(buffer, "cpuEfficiency", a.CPUEfficiency(), ",")
+	jsonEncodeFloat64(buffer, "gpuHours", a.GPUHours, ",")
+	jsonEncodeFloat64(buffer, "gpuCost", a.GPUCost, ",")
+	jsonEncodeFloat64(buffer, "networkCost", a.NetworkCost, ",")
+	jsonEncodeFloat64(buffer, "pvBytes", a.PVBytes(), ",")
+	jsonEncodeFloat64(buffer, "pvByteHours", a.PVByteHours, ",")
+	jsonEncodeFloat64(buffer, "pvCost", a.PVCost, ",")
+	jsonEncodeFloat64(buffer, "ramBytes", a.RAMBytes(), ",")
+	jsonEncodeFloat64(buffer, "ramByteRequestAverage", a.RAMBytesRequestAverage, ",")
+	jsonEncodeFloat64(buffer, "ramByteUsageAverage", a.RAMBytesUsageAverage, ",")
+	jsonEncodeFloat64(buffer, "ramByteHours", a.RAMByteHours, ",")
+	jsonEncodeFloat64(buffer, "ramCost", a.RAMCost, ",")
+	jsonEncodeFloat64(buffer, "ramEfficiency", a.RAMEfficiency(), ",")
+	jsonEncodeFloat64(buffer, "sharedCost", a.SharedCost, ",")
+	jsonEncodeFloat64(buffer, "externalCost", a.ExternalCost, ",")
+	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), ",")
+	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
 }
 
 // Resolution returns the duration of time covered by the Allocation
@@ -219,32 +318,36 @@ func (a *Allocation) IsUnallocated() bool {
 	return strings.Contains(a.Name, UnallocatedSuffix)
 }
 
-// Share works like Add, but converts the entire cost of the given Allocation
-// to SharedCost, rather than adding to the individual resource costs.
+// Minutes returns the number of minutes the Allocation represents, as defined
+// by the difference between the end and start times.
+func (a *Allocation) Minutes() float64 {
+	return a.End.Sub(a.Start).Minutes()
+}
+
+// Share adds the TotalCost of the given Allocation to the SharedCost of the
+// receiving Allocation. No Start, End, Window, or Properties are considered.
+// Neither Allocation is mutated; a new Allocation is always returned.
 func (a *Allocation) Share(that *Allocation) (*Allocation, error) {
-	if a == nil {
-		return that.Clone(), nil
+	if that == nil {
+		return a.Clone(), nil
 	}
 
-	if !a.Start.Equal(that.Start) {
-		return nil, fmt.Errorf("mismatched start time: expected %s, received %s", a.Start, that.Start)
-	}
-	if !a.End.Equal(that.End) {
-		return nil, fmt.Errorf("mismatched start time: expected %s, received %s", a.End, that.End)
+	if a == nil {
+		return nil, fmt.Errorf("cannot share with nil Allocation")
 	}
 
 	agg := a.Clone()
-	agg.add(that, true, false)
+	agg.SharedCost += that.TotalCost()
 
 	return agg, nil
 }
 
 // String represents the given Allocation as a string
 func (a *Allocation) String() string {
-	return fmt.Sprintf("%s%s=%.2f", a.Name, NewWindow(&a.Start, &a.End), a.TotalCost)
+	return fmt.Sprintf("%s%s=%.2f", a.Name, NewWindow(&a.Start, &a.End), a.TotalCost())
 }
 
-func (a *Allocation) add(that *Allocation, isShared, isAccumulating bool) {
+func (a *Allocation) add(that *Allocation) {
 	if a == nil {
 		log.Warningf("Allocation.AggregateBy: trying to add a nil receiver")
 		return
@@ -271,66 +374,60 @@ func (a *Allocation) add(that *Allocation, isShared, isAccumulating bool) {
 		}
 	}
 
-	if that.ActiveStart.Before(a.ActiveStart) {
-		a.ActiveStart = that.ActiveStart
-	}
+	// Expand the window to encompass both Allocations
+	a.Window = a.Window.Expand(that.Window)
 
-	if isAccumulating {
-		if a.Start.After(that.Start) {
-			a.Start = that.Start
-		}
+	// Sum non-cumulative fields by turning them into cumulative, adding them,
+	// and then converting them back into averages after minutes have been
+	// combined (just below).
+	cpuReqCoreMins := a.CPUCoreRequestAverage * a.Minutes()
+	cpuReqCoreMins += that.CPUCoreRequestAverage * that.Minutes()
 
-		if a.End.Before(that.End) {
-			a.End = that.End
-		}
+	cpuUseCoreMins := a.CPUCoreUsageAverage * a.Minutes()
+	cpuUseCoreMins += that.CPUCoreUsageAverage * that.Minutes()
 
-		a.Minutes += that.Minutes
-	} else if that.Minutes > a.Minutes {
-		a.Minutes = that.Minutes
-	}
+	ramReqByteMins := a.RAMBytesRequestAverage * a.Minutes()
+	ramReqByteMins += that.RAMBytesRequestAverage * that.Minutes()
 
-	// isShared determines whether the given allocation should be spread evenly
-	// across resources (e.g. sharing idle allocation) or lumped into a shared
-	// cost category (e.g. sharing namespace, labels).
-	if isShared {
-		a.SharedCost += that.TotalCost
-	} else {
-		a.CPUCoreHours += that.CPUCoreHours
-		a.GPUHours += that.GPUHours
-		a.RAMByteHours += that.RAMByteHours
-		a.PVByteHours += that.PVByteHours
-
-		aggCPUCost := a.CPUCost + that.CPUCost
-		if aggCPUCost > 0 {
-			a.CPUEfficiency = (a.CPUEfficiency*a.CPUCost + that.CPUEfficiency*that.CPUCost) / aggCPUCost
-		} else {
-			a.CPUEfficiency = 0.0
-		}
-
-		aggRAMCost := a.RAMCost + that.RAMCost
-		if aggRAMCost > 0 {
-			a.RAMEfficiency = (a.RAMEfficiency*a.RAMCost + that.RAMEfficiency*that.RAMCost) / aggRAMCost
-		} else {
-			a.RAMEfficiency = 0.0
-		}
+	ramUseByteMins := a.RAMBytesUsageAverage * a.Minutes()
+	ramUseByteMins += that.RAMBytesUsageAverage * that.Minutes()
 
-		aggTotalCost := a.TotalCost + that.TotalCost
-		if aggTotalCost > 0 {
-			a.TotalEfficiency = (a.TotalEfficiency*(a.TotalCost-a.ExternalCost) + that.TotalEfficiency*(that.TotalCost-that.ExternalCost)) / (aggTotalCost - a.ExternalCost - that.ExternalCost)
-		} else {
-			aggTotalCost = 0.0
-		}
-
-		a.SharedCost += that.SharedCost
-		a.ExternalCost += that.ExternalCost
-		a.CPUCost += that.CPUCost
-		a.GPUCost += that.GPUCost
-		a.NetworkCost += that.NetworkCost
-		a.RAMCost += that.RAMCost
-		a.PVCost += that.PVCost
+	// Expand Start and End to be the "max" of among the given Allocations
+	if that.Start.Before(a.Start) {
+		a.Start = that.Start
+	}
+	if that.End.After(a.End) {
+		a.End = that.End
 	}
 
-	a.TotalCost += that.TotalCost
+	// Convert cumulative request and usage back into rates
+	// TODO:TEST write a unit test that fails if this is done incorrectly
+	if a.Minutes() > 0 {
+		a.CPUCoreRequestAverage = cpuReqCoreMins / a.Minutes()
+		a.CPUCoreUsageAverage = cpuUseCoreMins / a.Minutes()
+		a.RAMBytesRequestAverage = ramReqByteMins / a.Minutes()
+		a.RAMBytesUsageAverage = ramUseByteMins / a.Minutes()
+	} else {
+		a.CPUCoreRequestAverage = 0.0
+		a.CPUCoreUsageAverage = 0.0
+		a.RAMBytesRequestAverage = 0.0
+		a.RAMBytesUsageAverage = 0.0
+	}
+
+	// Sum all cumulative resource fields
+	a.CPUCoreHours += that.CPUCoreHours
+	a.GPUHours += that.GPUHours
+	a.RAMByteHours += that.RAMByteHours
+	a.PVByteHours += that.PVByteHours
+
+	// Sum all cumulative cost fields
+	a.CPUCost += that.CPUCost
+	a.GPUCost += that.GPUCost
+	a.RAMCost += that.RAMCost
+	a.PVCost += that.PVCost
+	a.NetworkCost += that.NetworkCost
+	a.SharedCost += that.SharedCost
+	a.ExternalCost += that.ExternalCost
 }
 
 // AllocationSet stores a set of Allocations, each with a unique name, that share
@@ -408,9 +505,6 @@ func (as *AllocationSet) AggregateBy(properties Properties, options *AllocationA
 	// 10. If the merge idle option is enabled, merge any remaining idle
 	//     allocations into a single idle allocation
 
-	// TODO niko/etl revisit (ShareIdle: ShareEven) case, which is probably wrong
-	// (and, frankly, ill-defined; i.e. evenly across clusters? within clusters?)
-
 	if options == nil {
 		options = &AllocationAggregationOptions{}
 	}
@@ -674,7 +768,6 @@ func (as *AllocationSet) AggregateBy(properties Properties, options *AllocationA
 				alloc.CPUCost += idleCPUCost
 				alloc.GPUCost += idleGPUCost
 				alloc.RAMCost += idleRAMCost
-				alloc.TotalCost += idleCPUCost + idleGPUCost + idleRAMCost
 			}
 		}
 
@@ -785,7 +878,6 @@ func (as *AllocationSet) AggregateBy(properties Properties, options *AllocationA
 				idleAlloc.CPUCoreHours *= resourceCoeffs["cpu"]
 				idleAlloc.RAMCost *= resourceCoeffs["ram"]
 				idleAlloc.RAMByteHours *= resourceCoeffs["ram"]
-				idleAlloc.TotalCost = idleAlloc.CPUCost + idleAlloc.RAMCost
 			}
 		}
 	}
@@ -799,8 +891,7 @@ func (as *AllocationSet) AggregateBy(properties Properties, options *AllocationA
 					continue
 				}
 
-				alloc.SharedCost += sharedAlloc.TotalCost * shareCoefficients[alloc.Name]
-				alloc.TotalCost += sharedAlloc.TotalCost * shareCoefficients[alloc.Name]
+				alloc.SharedCost += sharedAlloc.TotalCost() * shareCoefficients[alloc.Name]
 			}
 		}
 	}
@@ -892,8 +983,8 @@ func computeShareCoeffs(properties Properties, options *AllocationAggregationOpt
 		} else {
 			// Both are additive for weighted distribution, where each
 			// cumulative coefficient will be divided by the total.
-			coeffs[name] += alloc.TotalCost
-			total += alloc.TotalCost
+			coeffs[name] += alloc.TotalCost()
+			total += alloc.TotalCost()
 		}
 	}
 
@@ -1031,13 +1122,13 @@ func computeIdleCoeffs(properties Properties, options *AllocationAggregationOpti
 	return coeffs, nil
 }
 
-func (alloc *Allocation) generateKey(properties Properties) (string, error) {
+func (a *Allocation) generateKey(properties Properties) (string, error) {
 	// Names will ultimately be joined into a single name, which uniquely
 	// identifies allocations.
 	names := []string{}
 
 	if properties.HasCluster() {
-		cluster, err := alloc.Properties.GetCluster()
+		cluster, err := a.Properties.GetCluster()
 		if err != nil {
 			return "", err
 		}
@@ -1045,7 +1136,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasNode() {
-		node, err := alloc.Properties.GetNode()
+		node, err := a.Properties.GetNode()
 		if err != nil {
 			return "", err
 		}
@@ -1053,7 +1144,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasNamespace() {
-		namespace, err := alloc.Properties.GetNamespace()
+		namespace, err := a.Properties.GetNamespace()
 		if err != nil {
 			return "", err
 		}
@@ -1061,7 +1152,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasControllerKind() {
-		controllerKind, err := alloc.Properties.GetControllerKind()
+		controllerKind, err := a.Properties.GetControllerKind()
 		if err != nil {
 			// Indicate that allocation has no controller
 			controllerKind = UnallocatedSuffix
@@ -1076,13 +1167,13 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 
 	if properties.HasController() {
 		if !properties.HasControllerKind() {
-			controllerKind, err := alloc.Properties.GetControllerKind()
+			controllerKind, err := a.Properties.GetControllerKind()
 			if err == nil {
 				names = append(names, controllerKind)
 			}
 		}
 
-		controller, err := alloc.Properties.GetController()
+		controller, err := a.Properties.GetController()
 		if err != nil {
 			// Indicate that allocation has no controller
 			controller = UnallocatedSuffix
@@ -1092,7 +1183,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasPod() {
-		pod, err := alloc.Properties.GetPod()
+		pod, err := a.Properties.GetPod()
 		if err != nil {
 			return "", err
 		}
@@ -1101,7 +1192,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasContainer() {
-		container, err := alloc.Properties.GetContainer()
+		container, err := a.Properties.GetContainer()
 		if err != nil {
 			return "", err
 		}
@@ -1110,12 +1201,11 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasService() {
-		services, err := alloc.Properties.GetServices()
+		services, err := a.Properties.GetServices()
 		if err != nil {
 			// Indicate that allocation has no services
 			names = append(names, UnallocatedSuffix)
 		} else {
-			// TODO niko/etl support multi-service aggregation
 			if len(services) > 0 {
 				for _, service := range services {
 					names = append(names, service)
@@ -1129,7 +1219,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasAnnotations() {
-		annotations, err := alloc.Properties.GetAnnotations() // annotations that the individual allocation possesses
+		annotations, err := a.Properties.GetAnnotations() // annotations that the individual allocation possesses
 		if err != nil {
 			// Indicate that allocation has no annotations
 			names = append(names, UnallocatedSuffix)
@@ -1165,7 +1255,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasLabel() {
-		labels, err := alloc.Properties.GetLabels() // labels that the individual allocation possesses
+		labels, err := a.Properties.GetLabels() // labels that the individual allocation possesses
 		if err != nil {
 			// Indicate that allocation has no labels
 			names = append(names, UnallocatedSuffix)
@@ -1203,7 +1293,7 @@ func (alloc *Allocation) generateKey(properties Properties) (string, error) {
 	return strings.Join(names, "/"), nil
 }
 
-// TODO clean up
+// TODO:CLEANUP get rid of this
 // Helper function to check for slice membership. Not sure if repeated elsewhere in our codebase.
 func indexOf(v string, arr []string) int {
 	for i, s := range arr {
@@ -1332,7 +1422,7 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 		if s, ok := clusterStarts[cluster]; !ok || a.Start.Before(s) {
 			clusterStarts[cluster] = a.Start
 		}
-		if e, ok := clusterEnds[cluster]; !ok || a.End.Before(e) {
+		if e, ok := clusterEnds[cluster]; !ok || a.End.After(e) {
 			clusterEnds[cluster] = a.End
 		}
 
@@ -1357,15 +1447,14 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 
 		idleAlloc := &Allocation{
 			Name:       fmt.Sprintf("%s/%s", cluster, IdleSuffix),
+			Window:     window.Clone(),
 			Properties: Properties{ClusterProp: cluster},
 			Start:      start,
 			End:        end,
-			Minutes:    end.Sub(start).Minutes(), // TODO deprecate w/ niko/allocation-minutes
 			CPUCost:    resources["cpu"],
 			GPUCost:    resources["gpu"],
 			RAMCost:    resources["ram"],
 		}
-		idleAlloc.TotalCost = idleAlloc.CPUCost + idleAlloc.GPUCost + idleAlloc.RAMCost
 
 		// Do not continue if multiple idle allocations are computed for a
 		// single cluster.
@@ -1491,10 +1580,10 @@ func (as *AllocationSet) IdleAllocations() map[string]*Allocation {
 // but only if the Allocation is valid, i.e. matches the AllocationSet's window. If
 // there is no existing entry, one is created. Nil error response indicates success.
 func (as *AllocationSet) Insert(that *Allocation) error {
-	return as.insert(that, false)
+	return as.insert(that)
 }
 
-func (as *AllocationSet) insert(that *Allocation, accumulate bool) error {
+func (as *AllocationSet) insert(that *Allocation) error {
 	if as == nil {
 		return fmt.Errorf("cannot insert into nil AllocationSet")
 	}
@@ -1519,7 +1608,7 @@ func (as *AllocationSet) insert(that *Allocation, accumulate bool) error {
 	if _, ok := as.allocations[that.Name]; !ok {
 		as.allocations[that.Name] = that
 	} else {
-		as.allocations[that.Name].add(that, false, accumulate)
+		as.allocations[that.Name].add(that)
 	}
 
 	// If the given Allocation is an external one, record that
@@ -1641,7 +1730,7 @@ func (as *AllocationSet) TotalCost() float64 {
 
 	tc := 0.0
 	for _, a := range as.allocations {
-		tc += a.TotalCost
+		tc += a.TotalCost()
 	}
 	return tc
 }
@@ -1661,12 +1750,6 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 		return as, nil
 	}
 
-	if that.Start().Before(as.End()) {
-		timefmt := "2006-01-02T15:04:05"
-		err := fmt.Sprintf("that [%s, %s); that [%s, %s)\n", as.Start().Format(timefmt), as.End().Format(timefmt), that.Start().Format(timefmt), that.End().Format(timefmt))
-		return nil, fmt.Errorf("error accumulating AllocationSets: overlapping windows: %s", err)
-	}
-
 	// Set start, end to min(start), max(end)
 	start := as.Start()
 	end := as.End()
@@ -1686,26 +1769,14 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 	defer that.RUnlock()
 
 	for _, alloc := range as.allocations {
-		// Change Start and End to match the new window. However, do not
-		// change Minutes because that will be accounted for during the
-		// insert step, if in fact there are two allocations to add.
-		alloc.Start = start
-		alloc.End = end
-
-		err := acc.insert(alloc, true)
+		err := acc.insert(alloc)
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	for _, alloc := range that.allocations {
-		// Change Start and End to match the new window. However, do not
-		// change Minutes because that will be accounted for during the
-		// insert step, if in fact there are two allocations to add.
-		alloc.Start = start
-		alloc.End = end
-
-		err := acc.insert(alloc, true)
+		err := acc.insert(alloc)
 		if err != nil {
 			return nil, err
 		}

pkg/kubecost/allocation_test.go

@@ -1,12 +1,13 @@
 package kubecost
 
 import (
+	"encoding/json"
 	"fmt"
 	"math"
 	"testing"
 	"time"
 
-	util "github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 const day = 24 * time.Hour
@@ -32,24 +33,24 @@ func NewUnitAllocation(name string, start time.Time, resolution time.Duration, p
 	end := start.Add(resolution)
 
 	alloc := &Allocation{
-		Name:            name,
-		Properties:      *properties,
-		Start:           start,
-		End:             end,
-		Minutes:         1440,
-		CPUCoreHours:    1,
-		CPUCost:         1,
-		CPUEfficiency:   1,
-		GPUHours:        1,
-		GPUCost:         1,
-		NetworkCost:     1,
-		PVByteHours:     1,
-		PVCost:          1,
-		RAMByteHours:    1,
-		RAMCost:         1,
-		RAMEfficiency:   1,
-		TotalCost:       5,
-		TotalEfficiency: 1,
+		Name:                   name,
+		Properties:             *properties,
+		Window:                 NewWindow(&start, &end).Clone(),
+		Start:                  start,
+		End:                    end,
+		CPUCoreHours:           1,
+		CPUCost:                1,
+		CPUCoreRequestAverage:  1,
+		CPUCoreUsageAverage:    1,
+		GPUHours:               1,
+		GPUCost:                1,
+		NetworkCost:            1,
+		PVByteHours:            1,
+		PVCost:                 1,
+		RAMByteHours:           1,
+		RAMCost:                1,
+		RAMBytesRequestAverage: 1,
+		RAMBytesUsageAverage:   1,
 	}
 
 	// If idle allocation, remove non-idle costs, but maintain total cost
@@ -85,21 +86,356 @@ func TestAllocation_Add(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Allocation.Add unexpected error: %s", err)
 	}
-	if nilZeroSum == nil || nilZeroSum.TotalCost != 0.0 {
+	if nilZeroSum == nil || nilZeroSum.TotalCost() != 0.0 {
 		t.Fatalf("Allocation.Add failed; exp: 0.0; act: %s", nilZeroSum)
 	}
 
-	// TODO niko/etl more
+	cpuPrice := 0.02
+	gpuPrice := 2.00
+	ramPrice := 0.01
+	pvPrice := 0.00005
+	gib := 1024.0 * 1024.0 * 1024.0
+
+	s1 := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC)
+	e1 := time.Date(2021, time.January, 1, 12, 0, 0, 0, time.UTC)
+	hrs1 := e1.Sub(s1).Hours()
+	a1 := &Allocation{
+		Start:                  s1,
+		End:                    e1,
+		CPUCoreHours:           2.0 * hrs1,
+		CPUCoreRequestAverage:  2.0,
+		CPUCoreUsageAverage:    1.0,
+		CPUCost:                2.0 * hrs1 * cpuPrice,
+		GPUHours:               1.0 * hrs1,
+		GPUCost:                1.0 * hrs1 * gpuPrice,
+		PVByteHours:            100.0 * gib * hrs1,
+		PVCost:                 100.0 * hrs1 * pvPrice,
+		RAMByteHours:           8.0 * gib * hrs1,
+		RAMBytesRequestAverage: 8.0 * gib,
+		RAMBytesUsageAverage:   4.0 * gib,
+		RAMCost:                8.0 * hrs1 * ramPrice,
+		SharedCost:             2.00,
+		ExternalCost:           1.00,
+	}
+	a1b := a1.Clone()
+
+	s2 := time.Date(2021, time.January, 1, 6, 0, 0, 0, time.UTC)
+	e2 := time.Date(2021, time.January, 1, 24, 0, 0, 0, time.UTC)
+	hrs2 := e1.Sub(s1).Hours()
+	a2 := &Allocation{
+		Start:                  s2,
+		End:                    e2,
+		CPUCoreHours:           1.0 * hrs2,
+		CPUCoreRequestAverage:  1.0,
+		CPUCoreUsageAverage:    1.0,
+		CPUCost:                1.0 * hrs2 * cpuPrice,
+		GPUHours:               0.0,
+		GPUCost:                0.0,
+		PVByteHours:            0,
+		PVCost:                 0,
+		RAMByteHours:           8.0 * gib * hrs2,
+		RAMBytesRequestAverage: 0.0,
+		RAMBytesUsageAverage:   8.0 * gib,
+		RAMCost:                8.0 * hrs2 * ramPrice,
+		NetworkCost:            0.01,
+		SharedCost:             0.00,
+		ExternalCost:           1.00,
+	}
+	a2b := a2.Clone()
+
+	act, err := a1.Add(a2)
+	if err != nil {
+		t.Fatalf("Allocation.Add: unexpected error: %s", err)
+	}
+
+	// Neither Allocation should be mutated
+	if !a1.Equal(a1b) {
+		t.Fatalf("Allocation.Add: a1 illegally mutated")
+	}
+	if !a2.Equal(a2b) {
+		t.Fatalf("Allocation.Add: a1 illegally mutated")
+	}
+
+	// Costs should be cumulative
+	if !util.IsApproximately(a1.TotalCost()+a2.TotalCost(), act.TotalCost()) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.TotalCost()+a2.TotalCost(), act.TotalCost())
+	}
+	if !util.IsApproximately(a1.CPUCost+a2.CPUCost, act.CPUCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.CPUCost+a2.CPUCost, act.CPUCost)
+	}
+	if !util.IsApproximately(a1.GPUCost+a2.GPUCost, act.GPUCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.GPUCost+a2.GPUCost, act.GPUCost)
+	}
+	if !util.IsApproximately(a1.RAMCost+a2.RAMCost, act.RAMCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.RAMCost+a2.RAMCost, act.RAMCost)
+	}
+	if !util.IsApproximately(a1.PVCost+a2.PVCost, act.PVCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.PVCost+a2.PVCost, act.PVCost)
+	}
+	if !util.IsApproximately(a1.NetworkCost+a2.NetworkCost, act.NetworkCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.NetworkCost+a2.NetworkCost, act.NetworkCost)
+	}
+	if !util.IsApproximately(a1.SharedCost+a2.SharedCost, act.SharedCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.SharedCost+a2.SharedCost, act.SharedCost)
+	}
+	if !util.IsApproximately(a1.ExternalCost+a2.ExternalCost, act.ExternalCost) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.ExternalCost+a2.ExternalCost, act.ExternalCost)
+	}
+
+	// ResourceHours should be cumulative
+	if !util.IsApproximately(a1.CPUCoreHours+a2.CPUCoreHours, act.CPUCoreHours) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.CPUCoreHours+a2.CPUCoreHours, act.CPUCoreHours)
+	}
+	if !util.IsApproximately(a1.RAMByteHours+a2.RAMByteHours, act.RAMByteHours) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.RAMByteHours+a2.RAMByteHours, act.RAMByteHours)
+	}
+	if !util.IsApproximately(a1.PVByteHours+a2.PVByteHours, act.PVByteHours) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", a1.PVByteHours+a2.PVByteHours, act.PVByteHours)
+	}
+
+	// Minutes should be the duration between min(starts) and max(ends)
+	if !act.Start.Equal(a1.Start) || !act.End.Equal(a2.End) {
+		t.Fatalf("Allocation.Add: expected %s; actual %s", NewWindow(&a1.Start, &a2.End), NewWindow(&act.Start, &act.End))
+	}
+	if act.Minutes() != 1440.0 {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 1440.0, act.Minutes())
+	}
+
+	// Requests and Usage should be averaged correctly
+	// CPU requests = (2.0*12.0 + 1.0*18.0)/(24.0) = 1.75
+	// CPU usage = (1.0*12.0 + 1.0*18.0)/(24.0) = 1.25
+	// RAM requests = (8.0*12.0 + 0.0*18.0)/(24.0) = 4.00
+	// RAM usage = (4.0*12.0 + 8.0*18.0)/(24.0) = 8.00
+	if !util.IsApproximately(1.75, act.CPUCoreRequestAverage) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 1.75, act.CPUCoreRequestAverage)
+	}
+	if !util.IsApproximately(1.25, act.CPUCoreUsageAverage) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 1.25, act.CPUCoreUsageAverage)
+	}
+	if !util.IsApproximately(4.00*gib, act.RAMBytesRequestAverage) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 4.00*gib, act.RAMBytesRequestAverage)
+	}
+	if !util.IsApproximately(8.00*gib, act.RAMBytesUsageAverage) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 8.00*gib, act.RAMBytesUsageAverage)
+	}
+
+	// Efficiency should be computed accurately from new request/usage
+	// CPU efficiency = 1.25/1.75 = 0.7142857
+	// RAM efficiency = 8.00/4.00 = 2.0000000
+	// Total efficiency = (0.7142857*0.72 + 2.0*1.92)/(2.64) = 1.6493506
+	if !util.IsApproximately(0.7142857, act.CPUEfficiency()) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 0.7142857, act.CPUEfficiency())
+	}
+	if !util.IsApproximately(2.0000000, act.RAMEfficiency()) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 2.0000000, act.RAMEfficiency())
+	}
+	if !util.IsApproximately(1.6493506, act.TotalEfficiency()) {
+		t.Fatalf("Allocation.Add: expected %f; actual %f", 1.6493506, act.TotalEfficiency())
+	}
 }
 
-// TODO niko/etl
-// func TestAllocation_Clone(t *testing.T) {}
+func TestAllocation_Share(t *testing.T) {
+	cpuPrice := 0.02
+	gpuPrice := 2.00
+	ramPrice := 0.01
+	pvPrice := 0.00005
+	gib := 1024.0 * 1024.0 * 1024.0
+
+	s1 := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC)
+	e1 := time.Date(2021, time.January, 1, 12, 0, 0, 0, time.UTC)
+	hrs1 := e1.Sub(s1).Hours()
+	a1 := &Allocation{
+		Start:                  s1,
+		End:                    e1,
+		CPUCoreHours:           2.0 * hrs1,
+		CPUCoreRequestAverage:  2.0,
+		CPUCoreUsageAverage:    1.0,
+		CPUCost:                2.0 * hrs1 * cpuPrice,
+		GPUHours:               1.0 * hrs1,
+		GPUCost:                1.0 * hrs1 * gpuPrice,
+		PVByteHours:            100.0 * gib * hrs1,
+		PVCost:                 100.0 * hrs1 * pvPrice,
+		RAMByteHours:           8.0 * gib * hrs1,
+		RAMBytesRequestAverage: 8.0 * gib,
+		RAMBytesUsageAverage:   4.0 * gib,
+		RAMCost:                8.0 * hrs1 * ramPrice,
+		SharedCost:             2.00,
+		ExternalCost:           1.00,
+	}
+	a1b := a1.Clone()
+
+	s2 := time.Date(2021, time.January, 1, 6, 0, 0, 0, time.UTC)
+	e2 := time.Date(2021, time.January, 1, 24, 0, 0, 0, time.UTC)
+	hrs2 := e1.Sub(s1).Hours()
+	a2 := &Allocation{
+		Start:                  s2,
+		End:                    e2,
+		CPUCoreHours:           1.0 * hrs2,
+		CPUCoreRequestAverage:  1.0,
+		CPUCoreUsageAverage:    1.0,
+		CPUCost:                1.0 * hrs2 * cpuPrice,
+		GPUHours:               0.0,
+		GPUCost:                0.0,
+		PVByteHours:            0,
+		PVCost:                 0,
+		RAMByteHours:           8.0 * gib * hrs2,
+		RAMBytesRequestAverage: 0.0,
+		RAMBytesUsageAverage:   8.0 * gib,
+		RAMCost:                8.0 * hrs2 * ramPrice,
+		NetworkCost:            0.01,
+		SharedCost:             0.00,
+		ExternalCost:           1.00,
+	}
+	a2b := a2.Clone()
+
+	act, err := a1.Share(a2)
+	if err != nil {
+		t.Fatalf("Allocation.Share: unexpected error: %s", err)
+	}
 
-// TODO niko/etl
-// func TestAllocation_IsIdle(t *testing.T) {}
+	// Neither Allocation should be mutated
+	if !a1.Equal(a1b) {
+		t.Fatalf("Allocation.Share: a1 illegally mutated")
+	}
+	if !a2.Equal(a2b) {
+		t.Fatalf("Allocation.Share: a1 illegally mutated")
+	}
 
-func TestAllocation_String(t *testing.T) {
-	// TODO niko/etl
+	// SharedCost and TotalCost should reflect increase by a2.TotalCost
+	if !util.IsApproximately(a1.TotalCost()+a2.TotalCost(), act.TotalCost()) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.TotalCost()+a2.TotalCost(), act.TotalCost())
+	}
+	if !util.IsApproximately(a1.SharedCost+a2.TotalCost(), act.SharedCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.SharedCost+a2.TotalCost(), act.SharedCost)
+	}
+
+	// Costs should match before (expect TotalCost and SharedCost)
+	if !util.IsApproximately(a1.CPUCost, act.CPUCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.CPUCost, act.CPUCost)
+	}
+	if !util.IsApproximately(a1.GPUCost, act.GPUCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.GPUCost, act.GPUCost)
+	}
+	if !util.IsApproximately(a1.RAMCost, act.RAMCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.RAMCost, act.RAMCost)
+	}
+	if !util.IsApproximately(a1.PVCost, act.PVCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.PVCost, act.PVCost)
+	}
+	if !util.IsApproximately(a1.NetworkCost, act.NetworkCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.NetworkCost, act.NetworkCost)
+	}
+	if !util.IsApproximately(a1.ExternalCost, act.ExternalCost) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.ExternalCost, act.ExternalCost)
+	}
+
+	// ResourceHours should match before
+	if !util.IsApproximately(a1.CPUCoreHours, act.CPUCoreHours) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.CPUCoreHours, act.CPUCoreHours)
+	}
+	if !util.IsApproximately(a1.RAMByteHours, act.RAMByteHours) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.RAMByteHours, act.RAMByteHours)
+	}
+	if !util.IsApproximately(a1.PVByteHours, act.PVByteHours) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.PVByteHours, act.PVByteHours)
+	}
+
+	// Minutes should match before
+	if !act.Start.Equal(a1.Start) || !act.End.Equal(a1.End) {
+		t.Fatalf("Allocation.Share: expected %s; actual %s", NewWindow(&a1.Start, &a1.End), NewWindow(&act.Start, &act.End))
+	}
+	if act.Minutes() != a1.Minutes() {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.Minutes(), act.Minutes())
+	}
+
+	// Requests and Usage should match before
+	if !util.IsApproximately(a1.CPUCoreRequestAverage, act.CPUCoreRequestAverage) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.CPUCoreRequestAverage, act.CPUCoreRequestAverage)
+	}
+	if !util.IsApproximately(a1.CPUCoreUsageAverage, act.CPUCoreUsageAverage) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.CPUCoreUsageAverage, act.CPUCoreUsageAverage)
+	}
+	if !util.IsApproximately(a1.RAMBytesRequestAverage, act.RAMBytesRequestAverage) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.RAMBytesRequestAverage, act.RAMBytesRequestAverage)
+	}
+	if !util.IsApproximately(a1.RAMBytesUsageAverage, act.RAMBytesUsageAverage) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.RAMBytesUsageAverage, act.RAMBytesUsageAverage)
+	}
+
+	// Efficiency should match before
+	if !util.IsApproximately(a1.CPUEfficiency(), act.CPUEfficiency()) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.CPUEfficiency(), act.CPUEfficiency())
+	}
+	if !util.IsApproximately(a1.RAMEfficiency(), act.RAMEfficiency()) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.RAMEfficiency(), act.RAMEfficiency())
+	}
+	if !util.IsApproximately(a1.TotalEfficiency(), act.TotalEfficiency()) {
+		t.Fatalf("Allocation.Share: expected %f; actual %f", a1.TotalEfficiency(), act.TotalEfficiency())
+	}
+}
+
+func TestAllocation_MarshalJSON(t *testing.T) {
+	start := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC)
+	end := time.Date(2021, time.January, 2, 0, 0, 0, 0, time.UTC)
+	hrs := 24.0
+
+	gib := 1024.0 * 1024.0 * 1024.0
+
+	cpuPrice := 0.02
+	gpuPrice := 2.00
+	ramPrice := 0.01
+	pvPrice := 0.00005
+
+	before := &Allocation{
+		Name: "cluster1/namespace1/node1/pod1/container1",
+		Properties: Properties{
+			ClusterProp:   "cluster1",
+			NodeProp:      "node1",
+			NamespaceProp: "namespace1",
+			PodProp:       "pod1",
+			ContainerProp: "container1",
+		},
+		Window:                 NewWindow(&start, &end),
+		Start:                  start,
+		End:                    end,
+		CPUCoreHours:           2.0 * hrs,
+		CPUCoreRequestAverage:  2.0,
+		CPUCoreUsageAverage:    1.0,
+		CPUCost:                2.0 * hrs * cpuPrice,
+		GPUHours:               1.0 * hrs,
+		GPUCost:                1.0 * hrs * gpuPrice,
+		NetworkCost:            0.05,
+		PVByteHours:            100.0 * gib * hrs,
+		PVCost:                 100.0 * hrs * pvPrice,
+		RAMByteHours:           8.0 * gib * hrs,
+		RAMBytesRequestAverage: 8.0 * gib,
+		RAMBytesUsageAverage:   4.0 * gib,
+		RAMCost:                8.0 * hrs * ramPrice,
+		SharedCost:             2.00,
+		ExternalCost:           1.00,
+	}
+
+	data, err := json.Marshal(before)
+	if err != nil {
+		t.Fatalf("Allocation.MarshalJSON: unexpected error: %s", err)
+	}
+
+	after := &Allocation{}
+	err = json.Unmarshal(data, after)
+	if err != nil {
+		t.Fatalf("Allocation.UnmarshalJSON: unexpected error: %s", err)
+	}
+
+	// TODO:CLEANUP fix json marshaling of Window so that all of this works.
+	// In the meantime, just set the Window so that we can test the rest.
+	after.Window = before.Window.Clone()
+
+	fmt.Println(*before)
+	fmt.Println(*after)
+
+	if !after.Equal(before) {
+		t.Fatalf("Allocation.MarshalJSON: before and after are not equal")
+	}
 }
 
 func TestNewAllocationSet(t *testing.T) {
@@ -115,7 +451,6 @@ func generateAllocationSet(start time.Time) *AllocationSet {
 	a1i.CPUCost = 5.0
 	a1i.RAMCost = 15.0
 	a1i.GPUCost = 0.0
-	a1i.TotalCost = 20.0
 
 	a2i := NewUnitAllocation(fmt.Sprintf("cluster2/%s", IdleSuffix), start, day, &Properties{
 		ClusterProp: "cluster2",
@@ -123,7 +458,6 @@ func generateAllocationSet(start time.Time) *AllocationSet {
 	a2i.CPUCost = 5.0
 	a2i.RAMCost = 5.0
 	a2i.GPUCost = 0.0
-	a2i.TotalCost = 10.0
 
 	// Active allocations
 	a1111 := NewUnitAllocation("cluster1/namespace1/pod1/container1", start, day, &Properties{
@@ -133,7 +467,6 @@ func generateAllocationSet(start time.Time) *AllocationSet {
 		ContainerProp: "container1",
 	})
 	a1111.RAMCost = 11.00
-	a1111.TotalCost = 15.00
 
 	a11abc2 := NewUnitAllocation("cluster1/namespace1/pod-abc/container2", start, day, &Properties{
 		ClusterProp:   "cluster1",
@@ -288,8 +621,8 @@ func assertAllocationSetTotals(t *testing.T, as *AllocationSet, msg string, err
 func assertAllocationTotals(t *testing.T, as *AllocationSet, msg string, exps map[string]float64) {
 	as.Each(func(k string, a *Allocation) {
 		if exp, ok := exps[a.Name]; ok {
-			if math.Round(a.TotalCost*100) != math.Round(exp*100) {
-				t.Fatalf("AllocationSet.AggregateBy[%s]: expected total cost %.2f, actual %.2f", msg, exp, a.TotalCost)
+			if math.Round(a.TotalCost()*100) != math.Round(exp*100) {
+				t.Fatalf("AllocationSet.AggregateBy[%s]: expected total cost %.2f, actual %.2f", msg, exp, a.TotalCost())
 			}
 		} else {
 			t.Fatalf("AllocationSet.AggregateBy[%s]: unexpected allocation: %s", msg, a.Name)
@@ -305,8 +638,8 @@ func assertAllocationWindow(t *testing.T, as *AllocationSet, msg string, expStar
 		if !a.End.Equal(expEnd) {
 			t.Fatalf("AllocationSet.AggregateBy[%s]: expected end %s, actual %s", msg, expEnd, a.End)
 		}
-		if a.Minutes != expMinutes {
-			t.Fatalf("AllocationSet.AggregateBy[%s]: expected minutes %f, actual %f", msg, expMinutes, a.Minutes)
+		if a.Minutes() != expMinutes {
+			t.Fatalf("AllocationSet.AggregateBy[%s]: expected minutes %f, actual %f", msg, expMinutes, a.Minutes())
 		}
 	})
 }
@@ -1090,8 +1423,8 @@ func TestAllocationSet_ComputeIdleAllocations(t *testing.T) {
 	if idle, ok := idles["cluster1"]; !ok {
 		t.Fatalf("expected idle cost for %s", "cluster1")
 	} else {
-		if !util.IsApproximately(idle.TotalCost, 72.0) {
-			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster1", 72.0, idle.TotalCost)
+		if !util.IsApproximately(idle.TotalCost(), 72.0) {
+			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster1", 72.0, idle.TotalCost())
 		}
 	}
 	if !util.IsApproximately(idles["cluster1"].CPUCost, 44.0) {
@@ -1107,8 +1440,8 @@ func TestAllocationSet_ComputeIdleAllocations(t *testing.T) {
 	if idle, ok := idles["cluster2"]; !ok {
 		t.Fatalf("expected idle cost for %s", "cluster2")
 	} else {
-		if !util.IsApproximately(idle.TotalCost, 82.0) {
-			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster2", 82.0, idle.TotalCost)
+		if !util.IsApproximately(idle.TotalCost(), 82.0) {
+			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster2", 82.0, idle.TotalCost())
 		}
 	}
 
@@ -1177,8 +1510,8 @@ func TestAllocationSet_ComputeIdleAllocations(t *testing.T) {
 	if idle, ok := idles["cluster1"]; !ok {
 		t.Fatalf("expected idle cost for %s", "cluster1")
 	} else {
-		if !util.IsApproximately(idle.TotalCost, 72.0) {
-			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster1", 72.0, idle.TotalCost)
+		if !util.IsApproximately(idle.TotalCost(), 72.0) {
+			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster1", 72.0, idle.TotalCost())
 		}
 	}
 	if !util.IsApproximately(idles["cluster1"].CPUCost, 44.0) {
@@ -1194,8 +1527,8 @@ func TestAllocationSet_ComputeIdleAllocations(t *testing.T) {
 	if idle, ok := idles["cluster2"]; !ok {
 		t.Fatalf("expected idle cost for %s", "cluster2")
 	} else {
-		if !util.IsApproximately(idle.TotalCost, 82.0) {
-			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster2", 82.0, idle.TotalCost)
+		if !util.IsApproximately(idle.TotalCost(), 82.0) {
+			t.Fatalf("%s idle: expected total cost %f; got total cost %f", "cluster2", 82.0, idle.TotalCost())
 		}
 	}
 
@@ -1348,8 +1681,8 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if alloc.CPUCost != 2.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 2.0; actual %f", alloc.CPUCost)
 	}
-	if alloc.CPUEfficiency != 1.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.CPUEfficiency)
+	if alloc.CPUEfficiency() != 1.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.CPUEfficiency())
 	}
 	if alloc.GPUHours != 2.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 2.0; actual %f", alloc.GPUHours)
@@ -1372,14 +1705,14 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if alloc.RAMCost != 2.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 2.0; actual %f", alloc.RAMCost)
 	}
-	if alloc.RAMEfficiency != 1.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.RAMEfficiency)
+	if alloc.RAMEfficiency() != 1.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.RAMEfficiency())
 	}
-	if alloc.TotalCost != 10.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 10.0; actual %f", alloc.TotalCost)
+	if alloc.TotalCost() != 10.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 10.0; actual %f", alloc.TotalCost())
 	}
-	if alloc.TotalEfficiency != 1.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.TotalEfficiency)
+	if alloc.TotalEfficiency() != 1.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.TotalEfficiency())
 	}
 	if !alloc.Start.Equal(yesterday) {
 		t.Fatalf("accumulating AllocationSetRange: expected to start %s; actual %s", yesterday, alloc.Start)
@@ -1387,8 +1720,8 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if !alloc.End.Equal(tomorrow) {
 		t.Fatalf("accumulating AllocationSetRange: expected to end %s; actual %s", tomorrow, alloc.End)
 	}
-	if alloc.Minutes != 2880.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected %f minutes; actual %f", 2880.0, alloc.Minutes)
+	if alloc.Minutes() != 2880.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected %f minutes; actual %f", 2880.0, alloc.Minutes())
 	}
 }
 
@@ -1477,8 +1810,8 @@ func TestAllocationSetRange_InsertRange(t *testing.T) {
 			if !util.IsApproximately(a.NetworkCost, unit.NetworkCost) {
 				t.Fatalf("allocation %s: expected %f; got %f", k, unit.NetworkCost, a.NetworkCost)
 			}
-			if !util.IsApproximately(a.TotalCost, unit.TotalCost) {
-				t.Fatalf("allocation %s: expected %f; got %f", k, unit.TotalCost, a.TotalCost)
+			if !util.IsApproximately(a.TotalCost(), unit.TotalCost()) {
+				t.Fatalf("allocation %s: expected %f; got %f", k, unit.TotalCost(), a.TotalCost())
 			}
 		})
 	})
@@ -1525,8 +1858,8 @@ func TestAllocationSetRange_InsertRange(t *testing.T) {
 		if !util.IsApproximately(a.NetworkCost, 2*unit.NetworkCost) {
 			t.Fatalf("allocation %s: expected %f; got %f", k, unit.NetworkCost, a.NetworkCost)
 		}
-		if !util.IsApproximately(a.TotalCost, 2*unit.TotalCost) {
-			t.Fatalf("allocation %s: expected %f; got %f", k, unit.TotalCost, a.TotalCost)
+		if !util.IsApproximately(a.TotalCost(), 2*unit.TotalCost()) {
+			t.Fatalf("allocation %s: expected %f; got %f", k, unit.TotalCost(), a.TotalCost())
 		}
 	})
 	tAS, err := thisASR.Get(1)
@@ -1558,8 +1891,8 @@ func TestAllocationSetRange_InsertRange(t *testing.T) {
 		if !util.IsApproximately(a.NetworkCost, unit.NetworkCost) {
 			t.Fatalf("allocation %s: expected %f; got %f", k, unit.NetworkCost, a.NetworkCost)
 		}
-		if !util.IsApproximately(a.TotalCost, unit.TotalCost) {
-			t.Fatalf("allocation %s: expected %f; got %f", k, unit.TotalCost, a.TotalCost)
+		if !util.IsApproximately(a.TotalCost(), unit.TotalCost()) {
+			t.Fatalf("allocation %s: expected %f; got %f", k, unit.TotalCost(), a.TotalCost())
 		}
 	})
 }

pkg/kubecost/asset.go

@@ -5,7 +5,6 @@ import (
 	"encoding"
 	"encoding/json"
 	"fmt"
-	"math"
 	"strings"
 	"sync"
 	"time"
@@ -43,10 +42,10 @@ type Asset interface {
 	// Temporal values
 	Start() time.Time
 	End() time.Time
-	Minutes() float64
+	SetStartEnd(time.Time, time.Time)
 	Window() Window
 	ExpandWindow(Window)
-	SetStartEnd(time.Time, time.Time)
+	Minutes() float64
 
 	// Operations and comparisons
 	Add(Asset) Asset
@@ -217,8 +216,10 @@ func AssetToExternalAllocation(asset Asset, aggregateBy []string) (*Allocation,
 	return &Allocation{
 		Name:         strings.Join(names, "/"),
 		Properties:   props,
+		Window:       asset.Window().Clone(),
+		Start:        asset.Start(),
+		End:          asset.End(),
 		ExternalCost: asset.TotalCost(),
-		TotalCost:    asset.TotalCost(),
 	}, nil
 }
 
@@ -2724,15 +2725,17 @@ func (as *AssetSet) Get(key string) (Asset, bool) {
 // configured properties to determine the key under which the Asset will
 // be inserted.
 func (as *AssetSet) Insert(asset Asset) error {
-	if as.IsEmpty() {
-		as.Lock()
-		as.assets = map[string]Asset{}
-		as.Unlock()
+	if as == nil {
+		return fmt.Errorf("cannot Insert into nil AssetSet")
 	}
 
 	as.Lock()
 	defer as.Unlock()
 
+	if as.assets == nil {
+		as.assets = map[string]Asset{}
+	}
+
 	// Determine key into which to Insert the Asset.
 	k, err := key(asset, as.aggregateBy)
 	if err != nil {
@@ -2855,9 +2858,11 @@ func (as *AssetSet) accumulate(that *AssetSet) (*AssetSet, error) {
 	// Set start, end to min(start), max(end)
 	start := as.Start()
 	end := as.End()
+
 	if that.Start().Before(start) {
 		start = that.Start()
 	}
+
 	if that.End().After(end) {
 		end = that.End()
 	}
@@ -3015,33 +3020,6 @@ func (asr *AssetSetRange) Window() Window {
 	return NewWindow(&start, &end)
 }
 
-// TODO move everything below to a separate package
-
-func jsonEncodeFloat64(buffer *bytes.Buffer, name string, val float64, comma string) {
-	var encoding string
-	if math.IsNaN(val) {
-		encoding = fmt.Sprintf("\"%s\":null%s", name, comma)
-	} else {
-		encoding = fmt.Sprintf("\"%s\":%f%s", name, val, comma)
-	}
-
-	buffer.WriteString(encoding)
-}
-
-func jsonEncodeString(buffer *bytes.Buffer, name, val, comma string) {
-	buffer.WriteString(fmt.Sprintf("\"%s\":\"%s\"%s", name, val, comma))
-}
-
-func jsonEncode(buffer *bytes.Buffer, name string, obj interface{}, comma string) {
-	buffer.WriteString(fmt.Sprintf("\"%s\":", name))
-	if bytes, err := json.Marshal(obj); err != nil {
-		buffer.WriteString("null")
-	} else {
-		buffer.Write(bytes)
-	}
-	buffer.WriteString(comma)
-}
-
 // Returns true if string slices a and b contain all of the same strings, in any order.
 func sameContents(a, b []string) bool {
 	if len(a) != len(b) {

pkg/kubecost/asset_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	util "github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 var start1 = time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
@@ -153,7 +153,7 @@ func assertAssetSet(t *testing.T, as *AssetSet, msg string, window Window, exps
 				t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected total cost %.2f, actual %.2f", msg, key, exp, a.TotalCost())
 			}
 			if !a.Window().Equal(window) {
-				t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected window %s, actual %s", msg, key, window, as.Window)
+				t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected window %s, actual %s", msg, key, window, a.Window())
 			}
 		} else {
 			t.Fatalf("AssetSet.AggregateBy[%s]: unexpected asset: %s", msg, key)
@@ -1010,6 +1010,7 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 		generateAssetSet(startD1),
 		generateAssetSet(startD2),
 	)
+
 	err = asr.AggregateBy([]string{string(AssetTypeProp)}, nil)
 	as, err = asr.Accumulate()
 	if err != nil {
@@ -1093,8 +1094,8 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	if alloc.ExternalCost != 10.00 {
 		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
 	}
-	if alloc.TotalCost != 10.00 {
-		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost)
+	if alloc.TotalCost() != 10.00 {
+		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
 	}
 
 	// 2) multi-prop full match
@@ -1114,8 +1115,8 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	if alloc.ExternalCost != 10.00 {
 		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
 	}
-	if alloc.TotalCost != 10.00 {
-		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost)
+	if alloc.TotalCost() != 10.00 {
+		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
 	}
 
 	// 3) multi-prop partial match
@@ -1132,8 +1133,8 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	if alloc.ExternalCost != 10.00 {
 		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
 	}
-	if alloc.TotalCost != 10.00 {
-		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost)
+	if alloc.TotalCost() != 10.00 {
+		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
 	}
 
 	// 3) no match

pkg/kubecost/bingen.go

@@ -21,4 +21,4 @@ package kubecost
 // @bingen:generate:AllocationSet
 // @bingen:generate:AllocationSetRange
 
-//go:generate bingen -package=kubecost -version=5 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=8 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/json.go

@@ -0,0 +1,35 @@
+package kubecost
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math"
+)
+
+// TODO move everything below to a separate package
+
+func jsonEncodeFloat64(buffer *bytes.Buffer, name string, val float64, comma string) {
+	var encoding string
+	if math.IsNaN(val) || math.IsInf(val, 0) {
+		encoding = fmt.Sprintf("\"%s\":null%s", name, comma)
+	} else {
+		encoding = fmt.Sprintf("\"%s\":%f%s", name, val, comma)
+	}
+
+	buffer.WriteString(encoding)
+}
+
+func jsonEncodeString(buffer *bytes.Buffer, name, val, comma string) {
+	buffer.WriteString(fmt.Sprintf("\"%s\":\"%s\"%s", name, val, comma))
+}
+
+func jsonEncode(buffer *bytes.Buffer, name string, obj interface{}, comma string) {
+	buffer.WriteString(fmt.Sprintf("\"%s\":", name))
+	if bytes, err := json.Marshal(obj); err != nil {
+		buffer.WriteString("null")
+	} else {
+		buffer.Write(bytes)
+	}
+	buffer.WriteString(comma)
+}

pkg/kubecost/properties.go

@@ -5,7 +5,7 @@ import (
 	"sort"
 	"strings"
 
-	util "github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 type Property string
@@ -57,6 +57,7 @@ type PropertyValue struct {
 }
 
 // Properties describes a set of Kubernetes objects.
+// TODO:CLEANUP make this a struct smdh
 type Properties map[Property]interface{}
 
 // TODO niko/etl make sure Services deep copy works correctly

pkg/kubecost/window.go

@@ -8,6 +8,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/thanos"
 	"github.com/kubecost/cost-model/pkg/util"
 )
 
@@ -393,7 +395,19 @@ func (w Window) ExpandEnd(end time.Time) Window {
 }
 
 func (w Window) Expand(that Window) Window {
-	return w.ExpandStart(*that.start).ExpandEnd(*that.end)
+	if that.start == nil {
+		w.start = nil
+	} else {
+		w = w.ExpandStart(*that.start)
+	}
+
+	if that.end == nil {
+		w.end = nil
+	} else {
+		w = w.ExpandEnd(*that.end)
+	}
+
+	return w
 }
 
 func (w Window) Hours() float64 {
@@ -416,10 +430,11 @@ func (w Window) IsOpen() bool {
 	return w.start == nil || w.end == nil
 }
 
+// TODO:CLEANUP make this unmarshalable (make Start and End public)
 func (w Window) MarshalJSON() ([]byte, error) {
 	buffer := bytes.NewBufferString("{")
-	buffer.WriteString(fmt.Sprintf("\"start\":\"%s\",", w.start.Format("2006-01-02T15:04:05-0700")))
-	buffer.WriteString(fmt.Sprintf("\"end\":\"%s\"", w.end.Format("2006-01-02T15:04:05-0700")))
+	buffer.WriteString(fmt.Sprintf("\"start\":\"%s\",", w.start.Format(time.RFC3339)))
+	buffer.WriteString(fmt.Sprintf("\"end\":\"%s\"", w.end.Format(time.RFC3339)))
 	buffer.WriteString("}")
 	return buffer.Bytes(), nil
 }
@@ -432,6 +447,86 @@ func (w Window) Minutes() float64 {
 	return w.end.Sub(*w.start).Minutes()
 }
 
+// Overlaps returns true iff the two given Windows share an amount of temporal
+// coverage.
+// TODO complete (with unit tests!) and then implement in AllocationSet.accumulate
+// TODO:CLEANUP
+// func (w Window) Overlaps(x Window) bool {
+// 	if (w.start == nil && w.end == nil) || (x.start == nil && x.end == nil) {
+// 		// one window is completely open, so overlap is guaranteed
+// 		// <---------->
+// 		//   ?------?
+// 		return true
+// 	}
+
+// 	// Neither window is completely open (nil, nil), but one or the other might
+// 	// still be future- or past-open.
+
+// 	if w.start == nil {
+// 		// w is past-open, future-closed
+// 		// <------]
+
+// 		if x.start != nil && !x.start.Before(*w.end) {
+// 			// x starts after w ends (or eq)
+// 			// <------]
+// 			//          [------?
+// 			return false
+// 		}
+
+// 		// <-----]
+// 		//    ?-----?
+// 		return true
+// 	}
+
+// 	if w.end == nil {
+// 		// w is future-open, past-closed
+// 		// [------>
+
+// 		if x.end != nil && !x.end.After(*w.end) {
+// 			// x ends before w begins (or eq)
+// 			//          [------>
+// 			// ?------]
+// 			return false
+// 		}
+
+// 		//    [------>
+// 		// ?------?
+// 		return true
+// 	}
+
+// 	// Now we know w is closed, but we don't know about x
+// 	//  [------]
+// 	//     ?------?
+// 	if x.start == nil {
+// 		// TODO
+// 	}
+
+// 	if x.end == nil {
+// 		// TODO
+// 	}
+
+// 	// Both are closed.
+
+// 	if !x.start.Before(*w.end) && !x.end.Before(*w.end) {
+// 		// x starts and ends after w ends
+// 		// [------]
+// 		//          [------]
+// 		return false
+// 	}
+
+// 	if !x.start.After(*w.start) && !x.end.After(*w.start) {
+// 		// x starts and ends before w starts
+// 		//          [------]
+// 		// [------]
+// 		return false
+// 	}
+
+// 	// w and x must overlap
+// 	//    [------]
+// 	// [------]
+// 	return true
+// }
+
 func (w Window) Set(start, end *time.Time) {
 	w.start = start
 	w.end = end
@@ -482,6 +577,50 @@ func (w Window) DurationOffset() (time.Duration, time.Duration, error) {
 	return duration, offset, nil
 }
 
+// DurationOffsetForPrometheus returns strings representing durations for the
+// duration and offset of the given window, factoring in the Thanos offset if
+// necessary. Whereas duration is a simple duration string (e.g. "1d"), the
+// offset includes the word "offset" (e.g. " offset 2d") so that the values
+// returned can be used directly in the formatting string "some_metric[%s]%s"
+// to generate the query "some_metric[1d] offset 2d".
+func (w Window) DurationOffsetForPrometheus() (string, string, error) {
+	duration, offset, err := w.DurationOffset()
+	if err != nil {
+		return "", "", err
+	}
+
+	// If using Thanos, increase offset to 3 hours, reducing the duration by
+	// equal measure to maintain the same starting point.
+	thanosDur := thanos.OffsetDuration()
+	if offset < thanosDur && env.IsThanosEnabled() {
+		diff := thanosDur - offset
+		offset += diff
+		duration -= diff
+	}
+
+	// If duration < 0, return an error
+	if duration < 0 {
+		return "", "", fmt.Errorf("negative duration: %s", duration)
+	}
+
+	// Negative offset means that the end time is in the future. Prometheus
+	// fails for non-positive offset values, so shrink the duration and
+	// remove the offset altogether.
+	if offset < 0 {
+		duration = duration + offset
+		offset = 0
+	}
+
+	durStr, offStr := util.DurationOffsetStrings(duration, offset)
+	if offset < time.Minute {
+		offStr = ""
+	} else {
+		offStr = " offset " + offStr
+	}
+
+	return durStr, offStr, nil
+}
+
 // DurationOffsetStrings returns formatted, Prometheus-compatible strings representing
 // the duration and offset of the window in terms of days, hours, minutes, or seconds;
 // e.g. ("7d", "1441m", "30m", "1s", "")

pkg/kubecost/window_test.go

@@ -2,8 +2,11 @@ package kubecost
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 	"time"
+
+	"github.com/kubecost/cost-model/pkg/env"
 )
 
 func TestRoundBack(t *testing.T) {
@@ -211,7 +214,7 @@ func TestParseWindowUTC(t *testing.T) {
 		t.Fatalf(`expect: window "month" to end before now; actual: %s ends after %s`, month, time.Now().UTC())
 	}
 
-	// TODO niko/etl lastweek
+	// TODO lastweek
 
 	lastmonth, err := ParseWindowUTC("lastmonth")
 	monthMinHours := float64(24 * 28)
@@ -542,30 +545,6 @@ func TestParseWindowWithOffsetString(t *testing.T) {
 
 }
 
-// TODO niko/etl
-// func TestWindow_Contains(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Duration(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_End(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Equal(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_ExpandStart(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_ExpandEnd(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Start(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_String(t *testing.T) {}
-
 func TestWindow_DurationOffsetStrings(t *testing.T) {
 	w, err := ParseWindowUTC("1d")
 	if err != nil {
@@ -624,3 +603,171 @@ func TestWindow_DurationOffsetStrings(t *testing.T) {
 		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
 	}
 }
+
+func TestWindow_DurationOffsetForPrometheus(t *testing.T) {
+	// Set-up and tear-down
+	thanosEnabled := env.GetBool(env.ThanosEnabledEnvVar, false)
+	defer env.SetBool(env.ThanosEnabledEnvVar, thanosEnabled)
+
+	// Test for Prometheus (env.IsThanosEnabled() == false)
+	env.SetBool(env.ThanosEnabledEnvVar, false)
+	if env.IsThanosEnabled() {
+		t.Fatalf("expected env.IsThanosEnabled() == false")
+	}
+
+	w, err := ParseWindowUTC("1d")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1d": %s`, err)
+	}
+	dur, off, err := w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1d" {
+		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
+	}
+	if off != "" {
+		t.Fatalf(`expect: offset to be ""; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("2h")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "2h": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "2h" {
+		t.Fatalf(`expect: window to be "2h"; actual: "%s"`, dur)
+	}
+	if off != "" {
+		t.Fatalf(`expect: offset to be ""; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("10m")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "10m": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "10m" {
+		t.Fatalf(`expect: window to be "10m"; actual: "%s"`, dur)
+	}
+	if off != "" {
+		t.Fatalf(`expect: offset to be ""; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("1589448338,1589534798")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1589448338,1589534798": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1441m" {
+		t.Fatalf(`expect: window to be "1441m"; actual: "%s"`, dur)
+	}
+	if !strings.HasPrefix(off, " offset ") {
+		t.Fatalf(`expect: offset to start with " offset "; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("yesterday")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "yesterday": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1d" {
+		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
+	}
+	if !strings.HasPrefix(off, " offset ") {
+		t.Fatalf(`expect: offset to start with " offset "; actual: "%s"`, off)
+	}
+
+	// Test for Thanos (env.IsThanosEnabled() == true)
+	env.SetBool(env.ThanosEnabledEnvVar, true)
+	if !env.IsThanosEnabled() {
+		t.Fatalf("expected env.IsThanosEnabled() == true")
+	}
+
+	w, err = ParseWindowUTC("1d")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1d": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "21h" {
+		t.Fatalf(`expect: window to be "21d"; actual: "%s"`, dur)
+	}
+	if off != " offset 3h" {
+		t.Fatalf(`expect: offset to be " offset 3h"; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("2h")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "2h": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err == nil {
+		t.Fatalf(`expected error (negative duration); got ("%s", "%s")`, dur, off)
+	}
+
+	w, err = ParseWindowUTC("10m")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1d": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err == nil {
+		t.Fatalf(`expected error (negative duration); got ("%s", "%s")`, dur, off)
+	}
+
+	w, err = ParseWindowUTC("1589448338,1589534798")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1589448338,1589534798": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1441m" {
+		t.Fatalf(`expect: window to be "1441m"; actual: "%s"`, dur)
+	}
+	if !strings.HasPrefix(off, " offset ") {
+		t.Fatalf(`expect: offset to start with " offset "; actual: "%s"`, off)
+	}
+}
+
+// TODO
+// func TestWindow_Overlaps(t *testing.T) {}
+
+// TODO
+// func TestWindow_Contains(t *testing.T) {}
+
+// TODO
+// func TestWindow_Duration(t *testing.T) {}
+
+// TODO
+// func TestWindow_End(t *testing.T) {}
+
+// TODO
+// func TestWindow_Equal(t *testing.T) {}
+
+// TODO
+// func TestWindow_ExpandStart(t *testing.T) {}
+
+// TODO
+// func TestWindow_ExpandEnd(t *testing.T) {}
+
+// TODO
+// func TestWindow_Start(t *testing.T) {}
+
+// TODO
+// func TestWindow_String(t *testing.T) {}

pkg/prom/result.go

@@ -224,12 +224,33 @@ func (qr *QueryResult) GetString(field string) (string, error) {
 
 	strField, ok := f.(string)
 	if !ok {
-		return "", fmt.Errorf("'%s' field is improperly formatted", field)
+		return "", fmt.Errorf("'%s' field is improperly formatted and cannot be converted to string", field)
 	}
 
 	return strField, nil
 }
 
+// GetStrings returns the requested fields, or an error if it does not exist
+func (qr *QueryResult) GetStrings(fields ...string) (map[string]string, error) {
+	values := map[string]string{}
+
+	for _, field := range fields {
+		f, ok := qr.Metric[field]
+		if !ok {
+			return nil, fmt.Errorf("'%s' field does not exist in data result vector", field)
+		}
+
+		value, ok := f.(string)
+		if !ok {
+			return nil, fmt.Errorf("'%s' field is improperly formatted and cannot be converted to string", field)
+		}
+
+		values[field] = value
+	}
+
+	return values, nil
+}
+
 // GetLabels returns all labels and their values from the query result
 func (qr *QueryResult) GetLabels() map[string]string {
 	result := make(map[string]string)

pkg/util/mapper/mapper.go

@@ -1,8 +1,10 @@
 package mapper
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
+	"time"
 )
 
 //--------------------------------------------------------------------------
@@ -85,6 +87,10 @@ type PrimitiveMapReader interface {
 	// is empty or fails to parse, the defaultValue parameter is returned.
 	GetBool(key string, defaultValue bool) bool
 
+	// GetDuration parses a time.Duration from the map key paramter. If the
+	// value is empty to fails to parse, the defaultValue is returned.
+	GetDuration(key string, defaultValue time.Duration) time.Duration
+
 	// GetList returns a string list which contains the value set by key split using the
 	// provided delimiter with each entry trimmed of space. If the value doesn't exist,
 	// nil is returned
@@ -130,6 +136,9 @@ type PrimitiveMapWriter interface {
 	// SetBool sets the map to a string formatted bool value.
 	SetBool(key string, value bool) error
 
+	// SetDuration sets the map to a string formatted time.Duration value
+	SetDuration(key string, duration time.Duration) error
+
 	// SetList sets the map's value at key to a string consistent of each value in the list separated
 	// by the provided delimiter.
 	SetList(key string, values []string, delimiter string) error
@@ -383,6 +392,19 @@ func (rom *readOnlyMapper) GetBool(key string, defaultValue bool) bool {
 	return b
 }
 
+// GetDuration parses a time.Duration from the read-only mapper key parameter.
+// If the value is empty or fails to parse, the defaultValue parameter is returned.
+func (rom *readOnlyMapper) GetDuration(key string, defaultValue time.Duration) time.Duration {
+	r := rom.getter.Get(key)
+
+	d, err := parseDuration(r)
+	if err != nil {
+		return defaultValue
+	}
+
+	return d
+}
+
 // GetList returns a string list which contains the value set by key split using the
 // provided delimiter with each entry trimmed of space. If the value doesn't exist,
 // nil is returned
@@ -464,8 +486,77 @@ func (wom *writeOnlyMapper) SetBool(key string, value bool) error {
 	return wom.setter.Set(key, strconv.FormatBool(value))
 }
 
+// SetDuration sets the map to a string formatted bool value.
+func (wom *writeOnlyMapper) SetDuration(key string, value time.Duration) error {
+	return wom.setter.Set(key, durationString(value))
+}
+
 // SetList sets the map's value at key to a string consistent of each value in the list separated
 // by the provided delimiter.
 func (wom *writeOnlyMapper) SetList(key string, values []string, delimiter string) error {
 	return wom.setter.Set(key, strings.Join(values, delimiter))
 }
+
+const (
+	secsPerMin  = 60.0
+	secsPerHour = 3600.0
+	secsPerDay  = 86400.0
+)
+
+// durationString converts duration to a string of the form "4d", "4h", "4m", or "4s" if
+// the number of seconds in the string is evenly divisible into an integer number of
+// days, hours, minutes, or seconds respectively.
+func durationString(duration time.Duration) string {
+	durSecs := int64(duration.Seconds())
+
+	durStr := ""
+	if durSecs > 0 {
+		if durSecs%secsPerDay == 0 {
+			// convert to days
+			durStr = fmt.Sprintf("%dd", durSecs/secsPerDay)
+		} else if durSecs%secsPerHour == 0 {
+			// convert to hours
+			durStr = fmt.Sprintf("%dh", durSecs/secsPerHour)
+		} else if durSecs%secsPerMin == 0 {
+			// convert to mins
+			durStr = fmt.Sprintf("%dm", durSecs/secsPerMin)
+		} else if durSecs > 0 {
+			// default to mins, as long as duration is positive
+			durStr = fmt.Sprintf("%ds", durSecs)
+		}
+	}
+
+	return durStr
+}
+
+func parseDuration(duration string) (time.Duration, error) {
+	var amountStr string
+	var unit time.Duration
+	switch {
+	case strings.HasSuffix(duration, "s"):
+		unit = time.Second
+		amountStr = strings.TrimSuffix(duration, "s")
+	case strings.HasSuffix(duration, "m"):
+		unit = time.Minute
+		amountStr = strings.TrimSuffix(duration, "m")
+	case strings.HasSuffix(duration, "h"):
+		unit = time.Hour
+		amountStr = strings.TrimSuffix(duration, "h")
+	case strings.HasSuffix(duration, "d"):
+		unit = 24.0 * time.Hour
+		amountStr = strings.TrimSuffix(duration, "d")
+	default:
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
+
+	if len(amountStr) == 0 {
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
+
+	amount, err := strconv.ParseInt(amountStr, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
+
+	return time.Duration(amount) * unit, nil
+}

pkg/util/time.go

@@ -32,11 +32,10 @@ const (
 	DaysPerMonth = 30.42
 )
 
-// DurationOffsetStrings converts a (duration, offset) pair to Prometheus-
-// compatible strings in terms of days, hours, minutes, or seconds.
-func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
+// DurationString converts a duration to a Prometheus-compatible string in
+// terms of days, hours, minutes, or seconds.
+func DurationString(duration time.Duration) string {
 	durSecs := int64(duration.Seconds())
-	offSecs := int64(offset.Seconds())
 
 	durStr := ""
 	if durSecs > 0 {
@@ -55,23 +54,13 @@ func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
 		}
 	}
 
-	offStr := ""
-	if offSecs > 0 {
-		if offSecs%SecsPerDay == 0 {
-			// convert to days
-			offStr = fmt.Sprintf("%dd", offSecs/SecsPerDay)
-		} else if offSecs%SecsPerHour == 0 {
-			// convert to hours
-			offStr = fmt.Sprintf("%dh", offSecs/SecsPerHour)
-		} else if offSecs%SecsPerMin == 0 {
-			// convert to mins
-			offStr = fmt.Sprintf("%dm", offSecs/SecsPerMin)
-		} else if offSecs > 0 {
-			// default to mins, as long as offation is positive
-			offStr = fmt.Sprintf("%ds", offSecs)
-		}
-	}
-	return durStr, offStr
+	return durStr
+}
+
+// DurationOffsetStrings converts a (duration, offset) pair to Prometheus-
+// compatible strings in terms of days, hours, minutes, or seconds.
+func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
+	return DurationString(duration), DurationString(offset)
 }
 
 // ParseDuration converts a Prometheus-style duration string into a Duration

test/cloud_test.go

@@ -279,7 +279,7 @@ func TestNodePriceFromCSVWithBadConfig(t *testing.T) {
 	fm := FakeClusterMap{}
 	d, _ := time.ParseDuration("1m")
 
-	model := costmodel.NewCostModel(fc, fm, d)
+	model := costmodel.NewCostModel(nil, nil, fc, fm, d)
 
 	_, err := model.GetNodeCost(c)
 	if err != nil {
@@ -333,7 +333,7 @@ func TestSourceMatchesFromCSV(t *testing.T) {
 	fm := FakeClusterMap{}
 	d, _ := time.ParseDuration("1m")
 
-	model := costmodel.NewCostModel(fc, fm, d)
+	model := costmodel.NewCostModel(nil, nil, fc, fm, d)
 
 	_, err = model.GetNodeCost(c)
 	if err != nil {