Resolve merge conflicts

pkg/costmodel/aggregation.go

@@ -47,6 +47,8 @@ type Aggregation struct {
 	Environment                string               `json:"environment"`
 	Cluster                    string               `json:"cluster,omitempty"`
 	Properties                 *kubecost.Properties `json:"-"`
+	Start                      time.Time            `json:"-"`
+	End                        time.Time            `json:"-"`
 	CPUAllocationHourlyAverage float64              `json:"cpuAllocationAverage"`
 	CPUAllocationVectors       []*util.Vector       `json:"-"`
 	CPUAllocationTotal         float64              `json:"-"`

pkg/costmodel/allocation.go

@@ -0,0 +1,1670 @@
+package costmodel
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/kubecost"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/kubecost/cost-model/pkg/util"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+const (
+	queryFmtPods                  = `avg(kube_pod_container_status_running{}) by (pod, namespace, cluster_id)[%s:%s]%s`
+	queryFmtRAMBytesAllocated     = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMRequests           = `avg(avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMUsage              = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtCPUCoresAllocated     = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtCPURequests           = `avg(avg_over_time(kube_pod_container_resource_requests_cpu_cores{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtCPUUsage              = `avg(rate(container_cpu_usage_seconds_total{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtGPUsRequested         = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeCostPerGPUHr      = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeIsSpot            = `avg_over_time(kubecost_node_is_spot[%s]%s)`
+	queryFmtPVCInfo               = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, cluster_id)[%s:%s]%s`
+	queryFmtPVBytes               = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, cluster_id)`
+	queryFmtPodPVCAllocation      = `avg(avg_over_time(pod_pvc_allocation[%s]%s)) by (persistentvolume, persistentvolumeclaim, pod, namespace, cluster_id)`
+	queryFmtPVCBytesRequested     = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s]%s)) by (persistentvolumeclaim, namespace, cluster_id)`
+	queryFmtPVCostPerGiBHour      = `avg(avg_over_time(pv_hourly_cost[%s]%s)) by (volumename, cluster_id)`
+	queryFmtNetZoneGiB            = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s]%s)) by (pod_name, namespace, cluster_id) / 1024 / 1024 / 1024`
+	queryFmtNetZoneCostPerGiB     = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s]%s)) by (cluster_id)`
+	queryFmtNetRegionGiB          = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s]%s)) by (pod_name, namespace, cluster_id) / 1024 / 1024 / 1024`
+	queryFmtNetRegionCostPerGiB   = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s]%s)) by (cluster_id)`
+	queryFmtNetInternetGiB        = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s]%s)) by (pod_name, namespace, cluster_id) / 1024 / 1024 / 1024`
+	queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s]%s)) by (cluster_id)`
+	queryFmtNamespaceLabels       = `avg_over_time(kube_namespace_labels[%s]%s)`
+	queryFmtNamespaceAnnotations  = `avg_over_time(kube_namespace_annotations[%s]%s)`
+	queryFmtPodLabels             = `avg_over_time(kube_pod_labels[%s]%s)`
+	queryFmtPodAnnotations        = `avg_over_time(kube_pod_annotations[%s]%s)`
+	queryFmtServiceLabels         = `avg_over_time(service_selector_labels[%s]%s)`
+	queryFmtDeploymentLabels      = `avg_over_time(deployment_match_labels[%s]%s)`
+	queryFmtStatefulSetLabels     = `avg_over_time(statefulSet_match_labels[%s]%s)`
+	queryFmtDaemonSetLabels       = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s]%s)) by (pod, owner_name, namespace, cluster_id)`
+	queryFmtJobLabels             = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s]%s)) by (pod, owner_name, namespace ,cluster_id)`
+)
+
+// ComputeAllocation uses the CostModel instance to compute an AllocationSet
+// for the window defined by the given start and end times. The Allocations
+// returned are unaggregated (i.e. down to the container level).
+func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
+	// 1. Build out Pod map from resolution-tuned, batched Pod start/end query
+	// 2. Run and apply the results of the remaining queries to
+	// 3. Build out AllocationSet from completed Pod map
+
+	// Create a window spanning the requested query
+	window := kubecost.NewWindow(&start, &end)
+
+	// TODO niko/computeallocation remove log
+	defer log.Profile(time.Now(), fmt.Sprintf("CostModel.ComputeAllocation: completed %s", window))
+
+	// Create an empty AllocationSet. For safety, in the case of an error, we
+	// should prefer to return this empty set with the error. (In the case of
+	// no error, of course we populate the set and return it.)
+	allocSet := kubecost.NewAllocationSet(start, end)
+
+	// (1) Build out Pod map
+
+	// Build out a map of Allocations as a mapping from pod-to-container-to-
+	// underlying-Allocation instance, starting with (start, end) so that we
+	// begin with minutes, from which we compute resource allocation and cost
+	// totals from measured rate data.
+	podMap := map[podKey]*Pod{}
+
+	// clusterStarts and clusterEnds record the earliest start and latest end
+	// times, respectively, on a cluster-basis. These are used for unmounted
+	// PVs and other "virtual" Allocations so that minutes are maximally
+	// accurate during start-up or spin-down of a cluster
+	clusterStart := map[string]time.Time{}
+	clusterEnd := map[string]time.Time{}
+
+	// TODO niko/computeallocation make this configurable?
+	batchSize := 6 * time.Hour
+
+	cm.buildPodMap(window, resolution, batchSize, podMap, clusterStart, clusterEnd)
+
+	// (2) Run and apply remaining queries
+
+	// Convert window (start, end) to (duration, offset) for querying Prometheus,
+	// including handling Thanos offset
+	durStr, offStr, err := window.DurationOffsetForPrometheus()
+	if err != nil {
+		// Negative duration, so return empty set
+		return allocSet, nil
+	}
+
+	// Convert resolution duration to a query-ready string
+	resStr := util.DurationString(resolution)
+
+	ctx := prom.NewContext(cm.PrometheusClient)
+	startQuerying := time.Now()
+
+	// TODO niko/computeallocation split into required and optional queries?
+
+	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, durStr, offStr)
+	resChRAMBytesAllocated := ctx.Query(queryRAMBytesAllocated)
+
+	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, durStr, offStr)
+	resChRAMRequests := ctx.Query(queryRAMRequests)
+
+	queryRAMUsage := fmt.Sprintf(queryFmtRAMUsage, durStr, offStr)
+	resChRAMUsage := ctx.Query(queryRAMUsage)
+
+	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, durStr, offStr)
+	resChCPUCoresAllocated := ctx.Query(queryCPUCoresAllocated)
+
+	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, durStr, offStr)
+	resChCPURequests := ctx.Query(queryCPURequests)
+
+	queryCPUUsage := fmt.Sprintf(queryFmtCPUUsage, durStr, offStr)
+	resChCPUUsage := ctx.Query(queryCPUUsage)
+
+	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, durStr, offStr)
+	resChGPUsRequested := ctx.Query(queryGPUsRequested)
+
+	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, durStr, offStr)
+	resChNodeCostPerCPUHr := ctx.Query(queryNodeCostPerCPUHr)
+
+	queryNodeCostPerRAMGiBHr := fmt.Sprintf(queryFmtNodeCostPerRAMGiBHr, durStr, offStr)
+	resChNodeCostPerRAMGiBHr := ctx.Query(queryNodeCostPerRAMGiBHr)
+
+	queryNodeCostPerGPUHr := fmt.Sprintf(queryFmtNodeCostPerGPUHr, durStr, offStr)
+	resChNodeCostPerGPUHr := ctx.Query(queryNodeCostPerGPUHr)
+
+	queryNodeIsSpot := fmt.Sprintf(queryFmtNodeIsSpot, durStr, offStr)
+	resChNodeIsSpot := ctx.Query(queryNodeIsSpot)
+
+	queryPVCInfo := fmt.Sprintf(queryFmtPVCInfo, durStr, resStr, offStr)
+	resChPVCInfo := ctx.Query(queryPVCInfo)
+
+	queryPVBytes := fmt.Sprintf(queryFmtPVBytes, durStr, offStr)
+	resChPVBytes := ctx.Query(queryPVBytes)
+
+	queryPodPVCAllocation := fmt.Sprintf(queryFmtPodPVCAllocation, durStr, offStr)
+	resChPodPVCAllocation := ctx.Query(queryPodPVCAllocation)
+
+	queryPVCBytesRequested := fmt.Sprintf(queryFmtPVCBytesRequested, durStr, offStr)
+	resChPVCBytesRequested := ctx.Query(queryPVCBytesRequested)
+
+	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, durStr, offStr)
+	resChPVCostPerGiBHour := ctx.Query(queryPVCostPerGiBHour)
+
+	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, durStr, offStr)
+	resChNetZoneGiB := ctx.Query(queryNetZoneGiB)
+
+	queryNetZoneCostPerGiB := fmt.Sprintf(queryFmtNetZoneCostPerGiB, durStr, offStr)
+	resChNetZoneCostPerGiB := ctx.Query(queryNetZoneCostPerGiB)
+
+	queryNetRegionGiB := fmt.Sprintf(queryFmtNetRegionGiB, durStr, offStr)
+	resChNetRegionGiB := ctx.Query(queryNetRegionGiB)
+
+	queryNetRegionCostPerGiB := fmt.Sprintf(queryFmtNetRegionCostPerGiB, durStr, offStr)
+	resChNetRegionCostPerGiB := ctx.Query(queryNetRegionCostPerGiB)
+
+	queryNetInternetGiB := fmt.Sprintf(queryFmtNetInternetGiB, durStr, offStr)
+	resChNetInternetGiB := ctx.Query(queryNetInternetGiB)
+
+	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, durStr, offStr)
+	resChNetInternetCostPerGiB := ctx.Query(queryNetInternetCostPerGiB)
+
+	queryNamespaceLabels := fmt.Sprintf(queryFmtNamespaceLabels, durStr, offStr)
+	resChNamespaceLabels := ctx.Query(queryNamespaceLabels)
+
+	queryNamespaceAnnotations := fmt.Sprintf(queryFmtNamespaceAnnotations, durStr, offStr)
+	resChNamespaceAnnotations := ctx.Query(queryNamespaceAnnotations)
+
+	queryPodLabels := fmt.Sprintf(queryFmtPodLabels, durStr, offStr)
+	resChPodLabels := ctx.Query(queryPodLabels)
+
+	queryPodAnnotations := fmt.Sprintf(queryFmtPodAnnotations, durStr, offStr)
+	resChPodAnnotations := ctx.Query(queryPodAnnotations)
+
+	queryServiceLabels := fmt.Sprintf(queryFmtServiceLabels, durStr, offStr)
+	resChServiceLabels := ctx.Query(queryServiceLabels)
+
+	queryDeploymentLabels := fmt.Sprintf(queryFmtDeploymentLabels, durStr, offStr)
+	resChDeploymentLabels := ctx.Query(queryDeploymentLabels)
+
+	queryStatefulSetLabels := fmt.Sprintf(queryFmtStatefulSetLabels, durStr, offStr)
+	resChStatefulSetLabels := ctx.Query(queryStatefulSetLabels)
+
+	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, durStr, offStr)
+	resChDaemonSetLabels := ctx.Query(queryDaemonSetLabels)
+
+	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, durStr, offStr)
+	resChJobLabels := ctx.Query(queryJobLabels)
+
+	resCPUCoresAllocated, _ := resChCPUCoresAllocated.Await()
+	resCPURequests, _ := resChCPURequests.Await()
+	resCPUUsage, _ := resChCPUUsage.Await()
+	resRAMBytesAllocated, _ := resChRAMBytesAllocated.Await()
+	resRAMRequests, _ := resChRAMRequests.Await()
+	resRAMUsage, _ := resChRAMUsage.Await()
+	resGPUsRequested, _ := resChGPUsRequested.Await()
+
+	resNodeCostPerCPUHr, _ := resChNodeCostPerCPUHr.Await()
+	resNodeCostPerRAMGiBHr, _ := resChNodeCostPerRAMGiBHr.Await()
+	resNodeCostPerGPUHr, _ := resChNodeCostPerGPUHr.Await()
+	resNodeIsSpot, _ := resChNodeIsSpot.Await()
+
+	resPVBytes, _ := resChPVBytes.Await()
+	resPVCostPerGiBHour, _ := resChPVCostPerGiBHour.Await()
+
+	resPVCInfo, _ := resChPVCInfo.Await()
+	resPVCBytesRequested, _ := resChPVCBytesRequested.Await()
+	resPodPVCAllocation, _ := resChPodPVCAllocation.Await()
+
+	resNetZoneGiB, _ := resChNetZoneGiB.Await()
+	resNetZoneCostPerGiB, _ := resChNetZoneCostPerGiB.Await()
+	resNetRegionGiB, _ := resChNetRegionGiB.Await()
+	resNetRegionCostPerGiB, _ := resChNetRegionCostPerGiB.Await()
+	resNetInternetGiB, _ := resChNetInternetGiB.Await()
+	resNetInternetCostPerGiB, _ := resChNetInternetCostPerGiB.Await()
+
+	resNamespaceLabels, _ := resChNamespaceLabels.Await()
+	resNamespaceAnnotations, _ := resChNamespaceAnnotations.Await()
+	resPodLabels, _ := resChPodLabels.Await()
+	resPodAnnotations, _ := resChPodAnnotations.Await()
+	resServiceLabels, _ := resChServiceLabels.Await()
+	resDeploymentLabels, _ := resChDeploymentLabels.Await()
+	resStatefulSetLabels, _ := resChStatefulSetLabels.Await()
+	resDaemonSetLabels, _ := resChDaemonSetLabels.Await()
+	resJobLabels, _ := resChJobLabels.Await()
+
+	log.Profile(startQuerying, "CostModel.ComputeAllocation: queries complete")
+
+	if ctx.HasErrors() {
+		for _, err := range ctx.Errors() {
+			log.Errorf("CostModel.ComputeAllocation: %s", err)
+		}
+
+		return allocSet, ctx.ErrorCollection()
+	}
+
+	defer log.Profile(time.Now(), "CostModel.ComputeAllocation: processing complete")
+
+	applyCPUCoresAllocated(podMap, resCPUCoresAllocated)
+	applyCPUCoresRequested(podMap, resCPURequests)
+	applyCPUCoresUsed(podMap, resCPUUsage)
+	applyRAMBytesAllocated(podMap, resRAMBytesAllocated)
+	applyRAMBytesRequested(podMap, resRAMRequests)
+	applyRAMBytesUsed(podMap, resRAMUsage)
+	applyGPUsRequested(podMap, resGPUsRequested)
+	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB)
+	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB)
+	applyNetworkAllocation(podMap, resNetInternetGiB, resNetInternetCostPerGiB)
+
+	// TODO niko/computeallocation pruneDuplicateData? (see costmodel.go)
+
+	namespaceLabels := resToNamespaceLabels(resNamespaceLabels)
+	podLabels := resToPodLabels(resPodLabels)
+	namespaceAnnotations := resToNamespaceAnnotations(resNamespaceAnnotations)
+	podAnnotations := resToPodAnnotations(resPodAnnotations)
+	applyLabels(podMap, namespaceLabels, podLabels)
+	applyAnnotations(podMap, namespaceAnnotations, podAnnotations)
+
+	serviceLabels := getServiceLabels(resServiceLabels)
+	applyServicesToPods(podMap, podLabels, serviceLabels)
+
+	podDeploymentMap := labelsToPodControllerMap(podLabels, resToDeploymentLabels(resDeploymentLabels))
+	podStatefulSetMap := labelsToPodControllerMap(podLabels, resToStatefulSetLabels(resStatefulSetLabels))
+	podDaemonSetMap := resToPodDaemonSetMap(resDaemonSetLabels)
+	podJobMap := resToPodJobMap(resJobLabels)
+	applyControllersToPods(podMap, podDeploymentMap)
+	applyControllersToPods(podMap, podStatefulSetMap)
+	applyControllersToPods(podMap, podDaemonSetMap)
+	applyControllersToPods(podMap, podJobMap)
+
+	// TODO breakdown network costs?
+
+	// Build out a map of Nodes with resource costs, discounts, and node types
+	// for converting resource allocation data to cumulative costs.
+	nodeMap := map[nodeKey]*Node{}
+
+	applyNodeCostPerCPUHr(nodeMap, resNodeCostPerCPUHr)
+	applyNodeCostPerRAMGiBHr(nodeMap, resNodeCostPerRAMGiBHr)
+	applyNodeCostPerGPUHr(nodeMap, resNodeCostPerGPUHr)
+	applyNodeSpot(nodeMap, resNodeIsSpot)
+	applyNodeDiscount(nodeMap, cm)
+
+	// Build out the map of all PVs with class, size and cost-per-hour.
+	// Note: this does not record time running, which we may want to
+	// include later for increased PV precision. (As long as the PV has
+	// a PVC, we get time running there, so this is only inaccurate
+	// for short-lived, unmounted PVs.)
+	pvMap := map[pvKey]*PV{}
+	buildPVMap(pvMap, resPVCostPerGiBHour)
+	applyPVBytes(pvMap, resPVBytes)
+
+	// Build out the map of all PVCs with time running, bytes requested,
+	// and connect to the correct PV from pvMap. (If no PV exists, that
+	// is noted, but does not result in any allocation/cost.)
+	pvcMap := map[pvcKey]*PVC{}
+	buildPVCMap(window, pvcMap, pvMap, resPVCInfo)
+	applyPVCBytesRequested(pvcMap, resPVCBytesRequested)
+
+	// Build out the relationships of pods to their PVCs. This step
+	// populates the PVC.Count field so that PVC allocation can be
+	// split appropriately among each pod's container allocation.
+	podPVCMap := map[podKey][]*PVC{}
+	buildPodPVCMap(podPVCMap, pvMap, pvcMap, podMap, resPodPVCAllocation)
+
+	// Identify unmounted PVs (PVs without PVCs) and add one Allocation per
+	// cluster representing each cluster's unmounted PVs (if necessary).
+	applyUnmountedPVs(window, podMap, pvMap, pvcMap)
+
+	// (3) Build out AllocationSet from Pod map
+
+	for _, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			cluster, _ := alloc.Properties.GetCluster()
+			node, _ := alloc.Properties.GetNode()
+			namespace, _ := alloc.Properties.GetNamespace()
+			pod, _ := alloc.Properties.GetPod()
+			container, _ := alloc.Properties.GetContainer()
+
+			podKey := newPodKey(cluster, namespace, pod)
+			nodeKey := newNodeKey(cluster, node)
+
+			if n, ok := nodeMap[nodeKey]; !ok {
+				if pod != kubecost.UnmountedSuffix {
+					log.Warningf("CostModel.ComputeAllocation: failed to find node %s for %s", nodeKey, alloc.Name)
+				}
+			} else {
+				alloc.CPUCost = alloc.CPUCoreHours * n.CostPerCPUHr
+				alloc.RAMCost = (alloc.RAMByteHours / 1024 / 1024 / 1024) * n.CostPerRAMGiBHr
+				alloc.GPUCost = alloc.GPUHours * n.CostPerGPUHr
+			}
+
+			if pvcs, ok := podPVCMap[podKey]; ok {
+				for _, pvc := range pvcs {
+					// Determine the (start, end) of the relationship between the
+					// given PVC and the associated Allocation so that a precise
+					// number of hours can be used to compute cumulative cost.
+					s, e := alloc.Start, alloc.End
+					if pvc.Start.After(alloc.Start) {
+						s = pvc.Start
+					}
+					if pvc.End.Before(alloc.End) {
+						e = pvc.End
+					}
+					minutes := e.Sub(s).Minutes()
+					hrs := minutes / 60.0
+
+					count := float64(pvc.Count)
+					if pvc.Count < 1 {
+						count = 1
+					}
+
+					gib := pvc.Bytes / 1024 / 1024 / 1024
+					cost := pvc.Volume.CostPerGiBHour * gib * hrs
+
+					// Apply the size and cost of the PV to the allocation, each
+					// weighted by count (i.e. the number of containers in the pod)
+					alloc.PVByteHours += pvc.Bytes * hrs / count
+					alloc.PVCost += cost / count
+				}
+			}
+
+			alloc.TotalCost = 0.0
+			alloc.TotalCost += alloc.CPUCost
+			alloc.TotalCost += alloc.RAMCost
+			alloc.TotalCost += alloc.GPUCost
+			alloc.TotalCost += alloc.PVCost
+			alloc.TotalCost += alloc.NetworkCost
+			alloc.TotalCost += alloc.SharedCost
+			alloc.TotalCost += alloc.ExternalCost
+
+			// Make sure that the name is correct (node may not be present at this
+			// point due to it missing from queryMinutes) then insert.
+			alloc.Name = fmt.Sprintf("%s/%s/%s/%s/%s", cluster, node, namespace, pod, container)
+			allocSet.Set(alloc)
+		}
+	}
+
+	return allocSet, nil
+}
+
+func (cm *CostModel) buildPodMap(window kubecost.Window, resolution, maxBatchSize time.Duration, podMap map[podKey]*Pod, clusterStart, clusterEnd map[string]time.Time) error {
+	// Assumes that window is positive and closed
+	start, end := *window.Start(), *window.End()
+
+	// Convert resolution duration to a query-ready string
+	resStr := util.DurationString(resolution)
+
+	ctx := prom.NewContext(cm.PrometheusClient)
+	profile := time.Now()
+
+	// Query for (start, end) by (pod, namespace, cluster) over the given
+	// window, using the given resolution, and if necessary in batches no
+	// larger than the given maximum batch size. If working in batches, track
+	// overall progress by starting with (window.start, window.start) and
+	// querying in batches no larger than maxBatchSize from start-to-end,
+	// folding each result set into podMap as the results come back.
+	coverage := kubecost.NewWindow(&start, &start)
+
+	numQuery := 1
+	for coverage.End().Before(end) {
+		// Determine the (start, end) of the current batch
+		batchStart := *coverage.End()
+		batchEnd := coverage.End().Add(maxBatchSize)
+		if batchEnd.After(end) {
+			batchEnd = end
+		}
+		batchWindow := kubecost.NewWindow(&batchStart, &batchEnd)
+
+		var resPods []*prom.QueryResult
+		var err error
+		maxTries := 3
+		numTries := 0
+		for resPods == nil && numTries < maxTries {
+			numTries++
+
+			// Convert window (start, end) to (duration, offset) for querying Prometheus,
+			// including handling Thanos offset
+			durStr, offStr, err := batchWindow.DurationOffsetForPrometheus()
+			if err != nil || durStr == "" {
+				// Negative duration, so set empty results and don't query
+				// TODO niko/computeallocation test this!!!
+				resPods = []*prom.QueryResult{}
+				err = nil
+				break
+			}
+
+			// Submit and profile query
+			queryPods := fmt.Sprintf(queryFmtPods, durStr, resStr, offStr)
+			queryProfile := time.Now()
+			resPods, err = ctx.Query(queryPods).Await()
+			if err != nil {
+				// TODO niko/computeallocation remove log
+				log.Profile(queryProfile, fmt.Sprintf("CostModel.ComputeAllocation: pod query batch %d try %d failed: %s", numQuery, numTries, queryPods))
+				resPods = nil
+			} else {
+				// TODO niko/computeallocation remove log
+				log.Profile(queryProfile, fmt.Sprintf("CostModel.ComputeAllocation: pod query batch %d try %d succeeded: %s", numQuery, numTries, queryPods))
+			}
+		}
+
+		if err != nil {
+			return err
+		}
+
+		applyPodResults(window, resolution, podMap, clusterStart, clusterEnd, resPods)
+
+		coverage = coverage.ExpandEnd(batchEnd)
+		numQuery++
+	}
+
+	log.Profile(profile, "CostModel.ComputeAllocation: pod map built")
+
+	return nil
+}
+
+func applyPodResults(window kubecost.Window, resolution time.Duration, podMap map[podKey]*Pod, clusterStart, clusterEnd map[string]time.Time, resPods []*prom.QueryResult) {
+	for _, res := range resPods {
+		if len(res.Values) == 0 {
+			log.Warningf("CostModel.ComputeAllocation: empty minutes result")
+			continue
+		}
+
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		labels, err := res.GetStrings("namespace", "pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: minutes query result missing field: %s", err)
+			continue
+		}
+
+		namespace := labels["namespace"]
+		pod := labels["pod"]
+		key := newPodKey(cluster, namespace, pod)
+
+		// allocStart and allocEnd are the timestamps of the first and last
+		// minutes the pod was running, respectively. We subtract one resolution
+		// from allocStart because this point will actually represent the end
+		// of the first minute. We don't subtract from allocEnd because it
+		// already represents the end of the last minute.
+		var allocStart, allocEnd time.Time
+		startAdjustmentCoeff, endAdjustmentCoeff := 1.0, 1.0
+		for _, datum := range res.Values {
+			t := time.Unix(int64(datum.Timestamp), 0)
+
+			if allocStart.IsZero() && datum.Value > 0 && window.Contains(t) {
+				// Set the start timestamp to the earliest non-zero timestamp
+				allocStart = t
+
+				// Record adjustment coefficient, i.e. the portion of the start
+				// timestamp to "ignore". That is, sometimes the value will be
+				// 0.5, meaning that we should discount the time running by
+				// half of the resolution the timestamp stands for.
+				startAdjustmentCoeff = (1.0 - datum.Value)
+			}
+
+			if datum.Value > 0 && window.Contains(t) {
+				// Set the end timestamp to the latest non-zero timestamp
+				allocEnd = t
+
+				// If the end timestamp differs from the start, then record the
+				// adjustment coefficient, i.e. the portion of the end
+				// timestamp to "ignore". That is, sometimes the value will be
+				// 0.5, meaning that we should discount the time running by
+				// half of the resolution the timestamp stands for.
+				if !allocStart.Equal(t) {
+					endAdjustmentCoeff = (1.0 - datum.Value)
+				}
+			}
+		}
+
+		if allocStart.IsZero() || allocEnd.IsZero() {
+			continue
+		}
+
+		// Adjust timestamps accorind to the resolution and the adjustment
+		// coefficients, as described above. That is, count the start timestamp
+		// from the beginning of the resolution, not the end. Then "reduce" the
+		// start and end by the correct amount, in the case that the "running"
+		// value of the first or last timestamp was not a full 1.0.
+		allocStart = allocStart.Add(-resolution)
+		allocStart = allocStart.Add(time.Duration(startAdjustmentCoeff) * resolution)
+		allocEnd = allocEnd.Add(-time.Duration(endAdjustmentCoeff) * resolution)
+
+		// Set start if unset or this datum's start time is earlier than the
+		// current earliest time.
+		if _, ok := clusterStart[cluster]; !ok || allocStart.Before(clusterStart[cluster]) {
+			clusterStart[cluster] = allocStart
+		}
+
+		// Set end if unset or this datum's end time is later than the
+		// current latest time.
+		if _, ok := clusterEnd[cluster]; !ok || allocEnd.After(clusterEnd[cluster]) {
+			clusterEnd[cluster] = allocEnd
+		}
+
+		if pod, ok := podMap[key]; ok {
+			// Pod has already been recorded, so update it accordingly
+			if allocStart.Before(pod.Start) {
+				pod.Start = allocStart
+			}
+			if allocEnd.After(pod.End) {
+				pod.End = allocEnd
+			}
+		} else {
+			// Pod has not been recorded yet, so insert it
+			podMap[key] = &Pod{
+				Window:      window.Clone(),
+				Start:       allocStart,
+				End:         allocEnd,
+				Key:         key,
+				Allocations: map[string]*kubecost.Allocation{},
+			}
+		}
+	}
+}
+
+func applyCPUCoresAllocated(podMap map[podKey]*Pod, resCPUCoresAllocated []*prom.QueryResult) {
+	for _, res := range resCPUCoresAllocated {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU allocation result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU allocation result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU allocation query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		cpuCores := res.Values[0].Value
+		hours := pod.Allocations[container].Minutes() / 60.0
+		pod.Allocations[container].CPUCoreHours = cpuCores * hours
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: CPU allocation query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyCPUCoresRequested(podMap map[podKey]*Pod, resCPUCoresRequested []*prom.QueryResult) {
+	for _, res := range resCPUCoresRequested {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU request result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU request result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU request query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].CPUCoreRequestAverage = res.Values[0].Value
+
+		// If CPU allocation is less than requests, set CPUCoreHours to
+		// request level.
+		if pod.Allocations[container].CPUCores() < res.Values[0].Value {
+			pod.Allocations[container].CPUCoreHours = res.Values[0].Value * (pod.Allocations[container].Minutes() / 60.0)
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: CPU request query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyCPUCoresUsed(podMap map[podKey]*Pod, resCPUCoresUsed []*prom.QueryResult) {
+	for _, res := range resCPUCoresUsed {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].CPUCoreUsageAverage = res.Values[0].Value
+	}
+}
+
+func applyRAMBytesAllocated(podMap map[podKey]*Pod, resRAMBytesAllocated []*prom.QueryResult) {
+	for _, res := range resRAMBytesAllocated {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM allocation result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM allocation result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM allocation query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		ramBytes := res.Values[0].Value
+		hours := pod.Allocations[container].Minutes() / 60.0
+		pod.Allocations[container].RAMByteHours = ramBytes * hours
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: RAM allocation query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyRAMBytesRequested(podMap map[podKey]*Pod, resRAMBytesRequested []*prom.QueryResult) {
+	for _, res := range resRAMBytesRequested {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM request result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM request result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM request query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].RAMBytesRequestAverage = res.Values[0].Value
+
+		// If RAM allocation is less than requests, set RAMByteHours to
+		// request level.
+		if pod.Allocations[container].RAMBytes() < res.Values[0].Value {
+			pod.Allocations[container].RAMByteHours = res.Values[0].Value * (pod.Allocations[container].Minutes() / 60.0)
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: RAM request query result missing 'node': %s", key)
+			continue
+		}
+		pod.Allocations[container].Properties.SetNode(node)
+	}
+}
+
+func applyRAMBytesUsed(podMap map[podKey]*Pod, resRAMBytesUsed []*prom.QueryResult) {
+	for _, res := range resRAMBytesUsed {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		pod.Allocations[container].RAMBytesUsageAverage = res.Values[0].Value
+	}
+}
+
+func applyGPUsRequested(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult) {
+	for _, res := range resGPUsRequested {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU request result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU request result for unidentified pod: %s", key)
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU request query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		// TODO niko/computeallocation remove log
+		log.Infof("CostModel.ComputeAllocation: GPU results: %s=%f", key, res.Values[0].Value)
+
+		hrs := pod.Allocations[container].Minutes() / 60.0
+		pod.Allocations[container].GPUHours = res.Values[0].Value * hrs
+	}
+}
+
+func applyNetworkAllocation(podMap map[podKey]*Pod, resNetworkGiB []*prom.QueryResult, resNetworkCostPerGiB []*prom.QueryResult) {
+	costPerGiBByCluster := map[string]float64{}
+
+	for _, res := range resNetworkCostPerGiB {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		costPerGiBByCluster[cluster] = res.Values[0].Value
+	}
+
+	for _, res := range resNetworkGiB {
+		podKey, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: Network allocation query result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[podKey]
+		if !ok {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: Network allocation query result for unidentified pod: %s", podKey)
+			continue
+		}
+
+		for _, alloc := range pod.Allocations {
+			gib := res.Values[0].Value / float64(len(pod.Allocations))
+			costPerGiB := costPerGiBByCluster[podKey.Cluster]
+			alloc.NetworkCost = gib * costPerGiB
+		}
+	}
+}
+
+func resToNamespaceLabels(resNamespaceLabels []*prom.QueryResult) map[string]map[string]string {
+	namespaceLabels := map[string]map[string]string{}
+
+	for _, res := range resNamespaceLabels {
+		namespace, err := res.GetString("namespace")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := namespaceLabels[namespace]; !ok {
+			namespaceLabels[namespace] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			namespaceLabels[namespace][k] = l
+		}
+	}
+
+	return namespaceLabels
+}
+
+func resToPodLabels(resPodLabels []*prom.QueryResult) map[podKey]map[string]string {
+	podLabels := map[podKey]map[string]string{}
+
+	for _, res := range resPodLabels {
+		podKey, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := podLabels[podKey]; !ok {
+			podLabels[podKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			podLabels[podKey][k] = l
+		}
+	}
+
+	return podLabels
+}
+
+func resToNamespaceAnnotations(resNamespaceAnnotations []*prom.QueryResult) map[string]map[string]string {
+	namespaceAnnotations := map[string]map[string]string{}
+
+	for _, res := range resNamespaceAnnotations {
+		namespace, err := res.GetString("namespace")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := namespaceAnnotations[namespace]; !ok {
+			namespaceAnnotations[namespace] = map[string]string{}
+		}
+
+		for k, l := range res.GetAnnotations() {
+			namespaceAnnotations[namespace][k] = l
+		}
+	}
+
+	return namespaceAnnotations
+}
+
+func resToPodAnnotations(resPodAnnotations []*prom.QueryResult) map[podKey]map[string]string {
+	podAnnotations := map[podKey]map[string]string{}
+
+	for _, res := range resPodAnnotations {
+		podKey, err := resultPodKey(res, "cluster_id", "namespace", "pod")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := podAnnotations[podKey]; !ok {
+			podAnnotations[podKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetAnnotations() {
+			podAnnotations[podKey][k] = l
+		}
+	}
+
+	return podAnnotations
+}
+
+func applyLabels(podMap map[podKey]*Pod, namespaceLabels map[string]map[string]string, podLabels map[podKey]map[string]string) {
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			allocLabels, err := alloc.Properties.GetLabels()
+			if err != nil {
+				allocLabels = map[string]string{}
+			}
+
+			// Apply namespace labels first, then pod labels so that pod labels
+			// overwrite namespace labels.
+			if labels, ok := namespaceLabels[key.Namespace]; ok {
+				for k, v := range labels {
+					allocLabels[k] = v
+				}
+			}
+			if labels, ok := podLabels[key]; ok {
+				for k, v := range labels {
+					allocLabels[k] = v
+				}
+			}
+
+			alloc.Properties.SetLabels(allocLabels)
+		}
+	}
+}
+
+func applyAnnotations(podMap map[podKey]*Pod, namespaceAnnotations map[string]map[string]string, podAnnotations map[podKey]map[string]string) {
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			allocAnnotations, err := alloc.Properties.GetAnnotations()
+			if err != nil {
+				allocAnnotations = map[string]string{}
+			}
+
+			// Apply namespace annotations first, then pod annotations so that
+			// pod labels overwrite namespace labels.
+			if labels, ok := namespaceAnnotations[key.Namespace]; ok {
+				for k, v := range labels {
+					allocAnnotations[k] = v
+				}
+			}
+			if labels, ok := podAnnotations[key]; ok {
+				for k, v := range labels {
+					allocAnnotations[k] = v
+				}
+			}
+
+			alloc.Properties.SetAnnotations(allocAnnotations)
+		}
+	}
+}
+
+func getServiceLabels(resServiceLabels []*prom.QueryResult) map[serviceKey]map[string]string {
+	serviceLabels := map[serviceKey]map[string]string{}
+
+	for _, res := range resServiceLabels {
+		serviceKey, err := resultServiceKey(res, "cluster_id", "namespace", "service")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := serviceLabels[serviceKey]; !ok {
+			serviceLabels[serviceKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			serviceLabels[serviceKey][k] = l
+		}
+	}
+
+	return serviceLabels
+}
+
+func resToDeploymentLabels(resDeploymentLabels []*prom.QueryResult) map[controllerKey]map[string]string {
+	deploymentLabels := map[controllerKey]map[string]string{}
+
+	for _, res := range resDeploymentLabels {
+		controllerKey, err := resultDeploymentKey(res, "cluster_id", "namespace", "deployment")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := deploymentLabels[controllerKey]; !ok {
+			deploymentLabels[controllerKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			deploymentLabels[controllerKey][k] = l
+		}
+	}
+
+	return deploymentLabels
+}
+
+func resToStatefulSetLabels(resStatefulSetLabels []*prom.QueryResult) map[controllerKey]map[string]string {
+	statefulSetLabels := map[controllerKey]map[string]string{}
+
+	for _, res := range resStatefulSetLabels {
+		controllerKey, err := resultStatefulSetKey(res, "cluster_id", "namespace", "statefulSet")
+		if err != nil {
+			continue
+		}
+
+		if _, ok := statefulSetLabels[controllerKey]; !ok {
+			statefulSetLabels[controllerKey] = map[string]string{}
+		}
+
+		for k, l := range res.GetLabels() {
+			statefulSetLabels[controllerKey][k] = l
+		}
+	}
+
+	return statefulSetLabels
+}
+
+func labelsToPodControllerMap(podLabels map[podKey]map[string]string, controllerLabels map[controllerKey]map[string]string) map[podKey]controllerKey {
+	podControllerMap := map[podKey]controllerKey{}
+
+	// For each controller, turn the labels into a selector and attempt to
+	// match it with each set of pod labels. A match indicates that the pod
+	// belongs to the controller.
+	for cKey, cLabels := range controllerLabels {
+		selector := labels.Set(cLabels).AsSelectorPreValidated()
+
+		for pKey, pLabels := range podLabels {
+			// If the pod is in a different cluster or namespace, there is
+			// no need to compare the labels.
+			if cKey.Cluster != pKey.Cluster || cKey.Namespace != pKey.Namespace {
+				continue
+			}
+
+			podLabelSet := labels.Set(pLabels)
+			if selector.Matches(podLabelSet) {
+				if _, ok := podControllerMap[pKey]; ok {
+					log.Warningf("CostModel.ComputeAllocation: PodControllerMap match already exists: %s matches %s and %s", pKey, podControllerMap[pKey], cKey)
+				}
+				podControllerMap[pKey] = cKey
+			}
+		}
+	}
+
+	return podControllerMap
+}
+
+func resToPodDaemonSetMap(resDaemonSetLabels []*prom.QueryResult) map[podKey]controllerKey {
+	daemonSetLabels := map[podKey]controllerKey{}
+
+	for _, res := range resDaemonSetLabels {
+		controllerKey, err := resultDaemonSetKey(res, "cluster_id", "namespace", "owner_name")
+		if err != nil {
+			continue
+		}
+
+		pod, err := res.GetString("pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: DaemonSetLabel result without pod: %s", controllerKey)
+		}
+
+		podKey := newPodKey(controllerKey.Cluster, controllerKey.Namespace, pod)
+
+		daemonSetLabels[podKey] = controllerKey
+	}
+
+	return daemonSetLabels
+}
+
+func resToPodJobMap(resJobLabels []*prom.QueryResult) map[podKey]controllerKey {
+	jobLabels := map[podKey]controllerKey{}
+
+	for _, res := range resJobLabels {
+		controllerKey, err := resultJobKey(res, "cluster_id", "namespace", "owner_name")
+		if err != nil {
+			continue
+		}
+
+		pod, err := res.GetString("pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: JobLabel result without pod: %s", controllerKey)
+		}
+
+		podKey := newPodKey(controllerKey.Cluster, controllerKey.Namespace, pod)
+
+		jobLabels[podKey] = controllerKey
+	}
+
+	return jobLabels
+}
+
+func applyServicesToPods(podMap map[podKey]*Pod, podLabels map[podKey]map[string]string, serviceLabels map[serviceKey]map[string]string) {
+	podServicesMap := map[podKey][]serviceKey{}
+
+	// For each service, turn the labels into a selector and attempt to
+	// match it with each set of pod labels. A match indicates that the pod
+	// belongs to the service.
+	for sKey, sLabels := range serviceLabels {
+		selector := labels.Set(sLabels).AsSelectorPreValidated()
+
+		for pKey, pLabels := range podLabels {
+			// If the pod is in a different cluster or namespace, there is
+			// no need to compare the labels.
+			if sKey.Cluster != pKey.Cluster || sKey.Namespace != pKey.Namespace {
+				continue
+			}
+
+			podLabelSet := labels.Set(pLabels)
+			if selector.Matches(podLabelSet) {
+				if _, ok := podServicesMap[pKey]; !ok {
+					podServicesMap[pKey] = []serviceKey{}
+				}
+				podServicesMap[pKey] = append(podServicesMap[pKey], sKey)
+			}
+		}
+	}
+
+	// For each allocation in each pod, attempt to find and apply the list of
+	// services associated with the allocation's pod.
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			if sKeys, ok := podServicesMap[key]; ok {
+				services := []string{}
+				for _, sKey := range sKeys {
+					services = append(services, sKey.Service)
+				}
+				alloc.Properties.SetServices(services)
+			}
+		}
+	}
+}
+
+func applyControllersToPods(podMap map[podKey]*Pod, podControllerMap map[podKey]controllerKey) {
+	for key, pod := range podMap {
+		for _, alloc := range pod.Allocations {
+			if controllerKey, ok := podControllerMap[key]; ok {
+				alloc.Properties.SetControllerKind(controllerKey.ControllerKind)
+				alloc.Properties.SetController(controllerKey.Controller)
+			}
+		}
+	}
+}
+
+func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*Node, resNodeCostPerCPUHr []*prom.QueryResult) {
+	for _, res := range resNodeCostPerCPUHr {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node CPU cost query result missing field: %s", err)
+			continue
+		}
+
+		instanceType, err := res.GetString("instance_type")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node CPU cost query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			nodeMap[key] = &Node{
+				Name:     node,
+				NodeType: instanceType,
+			}
+		}
+
+		nodeMap[key].CostPerCPUHr = res.Values[0].Value
+	}
+}
+
+func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*Node, resNodeCostPerRAMGiBHr []*prom.QueryResult) {
+	for _, res := range resNodeCostPerRAMGiBHr {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node RAM cost query result missing field: %s", err)
+			continue
+		}
+
+		instanceType, err := res.GetString("instance_type")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node RAM cost query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			nodeMap[key] = &Node{
+				Name:     node,
+				NodeType: instanceType,
+			}
+		}
+
+		nodeMap[key].CostPerRAMGiBHr = res.Values[0].Value
+	}
+}
+
+func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*Node, resNodeCostPerGPUHr []*prom.QueryResult) {
+	for _, res := range resNodeCostPerGPUHr {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node GPU cost query result missing field: %s", err)
+			continue
+		}
+
+		instanceType, err := res.GetString("instance_type")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node GPU cost query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			nodeMap[key] = &Node{
+				Name:     node,
+				NodeType: instanceType,
+			}
+		}
+
+		nodeMap[key].CostPerGPUHr = res.Values[0].Value
+	}
+}
+
+func applyNodeSpot(nodeMap map[nodeKey]*Node, resNodeIsSpot []*prom.QueryResult) {
+	for _, res := range resNodeIsSpot {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		node, err := res.GetString("node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node spot query result missing field: %s", err)
+			continue
+		}
+
+		key := newNodeKey(cluster, node)
+		if _, ok := nodeMap[key]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: Node spot  query result for missing node: %s", key)
+			continue
+		}
+
+		nodeMap[key].Preemptible = res.Values[0].Value > 0
+	}
+}
+
+func applyNodeDiscount(nodeMap map[nodeKey]*Node, cm *CostModel) {
+	if cm == nil {
+		return
+	}
+
+	c, err := cm.Provider.GetConfig()
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: applyNodeDiscount: %s", err)
+		return
+	}
+
+	discount, err := ParsePercentString(c.Discount)
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: applyNodeDiscount: %s", err)
+		return
+	}
+
+	negotiatedDiscount, err := ParsePercentString(c.NegotiatedDiscount)
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: applyNodeDiscount: %s", err)
+		return
+	}
+
+	for _, node := range nodeMap {
+		// TODO niko/computeallocation GKE Reserved Instances into account
+		node.Discount = cm.Provider.CombinedDiscountForNode(node.NodeType, node.Preemptible, discount, negotiatedDiscount)
+		node.CostPerCPUHr *= (1.0 - node.Discount)
+		node.CostPerRAMGiBHr *= (1.0 - node.Discount)
+	}
+}
+
+func buildPVMap(pvMap map[pvKey]*PV, resPVCostPerGiBHour []*prom.QueryResult) {
+	for _, res := range resPVCostPerGiBHour {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		name, err := res.GetString("volumename")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PV cost without volumename")
+			continue
+		}
+
+		key := newPVKey(cluster, name)
+
+		pvMap[key] = &PV{
+			Cluster:        cluster,
+			Name:           name,
+			CostPerGiBHour: res.Values[0].Value,
+		}
+	}
+}
+
+func applyPVBytes(pvMap map[pvKey]*PV, resPVBytes []*prom.QueryResult) {
+	for _, res := range resPVBytes {
+		key, err := resultPVKey(res, "cluster_id", "persistentvolume")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PV bytes query result missing field: %s", err)
+			continue
+		}
+
+		if _, ok := pvMap[key]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: PV bytes result for missing PV: %s", err)
+			continue
+		}
+
+		pvMap[key].Bytes = res.Values[0].Value
+	}
+}
+
+func buildPVCMap(window kubecost.Window, pvcMap map[pvcKey]*PVC, pvMap map[pvKey]*PV, resPVCInfo []*prom.QueryResult) {
+	for _, res := range resPVCInfo {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		values, err := res.GetStrings("persistentvolumeclaim", "storageclass", "volumename", "namespace")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PVC info query result missing field: %s", err)
+			continue
+		}
+
+		namespace := values["namespace"]
+		name := values["persistentvolumeclaim"]
+		volume := values["volumename"]
+		storageClass := values["storageclass"]
+
+		pvKey := newPVKey(cluster, volume)
+		pvcKey := newPVCKey(cluster, namespace, name)
+
+		// pvcStart and pvcEnd are the timestamps of the first and last minutes
+		// the PVC was running, respectively. We subtract 1m from pvcStart
+		// because this point will actually represent the end of the first
+		// minute. We don't subtract from pvcEnd because it already represents
+		// the end of the last minute.
+		var pvcStart, pvcEnd time.Time
+		for _, datum := range res.Values {
+			t := time.Unix(int64(datum.Timestamp), 0)
+			if pvcStart.IsZero() && datum.Value > 0 && window.Contains(t) {
+				pvcStart = t
+			}
+			if datum.Value > 0 && window.Contains(t) {
+				pvcEnd = t
+			}
+		}
+		if pvcStart.IsZero() || pvcEnd.IsZero() {
+			log.Warningf("CostModel.ComputeAllocation: PVC %s has no running time", pvcKey)
+		}
+		pvcStart = pvcStart.Add(-time.Minute)
+
+		if _, ok := pvMap[pvKey]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: PV missing for PVC info query result: %s", pvKey)
+			continue
+		}
+
+		pvMap[pvKey].StorageClass = storageClass
+
+		if _, ok := pvcMap[pvcKey]; !ok {
+			pvcMap[pvcKey] = &PVC{}
+		}
+
+		pvcMap[pvcKey].Name = name
+		pvcMap[pvcKey].Namespace = namespace
+		pvcMap[pvcKey].Volume = pvMap[pvKey]
+		pvcMap[pvcKey].Start = pvcStart
+		pvcMap[pvcKey].End = pvcEnd
+	}
+}
+
+func applyPVCBytesRequested(pvcMap map[pvcKey]*PVC, resPVCBytesRequested []*prom.QueryResult) {
+	for _, res := range resPVCBytesRequested {
+		key, err := resultPVCKey(res, "cluster_id", "namespace", "persistentvolumeclaim")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PVC bytes requested query result missing field: %s", err)
+			continue
+		}
+
+		if _, ok := pvcMap[key]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: PVC bytes requested result for missing PVC: %s", key)
+			continue
+		}
+
+		pvcMap[key].Bytes = res.Values[0].Value
+	}
+}
+
+func buildPodPVCMap(podPVCMap map[podKey][]*PVC, pvMap map[pvKey]*PV, pvcMap map[pvcKey]*PVC, podMap map[podKey]*Pod, resPodPVCAllocation []*prom.QueryResult) {
+	for _, res := range resPodPVCAllocation {
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		values, err := res.GetStrings("persistentvolume", "persistentvolumeclaim", "pod", "namespace")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: PVC allocation query result missing field: %s", err)
+			continue
+		}
+
+		namespace := values["namespace"]
+		pod := values["pod"]
+		name := values["persistentvolumeclaim"]
+		volume := values["persistentvolume"]
+
+		podKey := newPodKey(cluster, namespace, pod)
+		pvKey := newPVKey(cluster, volume)
+		pvcKey := newPVCKey(cluster, namespace, name)
+
+		if _, ok := pvMap[pvKey]; !ok {
+			log.Warningf("CostModel.ComputeAllocation: PV missing for PVC allocation query result: %s", pvKey)
+			continue
+		}
+
+		if _, ok := podPVCMap[podKey]; !ok {
+			podPVCMap[podKey] = []*PVC{}
+		}
+
+		pvc, ok := pvcMap[pvcKey]
+		if !ok {
+			log.Warningf("CostModel.ComputeAllocation: PVC missing for PVC allocation query: %s", pvcKey)
+			continue
+		}
+
+		count := 1
+		if pod, ok := podMap[podKey]; ok && len(pod.Allocations) > 0 {
+			count = len(pod.Allocations)
+		} else {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: PVC %s for missing pod %s", pvcKey, podKey)
+		}
+
+		pvc.Count = count
+		pvc.Mounted = true
+
+		podPVCMap[podKey] = append(podPVCMap[podKey], pvc)
+	}
+}
+
+func applyUnmountedPVs(window kubecost.Window, podMap map[podKey]*Pod, pvMap map[pvKey]*PV, pvcMap map[pvcKey]*PVC) {
+	unmountedPVBytes := map[string]float64{}
+	unmountedPVCost := map[string]float64{}
+
+	for _, pv := range pvMap {
+		mounted := false
+		for _, pvc := range pvcMap {
+			if pvc.Volume == nil {
+				continue
+			}
+			if pvc.Volume == pv {
+				mounted = true
+				break
+			}
+		}
+
+		if !mounted {
+			gib := pv.Bytes / 1024 / 1024 / 1024
+			hrs := window.Minutes() / 60.0 // TODO niko/computeallocation PV hours, not window hours?
+			cost := pv.CostPerGiBHour * gib * hrs
+			unmountedPVCost[pv.Cluster] += cost
+			unmountedPVBytes[pv.Cluster] += pv.Bytes
+		}
+	}
+
+	for cluster, amount := range unmountedPVCost {
+		container := kubecost.UnmountedSuffix
+		pod := kubecost.UnmountedSuffix
+		namespace := kubecost.UnmountedSuffix
+		node := ""
+
+		key := newPodKey(cluster, namespace, pod)
+		podMap[key] = &Pod{
+			Window:      window.Clone(),
+			Start:       *window.Start(),
+			End:         *window.End(),
+			Key:         key,
+			Allocations: map[string]*kubecost.Allocation{},
+		}
+
+		podMap[key].AppendContainer(container)
+		podMap[key].Allocations[container].Properties.SetCluster(cluster)
+		podMap[key].Allocations[container].Properties.SetNode(node)
+		podMap[key].Allocations[container].Properties.SetNamespace(namespace)
+		podMap[key].Allocations[container].Properties.SetPod(pod)
+		podMap[key].Allocations[container].Properties.SetContainer(container)
+		podMap[key].Allocations[container].PVByteHours = unmountedPVBytes[cluster] * window.Minutes() / 60.0
+		podMap[key].Allocations[container].PVCost = amount
+		podMap[key].Allocations[container].TotalCost = amount
+	}
+}
+
+func applyUnmountedPVCs(window kubecost.Window, podMap map[podKey]*Pod, pvcMap map[pvcKey]*PVC) {
+	unmountedPVCBytes := map[namespaceKey]float64{}
+	unmountedPVCCost := map[namespaceKey]float64{}
+
+	for _, pvc := range pvcMap {
+		if !pvc.Mounted && pvc.Volume != nil {
+			key := newNamespaceKey(pvc.Cluster, pvc.Namespace)
+
+			gib := pvc.Volume.Bytes / 1024 / 1024 / 1024
+			hrs := pvc.Minutes() / 60.0
+			cost := pvc.Volume.CostPerGiBHour * gib * hrs
+			unmountedPVCCost[key] += cost
+			unmountedPVCBytes[key] += pvc.Volume.Bytes
+		}
+	}
+
+	for key, amount := range unmountedPVCCost {
+		container := kubecost.UnmountedSuffix
+		pod := kubecost.UnmountedSuffix
+		namespace := key.Namespace
+		node := ""
+		cluster := key.Cluster
+
+		podKey := newPodKey(cluster, namespace, pod)
+		podMap[podKey] = &Pod{
+			Window:      window.Clone(),
+			Start:       *window.Start(),
+			End:         *window.End(),
+			Key:         podKey,
+			Allocations: map[string]*kubecost.Allocation{},
+		}
+
+		podMap[podKey].AppendContainer(container)
+		podMap[podKey].Allocations[container].Properties.SetCluster(cluster)
+		podMap[podKey].Allocations[container].Properties.SetNode(node)
+		podMap[podKey].Allocations[container].Properties.SetNamespace(namespace)
+		podMap[podKey].Allocations[container].Properties.SetPod(pod)
+		podMap[podKey].Allocations[container].Properties.SetContainer(container)
+		podMap[podKey].Allocations[container].PVByteHours = unmountedPVCBytes[key] * window.Minutes() / 60.0
+		podMap[podKey].Allocations[container].PVCost = amount
+		podMap[podKey].Allocations[container].TotalCost = amount
+	}
+}
+
+// TODO niko/computealloction comment
+type Pod struct {
+	Window      kubecost.Window
+	Start       time.Time
+	End         time.Time
+	Key         podKey
+	Allocations map[string]*kubecost.Allocation
+}
+
+// TODO niko/computealloction comment
+func (p Pod) AppendContainer(container string) {
+	name := fmt.Sprintf("%s/%s/%s/%s", p.Key.Cluster, p.Key.Namespace, p.Key.Pod, container)
+
+	alloc := &kubecost.Allocation{
+		Name:       name,
+		Properties: kubecost.Properties{},
+		Window:     p.Window.Clone(),
+		Start:      p.Start,
+		End:        p.End,
+	}
+	alloc.Properties.SetContainer(container)
+	alloc.Properties.SetPod(p.Key.Pod)
+	alloc.Properties.SetNamespace(p.Key.Namespace)
+	alloc.Properties.SetCluster(p.Key.Cluster)
+
+	p.Allocations[container] = alloc
+}
+
+// PVC describes a PersistentVolumeClaim
+// TODO move to pkg/kubecost?  [TODO:CLEANUP]
+// TODO add PersistentVolumeClaims field to type Allocation?  [TODO:CLEANUP]
+type PVC struct {
+	Bytes     float64   `json:"bytes"`
+	Count     int       `json:"count"`
+	Name      string    `json:"name"`
+	Cluster   string    `json:"cluster"`
+	Namespace string    `json:"namespace"`
+	Volume    *PV       `json:"persistentVolume"`
+	Mounted   bool      `json:"mounted"`
+	Start     time.Time `json:"start"`
+	End       time.Time `json:"end"`
+}
+
+// Cost computes the cumulative cost of the PVC
+func (pvc *PVC) Cost() float64 {
+	if pvc == nil || pvc.Volume == nil {
+		return 0.0
+	}
+
+	gib := pvc.Bytes / 1024 / 1024 / 1024
+	hrs := pvc.Minutes() / 60.0
+
+	return pvc.Volume.CostPerGiBHour * gib * hrs
+}
+
+// Minutes computes the number of minutes over which the PVC is defined
+func (pvc *PVC) Minutes() float64 {
+	if pvc == nil {
+		return 0.0
+	}
+
+	return pvc.End.Sub(pvc.Start).Minutes()
+}
+
+// String returns a string representation of the PVC
+func (pvc *PVC) String() string {
+	if pvc == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("%s/%s/%s{Bytes:%.2f, Cost:%.6f, Start,End:%s}", pvc.Cluster, pvc.Namespace, pvc.Name, pvc.Bytes, pvc.Cost(), kubecost.NewWindow(&pvc.Start, &pvc.End))
+}
+
+// PV describes a PersistentVolume
+// TODO move to pkg/kubecost? [TODO:CLEANUP]
+type PV struct {
+	Bytes          float64 `json:"bytes"`
+	CostPerGiBHour float64 `json:"costPerGiBHour"` // TODO niko/computeallocation GiB or GB?
+	Cluster        string  `json:"cluster"`
+	Name           string  `json:"name"`
+	StorageClass   string  `json:"storageClass"`
+}
+
+// String returns a string representation of the PV
+func (pv *PV) String() string {
+	if pv == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("%s/%s{Bytes:%.2f, Cost/GiB*Hr:%.6f, StorageClass:%s}", pv.Cluster, pv.Name, pv.Bytes, pv.CostPerGiBHour, pv.StorageClass)
+}

pkg/costmodel/cluster.go

@@ -382,23 +382,26 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, duration, o
 }
 
 type Node struct {
-	Cluster      string
-	Name         string
-	ProviderID   string
-	NodeType     string
-	CPUCost      float64
-	CPUCores     float64
-	GPUCost      float64
-	RAMCost      float64
-	RAMBytes     float64
-	Discount     float64
-	Preemptible  bool
-	CPUBreakdown *ClusterCostsBreakdown
-	RAMBreakdown *ClusterCostsBreakdown
-	Start        time.Time
-	End          time.Time
-	Minutes      float64
-	Labels       map[string]string
+	Cluster         string
+	Name            string
+	ProviderID      string
+	NodeType        string
+	CPUCost         float64
+	CPUCores        float64
+	GPUCost         float64
+	RAMCost         float64
+	RAMBytes        float64
+	Discount        float64
+	Preemptible     bool
+	CPUBreakdown    *ClusterCostsBreakdown
+	RAMBreakdown    *ClusterCostsBreakdown
+	Start           time.Time
+	End             time.Time
+	Minutes         float64
+	Labels          map[string]string
+	CostPerCPUHr    float64
+	CostPerRAMGiBHr float64
+	CostPerGPUHr    float64
 }
 
 // GKE lies about the number of cores e2 nodes have. This table
@@ -540,7 +543,7 @@ func ClusterNodes(cp cloud.Provider, client prometheus.Client, duration, offset
 	}
 
 	for _, node := range nodeMap {
-		// TODO take RI into account
+		// TODO take GKE Reserved Instances into account
 		node.Discount = cp.CombinedDiscountForNode(node.NodeType, node.Preemptible, discount, negotiatedDiscount)
 
 		// Apply all remaining resources to Idle

pkg/costmodel/costmodel.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/util"
+	prometheus "github.com/prometheus/client_golang/api"
 	prometheusClient "github.com/prometheus/client_golang/api"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,22 +47,26 @@ const (
 var isCron = regexp.MustCompile(`^(.+)-\d{10}$`)
 
 type CostModel struct {
-	Cache           clustercache.ClusterCache
-	ClusterMap      clusters.ClusterMap
-	ScrapeInterval  time.Duration
-	RequestGroup    *singleflight.Group
-	pricingMetadata *costAnalyzerCloud.PricingMatchMetadata
+	Cache            clustercache.ClusterCache
+	ClusterMap       clusters.ClusterMap
+	RequestGroup     *singleflight.Group
+	ScrapeInterval   time.Duration
+	PrometheusClient prometheus.Client
+	Provider         costAnalyzerCloud.Provider
+	pricingMetadata  *costAnalyzerCloud.PricingMatchMetadata
 }
 
-func NewCostModel(cache clustercache.ClusterCache, clusterMap clusters.ClusterMap, scrapeInterval time.Duration) *CostModel {
+func NewCostModel(client prometheus.Client, provider costAnalyzerCloud.Provider, cache clustercache.ClusterCache, clusterMap clusters.ClusterMap, scrapeInterval time.Duration) *CostModel {
 	// request grouping to prevent over-requesting the same data prior to caching
 	requestGroup := new(singleflight.Group)
 
 	return &CostModel{
-		Cache:          cache,
-		ClusterMap:     clusterMap,
-		RequestGroup:   requestGroup,
-		ScrapeInterval: scrapeInterval,
+		Cache:            cache,
+		ClusterMap:       clusterMap,
+		PrometheusClient: client,
+		Provider:         provider,
+		RequestGroup:     requestGroup,
+		ScrapeInterval:   scrapeInterval,
 	}
 }
 
@@ -1506,10 +1511,6 @@ func requestKeyFor(window kubecost.Window, resolution time.Duration, filterNames
 	return fmt.Sprintf("%s,%s,%s,%s,%s,%t", startKey, endKey, resolution.String(), filterNamespace, filterCluster, remoteEnabled)
 }
 
-// func (cm *CostModel) ComputeCostDataRange(cli prometheusClient.Client, cp costAnalyzerCloud.Provider,
-// 	startString, endString, windowString string, resolutionHours float64, filterNamespace string,
-// 	filterCluster string, remoteEnabled bool, offset string) (map[string]*CostData, error)
-
 // ComputeCostDataRange executes a range query for cost data.
 // Note that "offset" represents the time between the function call and "endString", and is also passed for convenience
 func (cm *CostModel) ComputeCostDataRange(cli prometheusClient.Client, cp costAnalyzerCloud.Provider, window kubecost.Window, resolution time.Duration, filterNamespace string, filterCluster string, remoteEnabled bool) (map[string]*CostData, error) {

pkg/costmodel/key.go

@@ -0,0 +1,348 @@
+package costmodel
+
+import (
+	"fmt"
+
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/prom"
+)
+
+type containerKey struct {
+	Cluster   string
+	Namespace string
+	Pod       string
+	Container string
+}
+
+func (k containerKey) String() string {
+	return fmt.Sprintf("%s/%s/%s/%s", k.Cluster, k.Namespace, k.Pod, k.Container)
+}
+
+func newContainerKey(cluster, namespace, pod, container string) containerKey {
+	return containerKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+		Pod:       pod,
+		Container: container,
+	}
+}
+
+func resultContainerKey(res *prom.QueryResult, clusterLabel, namespaceLabel, podLabel, containerLabel string) (containerKey, error) {
+	key := containerKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	pod, err := res.GetString(podLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Pod = pod
+
+	container, err := res.GetString(containerLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Container = container
+
+	return key, nil
+}
+
+type podKey struct {
+	Cluster   string
+	Namespace string
+	Pod       string
+}
+
+func (k podKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.Cluster, k.Namespace, k.Pod)
+}
+
+func newPodKey(cluster, namespace, pod string) podKey {
+	return podKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+		Pod:       pod,
+	}
+}
+
+func resultPodKey(res *prom.QueryResult, clusterLabel, namespaceLabel, podLabel string) (podKey, error) {
+	key := podKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	pod, err := res.GetString(podLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Pod = pod
+
+	return key, nil
+}
+
+type namespaceKey struct {
+	Cluster   string
+	Namespace string
+}
+
+func (k namespaceKey) String() string {
+	return fmt.Sprintf("%s/%s", k.Cluster, k.Namespace)
+}
+
+func newNamespaceKey(cluster, namespace string) namespaceKey {
+	return namespaceKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+	}
+}
+
+func resultNamespaceKey(res *prom.QueryResult, clusterLabel, namespaceLabel string) (namespaceKey, error) {
+	key := namespaceKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	return key, nil
+}
+
+type controllerKey struct {
+	Cluster        string
+	Namespace      string
+	ControllerKind string
+	Controller     string
+}
+
+func (k controllerKey) String() string {
+	return fmt.Sprintf("%s/%s/%s/%s", k.Cluster, k.Namespace, k.ControllerKind, k.Controller)
+}
+
+func newControllerKey(cluster, namespace, controllerKind, controller string) controllerKey {
+	return controllerKey{
+		Cluster:        cluster,
+		Namespace:      namespace,
+		ControllerKind: controllerKind,
+		Controller:     controller,
+	}
+}
+
+func resultControllerKey(controllerKind string, res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	key := controllerKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	controller, err := res.GetString(controllerLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Controller = controller
+
+	key.ControllerKind = controllerKind
+
+	return key, nil
+}
+
+func resultDeploymentKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("deployment", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+func resultStatefulSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("statefulset", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+func resultDaemonSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("daemonset", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+func resultJobKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("job", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
+type serviceKey struct {
+	Cluster   string
+	Namespace string
+	Service   string
+}
+
+func (k serviceKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.Cluster, k.Namespace, k.Service)
+}
+
+func newServiceKey(cluster, namespace, service string) serviceKey {
+	return serviceKey{
+		Cluster:   cluster,
+		Namespace: namespace,
+		Service:   service,
+	}
+}
+
+func resultServiceKey(res *prom.QueryResult, clusterLabel, namespaceLabel, serviceLabel string) (serviceKey, error) {
+	key := serviceKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	service, err := res.GetString(serviceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Service = service
+
+	return key, nil
+}
+
+type nodeKey struct {
+	Cluster string
+	Node    string
+}
+
+func (k nodeKey) String() string {
+	return fmt.Sprintf("%s/%s", k.Cluster, k.Node)
+}
+
+func newNodeKey(cluster, node string) nodeKey {
+	return nodeKey{
+		Cluster: cluster,
+		Node:    node,
+	}
+}
+
+func resultNodeKey(res *prom.QueryResult, clusterLabel, nodeLabel string) (nodeKey, error) {
+	key := nodeKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	node, err := res.GetString(nodeLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Node = node
+
+	return key, nil
+}
+
+type pvcKey struct {
+	Cluster               string
+	Namespace             string
+	PersistentVolumeClaim string
+}
+
+func (k pvcKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.Cluster, k.Namespace, k.PersistentVolumeClaim)
+}
+
+func newPVCKey(cluster, namespace, persistentVolumeClaim string) pvcKey {
+	return pvcKey{
+		Cluster:               cluster,
+		Namespace:             namespace,
+		PersistentVolumeClaim: persistentVolumeClaim,
+	}
+}
+
+func resultPVCKey(res *prom.QueryResult, clusterLabel, namespaceLabel, pvcLabel string) (pvcKey, error) {
+	key := pvcKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	namespace, err := res.GetString(namespaceLabel)
+	if err != nil {
+		return key, err
+	}
+	key.Namespace = namespace
+
+	pvc, err := res.GetString(pvcLabel)
+	if err != nil {
+		return key, err
+	}
+	key.PersistentVolumeClaim = pvc
+
+	return key, nil
+}
+
+type pvKey struct {
+	Cluster          string
+	PersistentVolume string
+}
+
+func (k pvKey) String() string {
+	return fmt.Sprintf("%s/%s", k.Cluster, k.PersistentVolume)
+}
+
+func newPVKey(cluster, persistentVolume string) pvKey {
+	return pvKey{
+		Cluster:          cluster,
+		PersistentVolume: persistentVolume,
+	}
+}
+
+func resultPVKey(res *prom.QueryResult, clusterLabel, persistentVolumeLabel string) (pvKey, error) {
+	key := pvKey{}
+
+	cluster, err := res.GetString(clusterLabel)
+	if err != nil {
+		cluster = env.GetClusterID()
+	}
+	key.Cluster = cluster
+
+	persistentVolume, err := res.GetString(persistentVolumeLabel)
+	if err != nil {
+		return key, err
+	}
+	key.PersistentVolume = persistentVolume
+
+	return key, nil
+}

pkg/costmodel/router.go

@@ -28,6 +28,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/thanos"
+	prometheus "github.com/prometheus/client_golang/api"
 	prometheusClient "github.com/prometheus/client_golang/api"
 	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
 	v1 "k8s.io/api/core/v1"
@@ -243,6 +244,7 @@ func ParsePercentString(percentStr string) (float64, error) {
 }
 
 // parseDuration converts a Prometheus-style duration string into a Duration
+// [TODO:CLEANUP] delete this. do it now.
 func ParseDuration(duration string) (*time.Duration, error) {
 	unitStr := duration[len(duration)-1:]
 	var unit time.Duration
@@ -1013,7 +1015,13 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		30 * day: maxCacheMinutes30d * time.Minute,
 	}
 
-	costModel := NewCostModel(k8sCache, clusterMap, scrapeInterval)
+	var pc prometheus.Client
+	if thanosClient != nil {
+		pc = thanosClient
+	} else {
+		pc = promCli
+	}
+	costModel := NewCostModel(pc, cloudProvider, k8sCache, clusterMap, scrapeInterval)
 	metricsEmitter := NewCostModelMetricsEmitter(promCli, k8sCache, cloudProvider, costModel)
 
 	a := &Accesses{

pkg/kubecost/allocation.go

@@ -1,6 +1,7 @@
 package kubecost
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"sort"
@@ -28,6 +29,9 @@ const SharedSuffix = "__shared__"
 // UnallocatedSuffix indicates an unallocated allocation property
 const UnallocatedSuffix = "__unallocated__"
 
+// UnmountedSuffix indicated allocation to an unmounted PV
+const UnmountedSuffix = "__unmounted__"
+
 // ShareWeighted indicates that a shared resource should be shared as a
 // proportion of the cost of the remaining allocations.
 const ShareWeighted = "__weighted__"
@@ -42,28 +46,29 @@ const ShareNone = "__none__"
 // Allocation is a unit of resource allocation and cost for a given window
 // of time and for a given kubernetes construct with its associated set of
 // properties.
+// TODO niko/computeallocation compute efficiency on the fly?
 type Allocation struct {
-	Name            string     `json:"name"`
-	Properties      Properties `json:"properties,omitempty"`
-	Start           time.Time  `json:"start"`
-	End             time.Time  `json:"end"`
-	Minutes         float64    `json:"minutes"`
-	ActiveStart     time.Time  `json:"-"`
-	CPUCoreHours    float64    `json:"cpuCoreHours"`
-	CPUCost         float64    `json:"cpuCost"`
-	CPUEfficiency   float64    `json:"cpuEfficiency"`
-	GPUHours        float64    `json:"gpuHours"`
-	GPUCost         float64    `json:"gpuCost"`
-	NetworkCost     float64    `json:"networkCost"`
-	PVByteHours     float64    `json:"pvByteHours"`
-	PVCost          float64    `json:"pvCost"`
-	RAMByteHours    float64    `json:"ramByteHours"`
-	RAMCost         float64    `json:"ramCost"`
-	RAMEfficiency   float64    `json:"ramEfficiency"`
-	SharedCost      float64    `json:"sharedCost"`
-	ExternalCost    float64    `json:"externalCost"`
-	TotalCost       float64    `json:"totalCost"`
-	TotalEfficiency float64    `json:"totalEfficiency"`
+	Name                   string     `json:"name"`
+	Properties             Properties `json:"properties,omitempty"`
+	Window                 Window     `json:"window"`
+	Start                  time.Time  `json:"start"`
+	End                    time.Time  `json:"end"`
+	CPUCoreHours           float64    `json:"cpuCoreHours"`
+	CPUCoreRequestAverage  float64    `json:"cpuCoreRequestAverage"`
+	CPUCoreUsageAverage    float64    `json:"cpuCoreUsageAverage"`
+	CPUCost                float64    `json:"cpuCost"`
+	GPUHours               float64    `json:"gpuHours"`
+	GPUCost                float64    `json:"gpuCost"`
+	NetworkCost            float64    `json:"networkCost"`
+	PVByteHours            float64    `json:"pvByteHours"`
+	PVCost                 float64    `json:"pvCost"`
+	RAMByteHours           float64    `json:"ramByteHours"`
+	RAMBytesRequestAverage float64    `json:"ramBytesRequestAverage"`
+	RAMBytesUsageAverage   float64    `json:"ramBytesUsageAverage"`
+	RAMCost                float64    `json:"ramCost"`
+	SharedCost             float64    `json:"sharedCost"`
+	ExternalCost           float64    `json:"externalCost"`
+	TotalCost              float64    `json:"totalCost"`
 }
 
 // AllocationMatchFunc is a function that can be used to match Allocations by
@@ -78,12 +83,13 @@ func (a *Allocation) Add(that *Allocation) (*Allocation, error) {
 		return that.Clone(), nil
 	}
 
-	if !a.Start.Equal(that.Start) || !a.End.Equal(that.End) {
-		return nil, fmt.Errorf("error adding Allocations: mismatched windows")
+	if that == nil {
+		return a.Clone(), nil
 	}
 
+	// Note: no need to clone "that", as add only mutates the receiver
 	agg := a.Clone()
-	agg.add(that, false, false)
+	agg.add(that)
 
 	return agg, nil
 }
@@ -95,27 +101,27 @@ func (a *Allocation) Clone() *Allocation {
 	}
 
 	return &Allocation{
-		Name:            a.Name,
-		Properties:      a.Properties.Clone(),
-		Start:           a.Start,
-		End:             a.End,
-		Minutes:         a.Minutes,
-		ActiveStart:     a.ActiveStart,
-		CPUCoreHours:    a.CPUCoreHours,
-		CPUCost:         a.CPUCost,
-		CPUEfficiency:   a.CPUEfficiency,
-		GPUHours:        a.GPUHours,
-		GPUCost:         a.GPUCost,
-		NetworkCost:     a.NetworkCost,
-		PVByteHours:     a.PVByteHours,
-		PVCost:          a.PVCost,
-		RAMByteHours:    a.RAMByteHours,
-		RAMCost:         a.RAMCost,
-		RAMEfficiency:   a.RAMEfficiency,
-		SharedCost:      a.SharedCost,
-		ExternalCost:    a.ExternalCost,
-		TotalCost:       a.TotalCost,
-		TotalEfficiency: a.TotalEfficiency,
+		Name:                   a.Name,
+		Properties:             a.Properties.Clone(),
+		Window:                 a.Window.Clone(),
+		Start:                  a.Start,
+		End:                    a.End,
+		CPUCoreHours:           a.CPUCoreHours,
+		CPUCoreRequestAverage:  a.CPUCoreRequestAverage,
+		CPUCoreUsageAverage:    a.CPUCoreUsageAverage,
+		CPUCost:                a.CPUCost,
+		GPUHours:               a.GPUHours,
+		GPUCost:                a.GPUCost,
+		NetworkCost:            a.NetworkCost,
+		PVByteHours:            a.PVByteHours,
+		PVCost:                 a.PVCost,
+		RAMByteHours:           a.RAMByteHours,
+		RAMBytesRequestAverage: a.RAMBytesRequestAverage,
+		RAMBytesUsageAverage:   a.RAMBytesUsageAverage,
+		RAMCost:                a.RAMCost,
+		SharedCost:             a.SharedCost,
+		ExternalCost:           a.ExternalCost,
+		TotalCost:              a.TotalCost,
 	}
 }
 
@@ -129,16 +135,16 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.Name != that.Name {
 		return false
 	}
-	if !a.Start.Equal(that.Start) {
+	if !a.Properties.Equal(&that.Properties) {
 		return false
 	}
-	if !a.End.Equal(that.End) {
+	if !a.Window.Equal(that.Window) {
 		return false
 	}
-	if a.Minutes != that.Minutes {
+	if !a.Start.Equal(that.Start) {
 		return false
 	}
-	if !a.ActiveStart.Equal(that.ActiveStart) {
+	if !a.End.Equal(that.End) {
 		return false
 	}
 	if a.CPUCoreHours != that.CPUCoreHours {
@@ -147,9 +153,6 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.CPUCost != that.CPUCost {
 		return false
 	}
-	if a.CPUEfficiency != that.CPUEfficiency {
-		return false
-	}
 	if a.GPUHours != that.GPUHours {
 		return false
 	}
@@ -171,9 +174,6 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.RAMCost != that.RAMCost {
 		return false
 	}
-	if a.RAMEfficiency != that.RAMEfficiency {
-		return false
-	}
 	if a.SharedCost != that.SharedCost {
 		return false
 	}
@@ -183,16 +183,113 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.TotalCost != that.TotalCost {
 		return false
 	}
-	if a.TotalEfficiency != that.TotalEfficiency {
-		return false
+
+	return true
+}
+
+// CPUEfficiency is the ratio of usage to request. If there is no request and
+// no usage or cost, then efficiency is zero. If there is no request, but there
+// is usage or cost, then efficiency is 100%.
+func (a *Allocation) CPUEfficiency() float64 {
+	if a.CPUCoreRequestAverage > 0 {
+		return a.CPUCoreUsageAverage / a.CPUCoreRequestAverage
 	}
-	if !a.Properties.Equal(&that.Properties) {
-		return false
+
+	if a.CPUCoreUsageAverage == 0.0 || a.CPUCost == 0.0 {
+		return 0.0
 	}
 
-	return true
+	return 1.0
+}
+
+// RAMEfficiency is the ratio of usage to request. If there is no request and
+// no usage or cost, then efficiency is zero. If there is no request, but there
+// is usage or cost, then efficiency is 100%.
+func (a *Allocation) RAMEfficiency() float64 {
+	if a.RAMBytesRequestAverage > 0 {
+		return a.RAMBytesUsageAverage / a.RAMBytesRequestAverage
+	}
+
+	if a.RAMBytesUsageAverage == 0.0 || a.RAMCost == 0.0 {
+		return 0.0
+	}
+
+	return 1.0
+}
+
+// TotalEfficiency is the cost-weighted average of CPU and RAM efficiency. If
+// there is no cost at all, then efficiency is zero.
+func (a *Allocation) TotalEfficiency() float64 {
+	if a.CPUCost+a.RAMCost > 0 {
+		ramCostEff := a.RAMEfficiency() * a.RAMCost
+		cpuCostEff := a.CPUEfficiency() * a.CPUCost
+		return (ramCostEff + cpuCostEff) / (a.CPUCost + a.RAMCost)
+	}
+
+	return 0.0
 }
 
+// CPUCores converts the Allocation's CPUCoreHours into average CPUCores
+func (a *Allocation) CPUCores() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.CPUCoreHours / (a.Minutes() / 60.0)
+}
+
+// RAMBytes converts the Allocation's RAMByteHours into average RAMBytes
+func (a *Allocation) RAMBytes() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.RAMByteHours / (a.Minutes() / 60.0)
+}
+
+// PVBytes converts the Allocation's PVByteHours into average PVBytes
+func (a *Allocation) PVBytes() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.PVByteHours / (a.Minutes() / 60.0)
+}
+
+// MarshalJSON implements json.Marshal interface
+func (a *Allocation) MarshalJSON() ([]byte, error) {
+	buffer := bytes.NewBufferString("{")
+	jsonEncodeString(buffer, "name", a.Name, ",")
+	jsonEncode(buffer, "properties", a.Properties, ",")
+	jsonEncode(buffer, "window", a.Window, ",")
+	jsonEncodeString(buffer, "start", a.Start.Format(timeFmt), ",")
+	jsonEncodeString(buffer, "end", a.End.Format(timeFmt), ",")
+	jsonEncodeFloat64(buffer, "minutes", a.Minutes(), ",")
+	jsonEncodeFloat64(buffer, "cpuCores", a.CPUCores(), ",")
+	jsonEncodeFloat64(buffer, "cpuCoreRequestAverage", a.CPUCoreRequestAverage, ",")
+	jsonEncodeFloat64(buffer, "cpuCoreUsageAverage", a.CPUCoreUsageAverage, ",")
+	jsonEncodeFloat64(buffer, "cpuCoreHours", a.CPUCoreHours, ",")
+	jsonEncodeFloat64(buffer, "cpuCost", a.CPUCost, ",")
+	jsonEncodeFloat64(buffer, "cpuEfficiency", a.CPUEfficiency(), ",")
+	jsonEncodeFloat64(buffer, "gpuHours", a.GPUHours, ",")
+	jsonEncodeFloat64(buffer, "gpuCost", a.GPUCost, ",")
+	jsonEncodeFloat64(buffer, "networkCost", a.NetworkCost, ",")
+	jsonEncodeFloat64(buffer, "pvBytes", a.PVBytes(), ",")
+	jsonEncodeFloat64(buffer, "pvByteHours", a.PVByteHours, ",")
+	jsonEncodeFloat64(buffer, "pvCost", a.PVCost, ",")
+	jsonEncodeFloat64(buffer, "ramBytes", a.RAMBytes(), ",")
+	jsonEncodeFloat64(buffer, "ramByteRequestAverage", a.RAMBytesRequestAverage, ",")
+	jsonEncodeFloat64(buffer, "ramByteUsageAverage", a.RAMBytesUsageAverage, ",")
+	jsonEncodeFloat64(buffer, "ramByteHours", a.RAMByteHours, ",")
+	jsonEncodeFloat64(buffer, "ramCost", a.RAMCost, ",")
+	jsonEncodeFloat64(buffer, "ramEfficiency", a.RAMEfficiency(), ",")
+	jsonEncodeFloat64(buffer, "sharedCost", a.SharedCost, ",")
+	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost, ",")
+	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), "")
+	buffer.WriteString("}")
+	return buffer.Bytes(), nil
+}
+
+// TODO niko/computeallocation
+// func (a *Allocation)UnmarshalJSON()
+
 // Resolution returns the duration of time covered by the Allocation
 func (a *Allocation) Resolution() time.Duration {
 	return a.End.Sub(a.Start)
@@ -219,22 +316,41 @@ func (a *Allocation) IsUnallocated() bool {
 	return strings.Contains(a.Name, UnallocatedSuffix)
 }
 
+// Minutes returns the number of minutes the Allocation represents, as defined
+// by the difference between the end and start times.
+func (a *Allocation) Minutes() float64 {
+	return a.End.Sub(a.Start).Minutes()
+}
+
 // Share works like Add, but converts the entire cost of the given Allocation
 // to SharedCost, rather than adding to the individual resource costs.
+// TODO niko/computeallocation unit test changes!!!
 func (a *Allocation) Share(that *Allocation) (*Allocation, error) {
-	if a == nil {
-		return that.Clone(), nil
-	}
+	if that == nil {
+		return a.Clone(), nil
+	}
+
+	// Convert all costs of shared Allocation to SharedCost, zero out all
+	// non-shared costs, then add.
+	share := that.Clone()
+	share.SharedCost += share.TotalCost
+	share.CPUCost = 0
+	share.CPUCoreHours = 0
+	share.RAMCost = 0
+	share.RAMByteHours = 0
+	share.GPUCost = 0
+	share.GPUHours = 0
+	share.PVCost = 0
+	share.PVByteHours = 0
+	share.NetworkCost = 0
+	share.ExternalCost = 0
 
-	if !a.Start.Equal(that.Start) {
-		return nil, fmt.Errorf("mismatched start time: expected %s, received %s", a.Start, that.Start)
-	}
-	if !a.End.Equal(that.End) {
-		return nil, fmt.Errorf("mismatched start time: expected %s, received %s", a.End, that.End)
+	if a == nil {
+		return share, nil
 	}
 
 	agg := a.Clone()
-	agg.add(that, true, false)
+	agg.add(that)
 
 	return agg, nil
 }
@@ -244,7 +360,7 @@ func (a *Allocation) String() string {
 	return fmt.Sprintf("%s%s=%.2f", a.Name, NewWindow(&a.Start, &a.End), a.TotalCost)
 }
 
-func (a *Allocation) add(that *Allocation, isShared, isAccumulating bool) {
+func (a *Allocation) add(that *Allocation) {
 	if a == nil {
 		log.Warningf("Allocation.AggregateBy: trying to add a nil receiver")
 		return
@@ -271,65 +387,53 @@ func (a *Allocation) add(that *Allocation, isShared, isAccumulating bool) {
 		}
 	}
 
-	if that.ActiveStart.Before(a.ActiveStart) {
-		a.ActiveStart = that.ActiveStart
-	}
+	// Expand the window to encompass both Allocations
+	a.Window = a.Window.Expand(that.Window)
 
-	if isAccumulating {
-		if a.Start.After(that.Start) {
-			a.Start = that.Start
-		}
+	// Sum non-cumulative fields by turning them into cumulative, adding them,
+	// and then converting them back into averages after minutes have been
+	// combined (just below).
+	cpuReqCoreMins := a.CPUCoreRequestAverage * a.Minutes()
+	cpuReqCoreMins += that.CPUCoreRequestAverage * that.Minutes()
 
-		if a.End.Before(that.End) {
-			a.End = that.End
-		}
+	cpuUseCoreMins := a.CPUCoreUsageAverage * a.Minutes()
+	cpuUseCoreMins += that.CPUCoreUsageAverage * that.Minutes()
 
-		a.Minutes += that.Minutes
-	} else if that.Minutes > a.Minutes {
-		a.Minutes = that.Minutes
-	}
+	ramReqByteMins := a.RAMBytesRequestAverage * a.Minutes()
+	ramReqByteMins += that.RAMBytesRequestAverage * that.Minutes()
 
-	// isShared determines whether the given allocation should be spread evenly
-	// across resources (e.g. sharing idle allocation) or lumped into a shared
-	// cost category (e.g. sharing namespace, labels).
-	if isShared {
-		a.SharedCost += that.TotalCost
-	} else {
-		a.CPUCoreHours += that.CPUCoreHours
-		a.GPUHours += that.GPUHours
-		a.RAMByteHours += that.RAMByteHours
-		a.PVByteHours += that.PVByteHours
-
-		aggCPUCost := a.CPUCost + that.CPUCost
-		if aggCPUCost > 0 {
-			a.CPUEfficiency = (a.CPUEfficiency*a.CPUCost + that.CPUEfficiency*that.CPUCost) / aggCPUCost
-		} else {
-			a.CPUEfficiency = 0.0
-		}
+	ramUseByteMins := a.RAMBytesUsageAverage * a.Minutes()
+	ramUseByteMins += that.RAMBytesUsageAverage * that.Minutes()
 
-		aggRAMCost := a.RAMCost + that.RAMCost
-		if aggRAMCost > 0 {
-			a.RAMEfficiency = (a.RAMEfficiency*a.RAMCost + that.RAMEfficiency*that.RAMCost) / aggRAMCost
-		} else {
-			a.RAMEfficiency = 0.0
-		}
+	// Expand Start and End to be the "max" of among the given Allocations
+	if that.Start.Before(a.Start) {
+		a.Start = that.Start
+	}
+	if that.End.After(a.End) {
+		a.End = that.End
+	}
 
-		aggTotalCost := a.TotalCost + that.TotalCost
-		if aggTotalCost > 0 {
-			a.TotalEfficiency = (a.TotalEfficiency*(a.TotalCost-a.ExternalCost) + that.TotalEfficiency*(that.TotalCost-that.ExternalCost)) / (aggTotalCost - a.ExternalCost - that.ExternalCost)
-		} else {
-			aggTotalCost = 0.0
-		}
+	// Convert cumulatuve request and usage back into rates
+	// TODO niko/computeallocation write a unit test that fails if this is done incorrectly
+	a.CPUCoreRequestAverage = cpuReqCoreMins / a.Minutes()
+	a.CPUCoreUsageAverage = cpuUseCoreMins / a.Minutes()
+	a.RAMBytesRequestAverage = ramReqByteMins / a.Minutes()
+	a.RAMBytesUsageAverage = ramUseByteMins / a.Minutes()
 
-		a.SharedCost += that.SharedCost
-		a.ExternalCost += that.ExternalCost
-		a.CPUCost += that.CPUCost
-		a.GPUCost += that.GPUCost
-		a.NetworkCost += that.NetworkCost
-		a.RAMCost += that.RAMCost
-		a.PVCost += that.PVCost
-	}
+	// Sum all cumulative resource fields
+	a.CPUCoreHours += that.CPUCoreHours
+	a.GPUHours += that.GPUHours
+	a.RAMByteHours += that.RAMByteHours
+	a.PVByteHours += that.PVByteHours
 
+	// Sum all cumulative cost fields
+	a.CPUCost += that.CPUCost
+	a.GPUCost += that.GPUCost
+	a.RAMCost += that.RAMCost
+	a.PVCost += that.PVCost
+	a.NetworkCost += that.NetworkCost
+	a.SharedCost += that.SharedCost
+	a.ExternalCost += that.ExternalCost
 	a.TotalCost += that.TotalCost
 }
 
@@ -1198,7 +1302,7 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 		if s, ok := clusterStarts[cluster]; !ok || a.Start.Before(s) {
 			clusterStarts[cluster] = a.Start
 		}
-		if e, ok := clusterEnds[cluster]; !ok || a.End.Before(e) {
+		if e, ok := clusterEnds[cluster]; !ok || a.End.After(e) {
 			clusterEnds[cluster] = a.End
 		}
 
@@ -1223,10 +1327,10 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 
 		idleAlloc := &Allocation{
 			Name:       fmt.Sprintf("%s/%s", cluster, IdleSuffix),
+			Window:     window.Clone(),
 			Properties: Properties{ClusterProp: cluster},
 			Start:      start,
 			End:        end,
-			Minutes:    end.Sub(start).Minutes(), // TODO deprecate w/ niko/allocation-minutes
 			CPUCost:    resources["cpu"],
 			GPUCost:    resources["gpu"],
 			RAMCost:    resources["ram"],
@@ -1357,10 +1461,10 @@ func (as *AllocationSet) IdleAllocations() map[string]*Allocation {
 // but only if the Allocation is valid, i.e. matches the AllocationSet's window. If
 // there is no existing entry, one is created. Nil error response indicates success.
 func (as *AllocationSet) Insert(that *Allocation) error {
-	return as.insert(that, false)
+	return as.insert(that)
 }
 
-func (as *AllocationSet) insert(that *Allocation, accumulate bool) error {
+func (as *AllocationSet) insert(that *Allocation) error {
 	if as == nil {
 		return fmt.Errorf("cannot insert into nil AllocationSet")
 	}
@@ -1385,7 +1489,7 @@ func (as *AllocationSet) insert(that *Allocation, accumulate bool) error {
 	if _, ok := as.allocations[that.Name]; !ok {
 		as.allocations[that.Name] = that
 	} else {
-		as.allocations[that.Name].add(that, false, accumulate)
+		as.allocations[that.Name].add(that)
 	}
 
 	// If the given Allocation is an external one, record that
@@ -1527,12 +1631,6 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 		return as, nil
 	}
 
-	if that.Start().Before(as.End()) {
-		timefmt := "2006-01-02T15:04:05"
-		err := fmt.Sprintf("that [%s, %s); that [%s, %s)\n", as.Start().Format(timefmt), as.End().Format(timefmt), that.Start().Format(timefmt), that.End().Format(timefmt))
-		return nil, fmt.Errorf("error accumulating AllocationSets: overlapping windows: %s", err)
-	}
-
 	// Set start, end to min(start), max(end)
 	start := as.Start()
 	end := as.End()
@@ -1552,26 +1650,14 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 	defer that.RUnlock()
 
 	for _, alloc := range as.allocations {
-		// Change Start and End to match the new window. However, do not
-		// change Minutes because that will be accounted for during the
-		// insert step, if in fact there are two allocations to add.
-		alloc.Start = start
-		alloc.End = end
-
-		err := acc.insert(alloc, true)
+		err := acc.insert(alloc)
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	for _, alloc := range that.allocations {
-		// Change Start and End to match the new window. However, do not
-		// change Minutes because that will be accounted for during the
-		// insert step, if in fact there are two allocations to add.
-		alloc.Start = start
-		alloc.End = end
-
-		err := acc.insert(alloc, true)
+		err := acc.insert(alloc)
 		if err != nil {
 			return nil, err
 		}

pkg/kubecost/allocation_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	util "github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 const day = 24 * time.Hour
@@ -32,24 +32,25 @@ func NewUnitAllocation(name string, start time.Time, resolution time.Duration, p
 	end := start.Add(resolution)
 
 	alloc := &Allocation{
-		Name:            name,
-		Properties:      *properties,
-		Start:           start,
-		End:             end,
-		Minutes:         1440,
-		CPUCoreHours:    1,
-		CPUCost:         1,
-		CPUEfficiency:   1,
-		GPUHours:        1,
-		GPUCost:         1,
-		NetworkCost:     1,
-		PVByteHours:     1,
-		PVCost:          1,
-		RAMByteHours:    1,
-		RAMCost:         1,
-		RAMEfficiency:   1,
-		TotalCost:       5,
-		TotalEfficiency: 1,
+		Name:                   name,
+		Properties:             *properties,
+		Window:                 NewWindow(&start, &end).Clone(),
+		Start:                  start,
+		End:                    end,
+		CPUCoreHours:           1,
+		CPUCost:                1,
+		CPUCoreRequestAverage:  1,
+		CPUCoreUsageAverage:    1,
+		GPUHours:               1,
+		GPUCost:                1,
+		NetworkCost:            1,
+		PVByteHours:            1,
+		PVCost:                 1,
+		RAMByteHours:           1,
+		RAMCost:                1,
+		RAMBytesRequestAverage: 1,
+		RAMBytesUsageAverage:   1,
+		TotalCost:              5,
 	}
 
 	// If idle allocation, remove non-idle costs, but maintain total cost
@@ -305,8 +306,8 @@ func assertAllocationWindow(t *testing.T, as *AllocationSet, msg string, expStar
 		if !a.End.Equal(expEnd) {
 			t.Fatalf("AllocationSet.AggregateBy[%s]: expected end %s, actual %s", msg, expEnd, a.End)
 		}
-		if a.Minutes != expMinutes {
-			t.Fatalf("AllocationSet.AggregateBy[%s]: expected minutes %f, actual %f", msg, expMinutes, a.Minutes)
+		if a.Minutes() != expMinutes {
+			t.Fatalf("AllocationSet.AggregateBy[%s]: expected minutes %f, actual %f", msg, expMinutes, a.Minutes())
 		}
 	})
 }
@@ -1221,8 +1222,8 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if alloc.CPUCost != 2.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 2.0; actual %f", alloc.CPUCost)
 	}
-	if alloc.CPUEfficiency != 1.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.CPUEfficiency)
+	if alloc.CPUEfficiency() != 1.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.CPUEfficiency())
 	}
 	if alloc.GPUHours != 2.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 2.0; actual %f", alloc.GPUHours)
@@ -1245,14 +1246,14 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if alloc.RAMCost != 2.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 2.0; actual %f", alloc.RAMCost)
 	}
-	if alloc.RAMEfficiency != 1.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.RAMEfficiency)
+	if alloc.RAMEfficiency() != 1.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.RAMEfficiency())
 	}
 	if alloc.TotalCost != 10.0 {
 		t.Fatalf("accumulating AllocationSetRange: expected 10.0; actual %f", alloc.TotalCost)
 	}
-	if alloc.TotalEfficiency != 1.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.TotalEfficiency)
+	if alloc.TotalEfficiency() != 1.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected 1.0; actual %f", alloc.TotalEfficiency())
 	}
 	if !alloc.Start.Equal(yesterday) {
 		t.Fatalf("accumulating AllocationSetRange: expected to start %s; actual %s", yesterday, alloc.Start)
@@ -1260,8 +1261,8 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if !alloc.End.Equal(tomorrow) {
 		t.Fatalf("accumulating AllocationSetRange: expected to end %s; actual %s", tomorrow, alloc.End)
 	}
-	if alloc.Minutes != 2880.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected %f minutes; actual %f", 2880.0, alloc.Minutes)
+	if alloc.Minutes() != 2880.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected %f minutes; actual %f", 2880.0, alloc.Minutes())
 	}
 }
 

pkg/kubecost/asset.go

@@ -5,7 +5,6 @@ import (
 	"encoding"
 	"encoding/json"
 	"fmt"
-	"math"
 	"strings"
 	"sync"
 	"time"
@@ -43,10 +42,10 @@ type Asset interface {
 	// Temporal values
 	Start() time.Time
 	End() time.Time
-	Minutes() float64
+	SetStartEnd(time.Time, time.Time)
 	Window() Window
 	ExpandWindow(Window)
-	SetStartEnd(time.Time, time.Time)
+	Minutes() float64
 
 	// Operations and comparisons
 	Add(Asset) Asset
@@ -217,6 +216,9 @@ func AssetToExternalAllocation(asset Asset, aggregateBy []string) (*Allocation,
 	return &Allocation{
 		Name:         strings.Join(names, "/"),
 		Properties:   props,
+		Window:       asset.Window().Clone(),
+		Start:        asset.Start(),
+		End:          asset.End(),
 		ExternalCost: asset.TotalCost(),
 		TotalCost:    asset.TotalCost(),
 	}, nil
@@ -2724,15 +2726,17 @@ func (as *AssetSet) Get(key string) (Asset, bool) {
 // configured properties to determine the key under which the Asset will
 // be inserted.
 func (as *AssetSet) Insert(asset Asset) error {
-	if as.IsEmpty() {
-		as.Lock()
-		as.assets = map[string]Asset{}
-		as.Unlock()
+	if as == nil {
+		return fmt.Errorf("cannot Insert into nil AssetSet")
 	}
 
 	as.Lock()
 	defer as.Unlock()
 
+	if as.assets == nil {
+		as.assets = map[string]Asset{}
+	}
+
 	// Determine key into which to Insert the Asset.
 	k, err := key(asset, as.aggregateBy)
 	if err != nil {
@@ -2855,9 +2859,11 @@ func (as *AssetSet) accumulate(that *AssetSet) (*AssetSet, error) {
 	// Set start, end to min(start), max(end)
 	start := as.Start()
 	end := as.End()
+
 	if that.Start().Before(start) {
 		start = that.Start()
 	}
+
 	if that.End().After(end) {
 		end = that.End()
 	}
@@ -3015,33 +3021,6 @@ func (asr *AssetSetRange) Window() Window {
 	return NewWindow(&start, &end)
 }
 
-// TODO move everything below to a separate package
-
-func jsonEncodeFloat64(buffer *bytes.Buffer, name string, val float64, comma string) {
-	var encoding string
-	if math.IsNaN(val) {
-		encoding = fmt.Sprintf("\"%s\":null%s", name, comma)
-	} else {
-		encoding = fmt.Sprintf("\"%s\":%f%s", name, val, comma)
-	}
-
-	buffer.WriteString(encoding)
-}
-
-func jsonEncodeString(buffer *bytes.Buffer, name, val, comma string) {
-	buffer.WriteString(fmt.Sprintf("\"%s\":\"%s\"%s", name, val, comma))
-}
-
-func jsonEncode(buffer *bytes.Buffer, name string, obj interface{}, comma string) {
-	buffer.WriteString(fmt.Sprintf("\"%s\":", name))
-	if bytes, err := json.Marshal(obj); err != nil {
-		buffer.WriteString("null")
-	} else {
-		buffer.Write(bytes)
-	}
-	buffer.WriteString(comma)
-}
-
 // Returns true if string slices a and b contain all of the same strings, in any order.
 func sameContents(a, b []string) bool {
 	if len(a) != len(b) {

pkg/kubecost/asset_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	util "github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 var start1 = time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
@@ -153,7 +153,7 @@ func assertAssetSet(t *testing.T, as *AssetSet, msg string, window Window, exps
 				t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected total cost %.2f, actual %.2f", msg, key, exp, a.TotalCost())
 			}
 			if !a.Window().Equal(window) {
-				t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected window %s, actual %s", msg, key, window, as.Window)
+				t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected window %s, actual %s", msg, key, window, a.Window())
 			}
 		} else {
 			t.Fatalf("AssetSet.AggregateBy[%s]: unexpected asset: %s", msg, key)
@@ -1010,6 +1010,7 @@ func TestAssetSetRange_Accumulate(t *testing.T) {
 		generateAssetSet(startD1),
 		generateAssetSet(startD2),
 	)
+
 	err = asr.AggregateBy([]string{string(AssetTypeProp)}, nil)
 	as, err = asr.Accumulate()
 	if err != nil {

pkg/kubecost/bingen.go

@@ -21,4 +21,4 @@ package kubecost
 // @bingen:generate:AllocationSet
 // @bingen:generate:AllocationSetRange
 
-//go:generate bingen -package=kubecost -version=5 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=7 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/json.go

@@ -0,0 +1,35 @@
+package kubecost
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math"
+)
+
+// TODO move everything below to a separate package
+
+func jsonEncodeFloat64(buffer *bytes.Buffer, name string, val float64, comma string) {
+	var encoding string
+	if math.IsNaN(val) || math.IsInf(val, 0) {
+		encoding = fmt.Sprintf("\"%s\":null%s", name, comma)
+	} else {
+		encoding = fmt.Sprintf("\"%s\":%f%s", name, val, comma)
+	}
+
+	buffer.WriteString(encoding)
+}
+
+func jsonEncodeString(buffer *bytes.Buffer, name, val, comma string) {
+	buffer.WriteString(fmt.Sprintf("\"%s\":\"%s\"%s", name, val, comma))
+}
+
+func jsonEncode(buffer *bytes.Buffer, name string, obj interface{}, comma string) {
+	buffer.WriteString(fmt.Sprintf("\"%s\":", name))
+	if bytes, err := json.Marshal(obj); err != nil {
+		buffer.WriteString("null")
+	} else {
+		buffer.Write(bytes)
+	}
+	buffer.WriteString(comma)
+}

pkg/kubecost/kubecost_codecs.go

@@ -25,7 +25,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 6
+	CodecVersion uint8 = 7
 )
 
 //--------------------------------------------------------------------------
@@ -125,17 +125,17 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteBytes(a)
 	// --- [end][write][reference](Properties) ---
 
-	// --- [begin][write][reference](time.Time) ---
-	b, errB := target.Start.MarshalBinary()
+	// --- [begin][write][struct](Window) ---
+	b, errB := target.Window.MarshalBinary()
 	if errB != nil {
 		return nil, errB
 	}
 	buff.WriteInt(len(b))
 	buff.WriteBytes(b)
-	// --- [end][write][reference](time.Time) ---
+	// --- [end][write][struct](Window) ---
 
 	// --- [begin][write][reference](time.Time) ---
-	c, errC := target.End.MarshalBinary()
+	c, errC := target.Start.MarshalBinary()
 	if errC != nil {
 		return nil, errC
 	}
@@ -143,9 +143,8 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteBytes(c)
 	// --- [end][write][reference](time.Time) ---
 
-	buff.WriteFloat64(target.Minutes) // write float64
 	// --- [begin][write][reference](time.Time) ---
-	d, errD := target.ActiveStart.MarshalBinary()
+	d, errD := target.End.MarshalBinary()
 	if errD != nil {
 		return nil, errD
 	}
@@ -153,20 +152,22 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteBytes(d)
 	// --- [end][write][reference](time.Time) ---
 
-	buff.WriteFloat64(target.CPUCoreHours)    // write float64
-	buff.WriteFloat64(target.CPUCost)         // write float64
-	buff.WriteFloat64(target.CPUEfficiency)   // write float64
-	buff.WriteFloat64(target.GPUHours)        // write float64
-	buff.WriteFloat64(target.GPUCost)         // write float64
-	buff.WriteFloat64(target.NetworkCost)     // write float64
-	buff.WriteFloat64(target.PVByteHours)     // write float64
-	buff.WriteFloat64(target.PVCost)          // write float64
-	buff.WriteFloat64(target.RAMByteHours)    // write float64
-	buff.WriteFloat64(target.RAMCost)         // write float64
-	buff.WriteFloat64(target.RAMEfficiency)   // write float64
-	buff.WriteFloat64(target.SharedCost)      // write float64
-	buff.WriteFloat64(target.TotalCost)       // write float64
-	buff.WriteFloat64(target.TotalEfficiency) // write float64
+	buff.WriteFloat64(target.CPUCoreHours)           // write float64
+	buff.WriteFloat64(target.CPUCoreRequestAverage)  // write float64
+	buff.WriteFloat64(target.CPUCoreUsageAverage)    // write float64
+	buff.WriteFloat64(target.CPUCost)                // write float64
+	buff.WriteFloat64(target.GPUHours)               // write float64
+	buff.WriteFloat64(target.GPUCost)                // write float64
+	buff.WriteFloat64(target.NetworkCost)            // write float64
+	buff.WriteFloat64(target.PVByteHours)            // write float64
+	buff.WriteFloat64(target.PVCost)                 // write float64
+	buff.WriteFloat64(target.RAMByteHours)           // write float64
+	buff.WriteFloat64(target.RAMBytesRequestAverage) // write float64
+	buff.WriteFloat64(target.RAMBytesUsageAverage)   // write float64
+	buff.WriteFloat64(target.RAMCost)                // write float64
+	buff.WriteFloat64(target.SharedCost)             // write float64
+	buff.WriteFloat64(target.ExternalCost)           // write float64
+	buff.WriteFloat64(target.TotalCost)              // write float64
 	return buff.Bytes(), nil
 }
 
@@ -208,16 +209,16 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	target.Properties = *b
 	// --- [end][read][reference](Properties) ---
 
-	// --- [begin][read][reference](time.Time) ---
-	e := &time.Time{}
+	// --- [begin][read][struct](Window) ---
+	e := &Window{}
 	f := buff.ReadInt()    // byte array length
 	g := buff.ReadBytes(f) // byte array
 	errB := e.UnmarshalBinary(g)
 	if errB != nil {
 		return errB
 	}
-	target.Start = *e
-	// --- [end][read][reference](time.Time) ---
+	target.Window = *e
+	// --- [end][read][struct](Window) ---
 
 	// --- [begin][read][reference](time.Time) ---
 	h := &time.Time{}
@@ -227,31 +228,31 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	if errC != nil {
 		return errC
 	}
-	target.End = *h
+	target.Start = *h
 	// --- [end][read][reference](time.Time) ---
 
-	n := buff.ReadFloat64() // read float64
-	target.Minutes = n
-
 	// --- [begin][read][reference](time.Time) ---
-	o := &time.Time{}
-	p := buff.ReadInt()    // byte array length
-	q := buff.ReadBytes(p) // byte array
-	errD := o.UnmarshalBinary(q)
+	n := &time.Time{}
+	o := buff.ReadInt()    // byte array length
+	p := buff.ReadBytes(o) // byte array
+	errD := n.UnmarshalBinary(p)
 	if errD != nil {
 		return errD
 	}
-	target.ActiveStart = *o
+	target.End = *n
 	// --- [end][read][reference](time.Time) ---
 
+	q := buff.ReadFloat64() // read float64
+	target.CPUCoreHours = q
+
 	r := buff.ReadFloat64() // read float64
-	target.CPUCoreHours = r
+	target.CPUCoreRequestAverage = r
 
 	s := buff.ReadFloat64() // read float64
-	target.CPUCost = s
+	target.CPUCoreUsageAverage = s
 
 	t := buff.ReadFloat64() // read float64
-	target.CPUEfficiency = t
+	target.CPUCost = t
 
 	u := buff.ReadFloat64() // read float64
 	target.GPUHours = u
@@ -272,19 +273,22 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	target.RAMByteHours = aa
 
 	bb := buff.ReadFloat64() // read float64
-	target.RAMCost = bb
+	target.RAMBytesRequestAverage = bb
 
 	cc := buff.ReadFloat64() // read float64
-	target.RAMEfficiency = cc
+	target.RAMBytesUsageAverage = cc
 
 	dd := buff.ReadFloat64() // read float64
-	target.SharedCost = dd
+	target.RAMCost = dd
 
 	ee := buff.ReadFloat64() // read float64
-	target.TotalCost = ee
+	target.SharedCost = ee
 
 	ff := buff.ReadFloat64() // read float64
-	target.TotalEfficiency = ff
+	target.ExternalCost = ff
+
+	gg := buff.ReadFloat64() // read float64
+	target.TotalCost = gg
 
 	return nil
 }
@@ -340,19 +344,33 @@ func (target *AllocationSet) MarshalBinary() (data []byte, err error) {
 		// --- [end][write][map](map[string]*Allocation) ---
 
 	}
-	if target.idleKeys == nil {
+	if target.externalKeys == nil {
 		buff.WriteUInt8(uint8(0)) // write nil byte
 	} else {
 		buff.WriteUInt8(uint8(1)) // write non-nil byte
 
 		// --- [begin][write][map](map[string]bool) ---
-		buff.WriteInt(len(target.idleKeys)) // map length
-		for kk, vv := range target.idleKeys {
+		buff.WriteInt(len(target.externalKeys)) // map length
+		for kk, vv := range target.externalKeys {
 			buff.WriteString(kk) // write string
 			buff.WriteBool(vv)   // write bool
 		}
 		// --- [end][write][map](map[string]bool) ---
 
+	}
+	if target.idleKeys == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		// --- [begin][write][map](map[string]bool) ---
+		buff.WriteInt(len(target.idleKeys)) // map length
+		for kkk, vvv := range target.idleKeys {
+			buff.WriteString(kkk) // write string
+			buff.WriteBool(vvv)   // write bool
+		}
+		// --- [end][write][map](map[string]bool) ---
+
 	}
 	// --- [begin][write][struct](Window) ---
 	b, errB := target.Window.MarshalBinary()
@@ -450,7 +468,7 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 
 	}
 	if buff.ReadUInt8() == uint8(0) {
-		target.idleKeys = nil
+		target.externalKeys = nil
 	} else {
 		// --- [begin][read][map](map[string]bool) ---
 		g := make(map[string]bool)
@@ -466,35 +484,56 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 
 			g[kk] = vv
 		}
-		target.idleKeys = g
+		target.externalKeys = g
+		// --- [end][read][map](map[string]bool) ---
+
+	}
+	if buff.ReadUInt8() == uint8(0) {
+		target.idleKeys = nil
+	} else {
+		// --- [begin][read][map](map[string]bool) ---
+		n := make(map[string]bool)
+		o := buff.ReadInt() // map len
+		for ii := 0; ii < o; ii++ {
+			var kkk string
+			p := buff.ReadString() // read string
+			kkk = p
+
+			var vvv bool
+			q := buff.ReadBool() // read bool
+			vvv = q
+
+			n[kkk] = vvv
+		}
+		target.idleKeys = n
 		// --- [end][read][map](map[string]bool) ---
 
 	}
 	// --- [begin][read][struct](Window) ---
-	n := &Window{}
-	o := buff.ReadInt()    // byte array length
-	p := buff.ReadBytes(o) // byte array
-	errB := n.UnmarshalBinary(p)
+	r := &Window{}
+	s := buff.ReadInt()    // byte array length
+	t := buff.ReadBytes(s) // byte array
+	errB := r.UnmarshalBinary(t)
 	if errB != nil {
 		return errB
 	}
-	target.Window = *n
+	target.Window = *r
 	// --- [end][read][struct](Window) ---
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.Warnings = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		r := buff.ReadInt() // array len
-		q := make([]string, r)
-		for ii := 0; ii < r; ii++ {
-			var s string
-			t := buff.ReadString() // read string
-			s = t
+		w := buff.ReadInt() // array len
+		u := make([]string, w)
+		for jj := 0; jj < w; jj++ {
+			var x string
+			y := buff.ReadString() // read string
+			x = y
 
-			q[ii] = s
+			u[jj] = x
 		}
-		target.Warnings = q
+		target.Warnings = u
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -502,16 +541,16 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 		target.Errors = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		w := buff.ReadInt() // array len
-		u := make([]string, w)
-		for jj := 0; jj < w; jj++ {
-			var x string
-			y := buff.ReadString() // read string
-			x = y
+		aa := buff.ReadInt() // array len
+		z := make([]string, aa)
+		for iii := 0; iii < aa; iii++ {
+			var bb string
+			cc := buff.ReadString() // read string
+			bb = cc
 
-			u[jj] = x
+			z[iii] = bb
 		}
-		target.Errors = u
+		target.Errors = z
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -1406,6 +1445,7 @@ func (target *Cloud) MarshalBinary() (data []byte, err error) {
 
 	buff.WriteFloat64(target.adjustment) // write float64
 	buff.WriteFloat64(target.Cost)       // write float64
+	buff.WriteFloat64(target.Credit)     // write float64
 	return buff.Bytes(), nil
 }
 
@@ -1513,6 +1553,9 @@ func (target *Cloud) UnmarshalBinary(data []byte) (err error) {
 	w := buff.ReadFloat64() // read float64
 	target.Cost = w
 
+	x := buff.ReadFloat64() // read float64
+	target.Credit = x
+
 	return nil
 }
 

pkg/kubecost/properties.go

@@ -5,7 +5,7 @@ import (
 	"sort"
 	"strings"
 
-	util "github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util"
 )
 
 type Property string
@@ -57,6 +57,7 @@ type PropertyValue struct {
 }
 
 // Properties describes a set of Kubernetes objects.
+// TODO make this a struct smdh [TODO:CLEANUP]
 type Properties map[Property]interface{}
 
 // TODO niko/etl make sure Services deep copy works correctly

pkg/kubecost/window.go

@@ -8,6 +8,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/thanos"
 	"github.com/kubecost/cost-model/pkg/util"
 )
 
@@ -393,7 +395,19 @@ func (w Window) ExpandEnd(end time.Time) Window {
 }
 
 func (w Window) Expand(that Window) Window {
-	return w.ExpandStart(*that.start).ExpandEnd(*that.end)
+	if that.start == nil {
+		w.start = nil
+	} else {
+		w = w.ExpandStart(*that.start)
+	}
+
+	if that.end == nil {
+		w.end = nil
+	} else {
+		w = w.ExpandEnd(*that.end)
+	}
+
+	return w
 }
 
 func (w Window) Hours() float64 {
@@ -432,6 +446,86 @@ func (w Window) Minutes() float64 {
 	return w.end.Sub(*w.start).Minutes()
 }
 
+// Overlaps returns true iff the two given Windows share and amount of temporal
+// coverage.
+// TODO complete (with unit tests!) and then implement in AllocationSet.accumulate
+// [TODO:CLEANUP]
+func (w Window) Overlaps(x Window) bool {
+	if (w.start == nil && w.end == nil) || (x.start == nil && x.end == nil) {
+		// one window is completely open, so overlap is guaranteed
+		// <---------->
+		//   ?------?
+		return true
+	}
+
+	// Neither window is completely open (nil, nil), but one or the other might
+	// still be future- or past-open.
+
+	if w.start == nil {
+		// w is past-open, future-closed
+		// <------]
+
+		if x.start != nil && !x.start.Before(*w.end) {
+			// x starts after w ends (or eq)
+			// <------]
+			//          [------?
+			return false
+		}
+
+		// <-----]
+		//    ?-----?
+		return true
+	}
+
+	if w.end == nil {
+		// w is future-open, past-closed
+		// [------>
+
+		if x.end != nil && !x.end.After(*w.end) {
+			// x ends before w begins (or eq)
+			//          [------>
+			// ?------]
+			return false
+		}
+
+		//    [------>
+		// ?------?
+		return true
+	}
+
+	// Now we know w is closed, but we don't know about x
+	//  [------]
+	//     ?------?
+	if x.start == nil {
+		// TODO
+	}
+
+	if x.end == nil {
+		// TODO
+	}
+
+	// Both are closed.
+
+	if !x.start.Before(*w.end) && !x.end.Before(*w.end) {
+		// x starts and ends after w ends
+		// [------]
+		//          [------]
+		return false
+	}
+
+	if !x.start.After(*w.start) && !x.end.After(*w.start) {
+		// x starts and ends before w starts
+		//          [------]
+		// [------]
+		return false
+	}
+
+	// w and x must overlap
+	//    [------]
+	// [------]
+	return true
+}
+
 func (w Window) Set(start, end *time.Time) {
 	w.start = start
 	w.end = end
@@ -482,6 +576,47 @@ func (w Window) DurationOffset() (time.Duration, time.Duration, error) {
 	return duration, offset, nil
 }
 
+// DurationOffsetForPrometheus returns durations representing the duration and
+// offset of the given window, factoring in the Thanos offset if necessary. The
+// duration is returned as
+func (w Window) DurationOffsetForPrometheus() (string, string, error) {
+	duration, offset, err := w.DurationOffset()
+	if err != nil {
+		return "", "", err
+	}
+
+	// If using Thanos, increase offset to 3 hours, reducing the duration by
+	// equal measure to maintain the same starting point.
+	thanosDur := thanos.OffsetDuration()
+	if offset < thanosDur && env.IsThanosEnabled() {
+		diff := thanosDur - offset
+		offset += diff
+		duration -= diff
+	}
+
+	// If duration < 0, return an error
+	if duration < 0 {
+		return "", "", fmt.Errorf("negative duration: %s", duration)
+	}
+
+	// Negative offset means that the end time is in the future. Prometheus
+	// fails for non-positive offset values, so shrink the duration and
+	// remove the offset altogether.
+	if offset < 0 {
+		duration = duration + offset
+		offset = 0
+	}
+
+	durStr, offStr := util.DurationOffsetStrings(duration, offset)
+	if offset < time.Minute {
+		offStr = ""
+	} else {
+		offStr = " offset " + offStr
+	}
+
+	return durStr, offStr, nil
+}
+
 // DurationOffsetStrings returns formatted, Prometheus-compatible strings representing
 // the duration and offset of the window in terms of days, hours, minutes, or seconds;
 // e.g. ("7d", "1441m", "30m", "1s", "")

pkg/kubecost/window_test.go

@@ -2,8 +2,11 @@ package kubecost
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 	"time"
+
+	"github.com/kubecost/cost-model/pkg/env"
 )
 
 func TestRoundBack(t *testing.T) {
@@ -211,7 +214,7 @@ func TestParseWindowUTC(t *testing.T) {
 		t.Fatalf(`expect: window "month" to end before now; actual: %s ends after %s`, month, time.Now().UTC())
 	}
 
-	// TODO niko/etl lastweek
+	// TODO lastweek
 
 	lastmonth, err := ParseWindowUTC("lastmonth")
 	monthMinHours := float64(24 * 28)
@@ -542,30 +545,6 @@ func TestParseWindowWithOffsetString(t *testing.T) {
 
 }
 
-// TODO niko/etl
-// func TestWindow_Contains(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Duration(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_End(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Equal(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_ExpandStart(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_ExpandEnd(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Start(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_String(t *testing.T) {}
-
 func TestWindow_DurationOffsetStrings(t *testing.T) {
 	w, err := ParseWindowUTC("1d")
 	if err != nil {
@@ -624,3 +603,171 @@ func TestWindow_DurationOffsetStrings(t *testing.T) {
 		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
 	}
 }
+
+func TestWindow_DurationOffsetForPrometheus(t *testing.T) {
+	// Set-up and tear-down
+	thanosEnabled := env.GetBool(env.ThanosEnabledEnvVar, false)
+	defer env.SetBool(env.ThanosEnabledEnvVar, thanosEnabled)
+
+	// Test for Prometheus (env.IsThanosEnabled() == false)
+	env.SetBool(env.ThanosEnabledEnvVar, false)
+	if env.IsThanosEnabled() {
+		t.Fatalf("expected env.IsThanosEnabled() == false")
+	}
+
+	w, err := ParseWindowUTC("1d")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1d": %s`, err)
+	}
+	dur, off, err := w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1d" {
+		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
+	}
+	if off != "" {
+		t.Fatalf(`expect: offset to be ""; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("2h")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "2h": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "2h" {
+		t.Fatalf(`expect: window to be "2h"; actual: "%s"`, dur)
+	}
+	if off != "" {
+		t.Fatalf(`expect: offset to be ""; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("10m")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "10m": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "10m" {
+		t.Fatalf(`expect: window to be "10m"; actual: "%s"`, dur)
+	}
+	if off != "" {
+		t.Fatalf(`expect: offset to be ""; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("1589448338,1589534798")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1589448338,1589534798": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1441m" {
+		t.Fatalf(`expect: window to be "1441m"; actual: "%s"`, dur)
+	}
+	if !strings.HasPrefix(off, " offset ") {
+		t.Fatalf(`expect: offset to start with " offset "; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("yesterday")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "yesterday": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1d" {
+		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
+	}
+	if !strings.HasPrefix(off, " offset ") {
+		t.Fatalf(`expect: offset to start with " offset "; actual: "%s"`, off)
+	}
+
+	// Test for Thanos (env.IsThanosEnabled() == true)
+	env.SetBool(env.ThanosEnabledEnvVar, true)
+	if !env.IsThanosEnabled() {
+		t.Fatalf("expected env.IsThanosEnabled() == true")
+	}
+
+	w, err = ParseWindowUTC("1d")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1d": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "21h" {
+		t.Fatalf(`expect: window to be "21d"; actual: "%s"`, dur)
+	}
+	if off != " offset 3h" {
+		t.Fatalf(`expect: offset to be " offset 3h"; actual: "%s"`, off)
+	}
+
+	w, err = ParseWindowUTC("2h")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "2h": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err == nil {
+		t.Fatalf(`expected error (negative duration); got ("%s", "%s")`, dur, off)
+	}
+
+	w, err = ParseWindowUTC("10m")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1d": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err == nil {
+		t.Fatalf(`expected error (negative duration); got ("%s", "%s")`, dur, off)
+	}
+
+	w, err = ParseWindowUTC("1589448338,1589534798")
+	if err != nil {
+		t.Fatalf(`unexpected error parsing "1589448338,1589534798": %s`, err)
+	}
+	dur, off, err = w.DurationOffsetForPrometheus()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if dur != "1441m" {
+		t.Fatalf(`expect: window to be "1441m"; actual: "%s"`, dur)
+	}
+	if !strings.HasPrefix(off, " offset ") {
+		t.Fatalf(`expect: offset to start with " offset "; actual: "%s"`, off)
+	}
+}
+
+// TODO
+// func TestWindow_Overlaps(t *testing.T) {}
+
+// TODO
+// func TestWindow_Contains(t *testing.T) {}
+
+// TODO
+// func TestWindow_Duration(t *testing.T) {}
+
+// TODO
+// func TestWindow_End(t *testing.T) {}
+
+// TODO
+// func TestWindow_Equal(t *testing.T) {}
+
+// TODO
+// func TestWindow_ExpandStart(t *testing.T) {}
+
+// TODO
+// func TestWindow_ExpandEnd(t *testing.T) {}
+
+// TODO
+// func TestWindow_Start(t *testing.T) {}
+
+// TODO
+// func TestWindow_String(t *testing.T) {}

pkg/prom/result.go

@@ -230,6 +230,27 @@ func (qr *QueryResult) GetString(field string) (string, error) {
 	return strField, nil
 }
 
+// GetStrings returns the requested fields, or an error if it does not exist
+func (qr *QueryResult) GetStrings(fields ...string) (map[string]string, error) {
+	values := map[string]string{}
+
+	for _, field := range fields {
+		f, ok := qr.Metric[field]
+		if !ok {
+			return nil, fmt.Errorf("'%s' field does not exist in data result vector", field)
+		}
+
+		value, ok := f.(string)
+		if !ok {
+			return nil, fmt.Errorf("'%s' field is improperly formatted", field)
+		}
+
+		values[field] = value
+	}
+
+	return values, nil
+}
+
 // GetLabels returns all labels and their values from the query result
 func (qr *QueryResult) GetLabels() map[string]string {
 	result := make(map[string]string)

pkg/util/mapper/mapper.go

@@ -1,8 +1,10 @@
 package mapper
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
+	"time"
 )
 
 //--------------------------------------------------------------------------
@@ -85,6 +87,10 @@ type PrimitiveMapReader interface {
 	// is empty or fails to parse, the defaultValue parameter is returned.
 	GetBool(key string, defaultValue bool) bool
 
+	// GetDuration parses a time.Duration from the map key paramter. If the
+	// value is empty to fails to parse, the defaultValue is returned.
+	GetDuration(key string, defaultValue time.Duration) time.Duration
+
 	// GetList returns a string list which contains the value set by key split using the
 	// provided delimiter with each entry trimmed of space. If the value doesn't exist,
 	// nil is returned
@@ -130,6 +136,9 @@ type PrimitiveMapWriter interface {
 	// SetBool sets the map to a string formatted bool value.
 	SetBool(key string, value bool) error
 
+	// SetDuration sets the map to a string formatted time.Duration value
+	SetDuration(key string, duration time.Duration) error
+
 	// SetList sets the map's value at key to a string consistent of each value in the list separated
 	// by the provided delimiter.
 	SetList(key string, values []string, delimiter string) error
@@ -383,6 +392,19 @@ func (rom *readOnlyMapper) GetBool(key string, defaultValue bool) bool {
 	return b
 }
 
+// GetDuration parses a time.Duration from the read-only mapper key parameter.
+// If the value is empty or fails to parse, the defaultValue parameter is returned.
+func (rom *readOnlyMapper) GetDuration(key string, defaultValue time.Duration) time.Duration {
+	r := rom.getter.Get(key)
+
+	d, err := parseDuration(r)
+	if err != nil {
+		return defaultValue
+	}
+
+	return d
+}
+
 // GetList returns a string list which contains the value set by key split using the
 // provided delimiter with each entry trimmed of space. If the value doesn't exist,
 // nil is returned
@@ -464,8 +486,74 @@ func (wom *writeOnlyMapper) SetBool(key string, value bool) error {
 	return wom.setter.Set(key, strconv.FormatBool(value))
 }
 
+// SetDuration sets the map to a string formatted bool value.
+func (wom *writeOnlyMapper) SetDuration(key string, value time.Duration) error {
+	return wom.setter.Set(key, durationString(value))
+}
+
 // SetList sets the map's value at key to a string consistent of each value in the list separated
 // by the provided delimiter.
 func (wom *writeOnlyMapper) SetList(key string, values []string, delimiter string) error {
 	return wom.setter.Set(key, strings.Join(values, delimiter))
 }
+
+const (
+	secsPerMin  = 60.0
+	secsPerHour = 3600.0
+	secsPerDay  = 86400.0
+)
+
+func durationString(duration time.Duration) string {
+	durSecs := int64(duration.Seconds())
+
+	durStr := ""
+	if durSecs > 0 {
+		if durSecs%secsPerDay == 0 {
+			// convert to days
+			durStr = fmt.Sprintf("%dd", durSecs/secsPerDay)
+		} else if durSecs%secsPerHour == 0 {
+			// convert to hours
+			durStr = fmt.Sprintf("%dh", durSecs/secsPerHour)
+		} else if durSecs%secsPerMin == 0 {
+			// convert to mins
+			durStr = fmt.Sprintf("%dm", durSecs/secsPerMin)
+		} else if durSecs > 0 {
+			// default to mins, as long as duration is positive
+			durStr = fmt.Sprintf("%ds", durSecs)
+		}
+	}
+
+	return durStr
+}
+
+func parseDuration(duration string) (time.Duration, error) {
+	var amountStr string
+	var unit time.Duration
+	switch {
+	case strings.HasSuffix(duration, "s"):
+		unit = time.Second
+		amountStr = strings.TrimSuffix(duration, "s")
+	case strings.HasSuffix(duration, "m"):
+		unit = time.Minute
+		amountStr = strings.TrimSuffix(duration, "m")
+	case strings.HasSuffix(duration, "h"):
+		unit = time.Hour
+		amountStr = strings.TrimSuffix(duration, "h")
+	case strings.HasSuffix(duration, "d"):
+		unit = 24.0 * time.Hour
+		amountStr = strings.TrimSuffix(duration, "d")
+	default:
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
+
+	if len(amountStr) == 0 {
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
+
+	amount, err := strconv.ParseInt(amountStr, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
+
+	return time.Duration(amount) * unit, nil
+}

pkg/util/time.go

@@ -32,11 +32,10 @@ const (
 	DaysPerMonth = 30.42
 )
 
-// DurationOffsetStrings converts a (duration, offset) pair to Prometheus-
-// compatible strings in terms of days, hours, minutes, or seconds.
-func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
+// DurationString converts a duration to a Prometheus-compatible string in
+// terms of days, hours, minutes, or seconds.
+func DurationString(duration time.Duration) string {
 	durSecs := int64(duration.Seconds())
-	offSecs := int64(offset.Seconds())
 
 	durStr := ""
 	if durSecs > 0 {
@@ -55,23 +54,13 @@ func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
 		}
 	}
 
-	offStr := ""
-	if offSecs > 0 {
-		if offSecs%SecsPerDay == 0 {
-			// convert to days
-			offStr = fmt.Sprintf("%dd", offSecs/SecsPerDay)
-		} else if offSecs%SecsPerHour == 0 {
-			// convert to hours
-			offStr = fmt.Sprintf("%dh", offSecs/SecsPerHour)
-		} else if offSecs%SecsPerMin == 0 {
-			// convert to mins
-			offStr = fmt.Sprintf("%dm", offSecs/SecsPerMin)
-		} else if offSecs > 0 {
-			// default to mins, as long as offation is positive
-			offStr = fmt.Sprintf("%ds", offSecs)
-		}
-	}
-	return durStr, offStr
+	return durStr
+}
+
+// DurationOffsetStrings converts a (duration, offset) pair to Prometheus-
+// compatible strings in terms of days, hours, minutes, or seconds.
+func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
+	return DurationString(duration), DurationString(offset)
 }
 
 // ParseDuration converts a Prometheus-style duration string into a Duration

test/cloud_test.go

@@ -279,7 +279,7 @@ func TestNodePriceFromCSVWithBadConfig(t *testing.T) {
 	fm := FakeClusterMap{}
 	d, _ := time.ParseDuration("1m")
 
-	model := costmodel.NewCostModel(fc, fm, d)
+	model := costmodel.NewCostModel(nil, nil, fc, fm, d)
 
 	_, err := model.GetNodeCost(c)
 	if err != nil {
@@ -333,7 +333,7 @@ func TestSourceMatchesFromCSV(t *testing.T) {
 	fm := FakeClusterMap{}
 	d, _ := time.ParseDuration("1m")
 
-	model := costmodel.NewCostModel(fc, fm, d)
+	model := costmodel.NewCostModel(nil, nil, fc, fm, d)
 
 	_, err = model.GetNodeCost(c)
 	if err != nil {