Merge pull request #798 from kubecost/develop

Merge develop into master

README.md

@@ -74,7 +74,7 @@ Resources are allocated based on the time-weighted maximum of resource Requests
 
 #### How do I set my AWS Spot bids for accurate allocation?
 
-Modify [spotCPU](https://github.com/kubecost/cost-model/blob/master/cloud/default.json#L5) and  [spotRAM](https://github.com/kubecost/cost-model/blob/master/cloud/default.json#L7) in default.json to the price of your bid. Allocation will use these bid prices, but it does not take into account what you are actually charged by AWS. Alternatively, you can provide an AWS key to allow access to the Spot data feed. This will provide accurate Spot prices. 
+Modify [spotCPU](https://github.com/kubecost/cost-model/blob/master/configs/default.json#L5) and  [spotRAM](https://github.com/kubecost/cost-model/blob/master/configs/default.json#L7) in default.json to the price of your bid. Allocation will use these bid prices, but it does not take into account what you are actually charged by AWS. Alternatively, you can provide an AWS key to allow access to the Spot data feed. This will provide accurate Spot prices. 
 
 #### Do I need a GCP billing API key?
 

pkg/costmodel/allocation.go

@@ -18,7 +18,7 @@ import (
 
 const (
 	queryFmtPods              = `avg(kube_pod_container_status_running{}) by (pod, namespace, cluster_id)[%s:%s]%s`
-	queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id, provider_id)`
 	queryFmtRAMRequests       = `avg(avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
 	queryFmtRAMUsageAvg       = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
 	queryFmtRAMUsageMax       = `max(max_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
@@ -36,9 +36,9 @@ const (
 	// https://prometheus.io/blog/2019/01/28/subquery-support/#examples
 	queryFmtCPUUsageMax           = `max(max_over_time(kubecost_savings_container_cpu_usage_seconds[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
 	queryFmtGPUsRequested         = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
-	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
-	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
-	queryFmtNodeCostPerGPUHr      = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
+	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type, provider_id)`
+	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, cluster_id, instance_type, provider_id)`
+	queryFmtNodeCostPerGPUHr      = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type, provider_id)`
 	queryFmtNodeIsSpot            = `avg_over_time(kubecost_node_is_spot[%s]%s)`
 	queryFmtPVCInfo               = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, cluster_id)[%s:%s]%s`
 	queryFmtPVBytes               = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, cluster_id)`
@@ -308,9 +308,9 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	// for converting resource allocation data to cumulative costs.
 	nodeMap := map[nodeKey]*NodePricing{}
 
-	applyNodeCostPerCPUHr(nodeMap, resNodeCostPerCPUHr)
-	applyNodeCostPerRAMGiBHr(nodeMap, resNodeCostPerRAMGiBHr)
-	applyNodeCostPerGPUHr(nodeMap, resNodeCostPerGPUHr)
+	applyNodeCostPerCPUHr(nodeMap, resNodeCostPerCPUHr, cm.Provider.ParseID)
+	applyNodeCostPerRAMGiBHr(nodeMap, resNodeCostPerRAMGiBHr, cm.Provider.ParseID)
+	applyNodeCostPerGPUHr(nodeMap, resNodeCostPerGPUHr, cm.Provider.ParseID)
 	applyNodeSpot(nodeMap, resNodeIsSpot)
 	applyNodeDiscount(nodeMap, cm)
 
@@ -357,10 +357,10 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 			nodeKey := newNodeKey(cluster, nodeName)
 
 			node := cm.getNodePricing(nodeMap, nodeKey)
+			alloc.Properties.ProviderID = node.ProviderID
 			alloc.CPUCost = alloc.CPUCoreHours * node.CostPerCPUHr
 			alloc.RAMCost = (alloc.RAMByteHours / 1024 / 1024 / 1024) * node.CostPerRAMGiBHr
 			alloc.GPUCost = alloc.GPUHours * node.CostPerGPUHr
-
 			if pvcs, ok := podPVCMap[podKey]; ok {
 				for _, pvc := range pvcs {
 					// Determine the (start, end) of the relationship between the
@@ -386,8 +386,18 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 
 					// Apply the size and cost of the PV to the allocation, each
 					// weighted by count (i.e. the number of containers in the pod)
-					alloc.PVByteHours += pvc.Bytes * hrs / count
-					alloc.PVCost += cost / count
+					// record the amount of total PVBytes Hours attributable to a given PV
+					if alloc.PVs == nil {
+						alloc.PVs = kubecost.PVAllocations{}
+					}
+					pvKey := kubecost.PVKey{
+						Cluster: pvc.Cluster,
+						Name:    pvc.Volume.Name,
+					}
+					alloc.PVs[pvKey] = &kubecost.PVAllocation{
+						ByteHours: pvc.Bytes * hrs / count,
+						Cost:      cost / count,
+					}
 				}
 			}
 
@@ -1282,7 +1292,8 @@ func applyControllersToPods(podMap map[podKey]*Pod, podControllerMap map[podKey]
 	}
 }
 
-func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr []*prom.QueryResult) {
+func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr []*prom.QueryResult,
+	providerIDParser func(string) string) {
 	for _, res := range resNodeCostPerCPUHr {
 		cluster, err := res.GetString("cluster_id")
 		if err != nil {
@@ -1301,11 +1312,18 @@ func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr
 			continue
 		}
 
+		providerID, err := res.GetString("provider_id")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node CPU cost query result missing field: %s", err)
+			continue
+		}
+
 		key := newNodeKey(cluster, node)
 		if _, ok := nodeMap[key]; !ok {
 			nodeMap[key] = &NodePricing{
-				Name:     node,
-				NodeType: instanceType,
+				Name:       node,
+				NodeType:   instanceType,
+				ProviderID: providerIDParser(providerID),
 			}
 		}
 
@@ -1313,7 +1331,8 @@ func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr
 	}
 }
 
-func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRAMGiBHr []*prom.QueryResult) {
+func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRAMGiBHr []*prom.QueryResult,
+	providerIDParser func(string) string) {
 	for _, res := range resNodeCostPerRAMGiBHr {
 		cluster, err := res.GetString("cluster_id")
 		if err != nil {
@@ -1332,11 +1351,18 @@ func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRA
 			continue
 		}
 
+		providerID, err := res.GetString("provider_id")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node RAM cost query result missing field: %s", err)
+			continue
+		}
+
 		key := newNodeKey(cluster, node)
 		if _, ok := nodeMap[key]; !ok {
 			nodeMap[key] = &NodePricing{
-				Name:     node,
-				NodeType: instanceType,
+				Name:       node,
+				NodeType:   instanceType,
+				ProviderID: providerIDParser(providerID),
 			}
 		}
 
@@ -1344,7 +1370,8 @@ func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRA
 	}
 }
 
-func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr []*prom.QueryResult) {
+func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr []*prom.QueryResult,
+	providerIDParser func(string) string) {
 	for _, res := range resNodeCostPerGPUHr {
 		cluster, err := res.GetString("cluster_id")
 		if err != nil {
@@ -1363,11 +1390,18 @@ func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr
 			continue
 		}
 
+		providerID, err := res.GetString("provider_id")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: Node GPU cost query result missing field: %s", err)
+			continue
+		}
+
 		key := newNodeKey(cluster, node)
 		if _, ok := nodeMap[key]; !ok {
 			nodeMap[key] = &NodePricing{
-				Name:     node,
-				NodeType: instanceType,
+				Name:       node,
+				NodeType:   instanceType,
+				ProviderID: providerIDParser(providerID),
 			}
 		}
 
@@ -1640,8 +1674,17 @@ func applyUnmountedPVs(window kubecost.Window, podMap map[podKey]*Pod, pvMap map
 		podMap[key].Allocations[container].Properties.Namespace = namespace
 		podMap[key].Allocations[container].Properties.Pod = pod
 		podMap[key].Allocations[container].Properties.Container = container
-		podMap[key].Allocations[container].PVByteHours = unmountedPVBytes[cluster] * window.Minutes() / 60.0
-		podMap[key].Allocations[container].PVCost = amount
+		pvKey := kubecost.PVKey{
+			Cluster: cluster,
+			Name:    kubecost.UnmountedSuffix,
+		}
+		unmountedBreakDown := kubecost.PVAllocations{
+			pvKey: {
+				ByteHours: unmountedPVBytes[cluster] * window.Minutes() / 60.0,
+				Cost:      amount,
+			},
+		}
+		podMap[key].Allocations[container].PVs = podMap[key].Allocations[container].PVs.Add(unmountedBreakDown)
 	}
 }
 
@@ -1683,8 +1726,18 @@ func applyUnmountedPVCs(window kubecost.Window, podMap map[podKey]*Pod, pvcMap m
 		podMap[podKey].Allocations[container].Properties.Namespace = namespace
 		podMap[podKey].Allocations[container].Properties.Pod = pod
 		podMap[podKey].Allocations[container].Properties.Container = container
-		podMap[podKey].Allocations[container].PVByteHours = unmountedPVCBytes[key] * window.Minutes() / 60.0
-		podMap[podKey].Allocations[container].PVCost = amount
+		pvKey := kubecost.PVKey{
+			Cluster: cluster,
+			Name:    kubecost.UnmountedSuffix,
+		}
+		unmountedBreakDown := kubecost.PVAllocations{
+			pvKey: {
+				ByteHours: unmountedPVCBytes[key] * window.Minutes() / 60.0,
+				Cost:      amount,
+			},
+		}
+		podMap[podKey].Allocations[container].PVs = podMap[podKey].Allocations[container].PVs.Add(unmountedBreakDown)
+
 	}
 }
 
@@ -1885,6 +1938,7 @@ func (cm *CostModel) getCustomNodePricing(spot bool) *NodePricing {
 type NodePricing struct {
 	Name            string
 	NodeType        string
+	ProviderID      string
 	Preemptible     bool
 	CostPerCPUHr    float64
 	CostPerRAMGiBHr float64

pkg/costmodel/costmodel.go

@@ -1057,7 +1057,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 
 			cpuToRAMRatio := defaultCPU / defaultRAM
 			if math.IsNaN(cpuToRAMRatio) {
-				klog.V(1).Infof("[Warning] cpuToRAMRatio[defaultCPU: %f / defaultRam: %f] is NaN. Setting to 0.", defaultCPU, defaultRAM)
+				klog.V(1).Infof("[Warning] cpuToRAMRatio[defaultCPU: %f / defaultRAM: %f] is NaN. Setting to 0.", defaultCPU, defaultRAM)
 				cpuToRAMRatio = 0
 			}
 
@@ -1137,7 +1137,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 
 			cpuToRAMRatio := defaultCPU / defaultRAM
 			if math.IsNaN(cpuToRAMRatio) {
-				klog.V(1).Infof("[Warning] cpuToRAMRatio[defaultCPU: %f / defaultRam: %f] is NaN. Setting to 0.", defaultCPU, defaultRAM)
+				klog.V(1).Infof("[Warning] cpuToRAMRatio[defaultCPU: %f / defaultRAM: %f] is NaN. Setting to 0.", defaultCPU, defaultRAM)
 				cpuToRAMRatio = 0
 			}
 

pkg/kubecost/allocation.go

@@ -47,31 +47,36 @@ const ShareNone = "__none__"
 // Allocation is a unit of resource allocation and cost for a given window
 // of time and for a given kubernetes construct with its associated set of
 // properties.
-// TODO:CLEANUP consider dropping name in favor of just AllocationProperties and an
+// TODO:CLEANUP consider dropping name in favor of just Allocation and an
 // Assets-style key() function for AllocationSet.
 type Allocation struct {
-	Name                   string                `json:"name"`
-	Properties             *AllocationProperties `json:"properties,omitempty"`
-	Window                 Window                `json:"window"`
-	Start                  time.Time             `json:"start"`
-	End                    time.Time             `json:"end"`
-	CPUCoreHours           float64               `json:"cpuCoreHours"`
-	CPUCoreRequestAverage  float64               `json:"cpuCoreRequestAverage"`
-	CPUCoreUsageAverage    float64               `json:"cpuCoreUsageAverage"`
-	CPUCost                float64               `json:"cpuCost"`
-	GPUHours               float64               `json:"gpuHours"`
-	GPUCost                float64               `json:"gpuCost"`
-	NetworkCost            float64               `json:"networkCost"`
-	LoadBalancerCost       float64               `json:"loadBalancerCost"`
-	PVByteHours            float64               `json:"pvByteHours"`
-	PVCost                 float64               `json:"pvCost"`
-	RAMByteHours           float64               `json:"ramByteHours"`
-	RAMBytesRequestAverage float64               `json:"ramByteRequestAverage"`
-	RAMBytesUsageAverage   float64               `json:"ramByteUsageAverage"`
-	RAMCost                float64               `json:"ramCost"`
-	SharedCost             float64               `json:"sharedCost"`
-	ExternalCost           float64               `json:"externalCost"`
-
+	Name                       string                `json:"name"`
+	Properties                 *AllocationProperties `json:"properties,omitempty"`
+	Window                     Window                `json:"window"`
+	Start                      time.Time             `json:"start"`
+	End                        time.Time             `json:"end"`
+	CPUCoreHours               float64               `json:"cpuCoreHours"`
+	CPUCoreRequestAverage      float64               `json:"cpuCoreRequestAverage"`
+	CPUCoreUsageAverage        float64               `json:"cpuCoreUsageAverage"`
+	CPUCost                    float64               `json:"cpuCost"`
+	CPUCostAdjustment          float64               `json:"cpuCostAdjustment"`
+	GPUHours                   float64               `json:"gpuHours"`
+	GPUCost                    float64               `json:"gpuCost"`
+	GPUCostAdjustment          float64               `json:"gpuCostAdjustment"`
+	NetworkCost                float64               `json:"networkCost"`
+	NetworkCostAdjustment      float64               `json:"networkCostAdjustment"`
+	LoadBalancerCost           float64               `json:"loadBalancerCost"`
+	LoadBalancerCostAdjustment float64               `json:"loadBalancerCostAdjustment"`
+	PVs                        PVAllocations         `json:"-"`
+	PVCostAdjustment           float64               `json:"pvCostAdjustment"`
+	RAMByteHours               float64               `json:"ramByteHours"`
+	RAMBytesRequestAverage     float64               `json:"ramByteRequestAverage"`
+	RAMBytesUsageAverage       float64               `json:"ramByteUsageAverage"`
+	RAMCost                    float64               `json:"ramCost"`
+	RAMCostAdjustment          float64               `json:"ramCostAdjustment"`
+	SharedCost                 float64               `json:"sharedCost"`
+	SharedCostAdjustment       float64               `json:"sharedCostAdjustment"`
+	ExternalCost               float64               `json:"externalCost"`
 	// RawAllocationOnly is a pointer so if it is not present it will be
 	// marshalled as null rather than as an object with Go default values.
 	RawAllocationOnly *RawAllocationOnlyData `json:"rawAllocationOnly"`
@@ -105,6 +110,59 @@ type RawAllocationOnlyData struct {
 	RAMBytesUsageMax float64 `json:"ramByteUsageMax"`
 }
 
+// PVAllocations is a map of Disk Asset Identifiers to the
+// usage of them by an Allocation as recorded in a PVAllocation
+type PVAllocations map[PVKey]*PVAllocation
+
+// Clone creates a deep copy of a PVAllocations
+func (pv *PVAllocations) Clone() PVAllocations {
+	if pv == nil || *pv == nil {
+		return nil
+	}
+	apv := *pv
+	clonePV := PVAllocations{}
+	for k, v := range apv {
+		clonePV[k] = &PVAllocation{
+			ByteHours: v.ByteHours,
+			Cost:      v.Cost,
+		}
+	}
+	return clonePV
+}
+
+// Add adds contents of that to the calling PVAllocations
+func (pv *PVAllocations) Add(that PVAllocations) PVAllocations {
+	apv := pv.Clone()
+	if that != nil {
+		if apv == nil {
+			apv = PVAllocations{}
+		}
+		for pvKey, thatPVAlloc := range that {
+			apvAlloc, ok := apv[pvKey]
+			if !ok {
+				apvAlloc = &PVAllocation{}
+			}
+			apvAlloc.Cost += thatPVAlloc.Cost
+			apvAlloc.ByteHours += thatPVAlloc.ByteHours
+			apv[pvKey] = apvAlloc
+		}
+	}
+	return apv
+}
+
+// PVKey for identifying Disk type assets
+type PVKey struct {
+	Cluster string `json:"cluster"`
+	Name    string `json:"name"`
+}
+
+// PVAllocation contains the byte hour usage
+// and cost of an Allocation for a single PV
+type PVAllocation struct {
+	ByteHours float64 `json:"byteHours"`
+	Cost      float64 `json:"cost"`
+}
+
 // AllocationMatchFunc is a function that can be used to match Allocations by
 // returning true for any given Allocation if a condition is met.
 type AllocationMatchFunc func(*Allocation) bool
@@ -135,28 +193,34 @@ func (a *Allocation) Clone() *Allocation {
 	}
 
 	return &Allocation{
-		Name:                   a.Name,
-		Properties:             a.Properties.Clone(),
-		Window:                 a.Window.Clone(),
-		Start:                  a.Start,
-		End:                    a.End,
-		CPUCoreHours:           a.CPUCoreHours,
-		CPUCoreRequestAverage:  a.CPUCoreRequestAverage,
-		CPUCoreUsageAverage:    a.CPUCoreUsageAverage,
-		CPUCost:                a.CPUCost,
-		GPUHours:               a.GPUHours,
-		GPUCost:                a.GPUCost,
-		NetworkCost:            a.NetworkCost,
-		LoadBalancerCost:       a.LoadBalancerCost,
-		PVByteHours:            a.PVByteHours,
-		PVCost:                 a.PVCost,
-		RAMByteHours:           a.RAMByteHours,
-		RAMBytesRequestAverage: a.RAMBytesRequestAverage,
-		RAMBytesUsageAverage:   a.RAMBytesUsageAverage,
-		RAMCost:                a.RAMCost,
-		SharedCost:             a.SharedCost,
-		ExternalCost:           a.ExternalCost,
-		RawAllocationOnly:      a.RawAllocationOnly.Clone(),
+		Name:                       a.Name,
+		Properties:                 a.Properties.Clone(),
+		Window:                     a.Window.Clone(),
+		Start:                      a.Start,
+		End:                        a.End,
+		CPUCoreHours:               a.CPUCoreHours,
+		CPUCoreRequestAverage:      a.CPUCoreRequestAverage,
+		CPUCoreUsageAverage:        a.CPUCoreUsageAverage,
+		CPUCost:                    a.CPUCost,
+		CPUCostAdjustment:          a.CPUCostAdjustment,
+		GPUHours:                   a.GPUHours,
+		GPUCost:                    a.GPUCost,
+		GPUCostAdjustment:          a.GPUCostAdjustment,
+		NetworkCost:                a.NetworkCost,
+		NetworkCostAdjustment:      a.NetworkCostAdjustment,
+		LoadBalancerCost:           a.LoadBalancerCost,
+		LoadBalancerCostAdjustment: a.LoadBalancerCostAdjustment,
+		PVs:                        a.PVs.Clone(),
+		PVCostAdjustment:           a.PVCostAdjustment,
+		RAMByteHours:               a.RAMByteHours,
+		RAMBytesRequestAverage:     a.RAMBytesRequestAverage,
+		RAMBytesUsageAverage:       a.RAMBytesUsageAverage,
+		RAMCost:                    a.RAMCost,
+		RAMCostAdjustment:          a.RAMCostAdjustment,
+		SharedCost:                 a.SharedCost,
+		SharedCostAdjustment:       a.SharedCostAdjustment,
+		ExternalCost:               a.ExternalCost,
+		RawAllocationOnly:          a.RawAllocationOnly.Clone(),
 	}
 }
 
@@ -202,22 +266,31 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.CPUCost, that.CPUCost) {
 		return false
 	}
+	if !util.IsApproximately(a.CPUCostAdjustment, that.CPUCostAdjustment) {
+		return false
+	}
 	if !util.IsApproximately(a.GPUHours, that.GPUHours) {
 		return false
 	}
 	if !util.IsApproximately(a.GPUCost, that.GPUCost) {
 		return false
 	}
+	if !util.IsApproximately(a.GPUCostAdjustment, that.GPUCostAdjustment) {
+		return false
+	}
 	if !util.IsApproximately(a.NetworkCost, that.NetworkCost) {
 		return false
 	}
+	if !util.IsApproximately(a.NetworkCostAdjustment, that.NetworkCostAdjustment) {
+		return false
+	}
 	if !util.IsApproximately(a.LoadBalancerCost, that.LoadBalancerCost) {
 		return false
 	}
-	if !util.IsApproximately(a.PVByteHours, that.PVByteHours) {
+	if !util.IsApproximately(a.LoadBalancerCostAdjustment, that.LoadBalancerCostAdjustment) {
 		return false
 	}
-	if !util.IsApproximately(a.PVCost, that.PVCost) {
+	if !util.IsApproximately(a.PVCostAdjustment, that.PVCostAdjustment) {
 		return false
 	}
 	if !util.IsApproximately(a.RAMByteHours, that.RAMByteHours) {
@@ -226,9 +299,15 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.RAMCost, that.RAMCost) {
 		return false
 	}
+	if !util.IsApproximately(a.RAMCostAdjustment, that.RAMCostAdjustment) {
+		return false
+	}
 	if !util.IsApproximately(a.SharedCost, that.SharedCost) {
 		return false
 	}
+	if !util.IsApproximately(a.SharedCostAdjustment, that.SharedCostAdjustment) {
+		return false
+	}
 	if !util.IsApproximately(a.ExternalCost, that.ExternalCost) {
 		return false
 	}
@@ -249,12 +328,77 @@ func (a *Allocation) Equal(that *Allocation) bool {
 		}
 	}
 
+	aPVs := a.PVs
+	thatPVs := that.PVs
+	if len(aPVs) == len(thatPVs) {
+		for k, pv := range aPVs {
+			tv, ok := thatPVs[k]
+			if !ok || *tv != *pv {
+				return false
+			}
+		}
+	} else {
+		return false
+	}
 	return true
 }
 
-// TotalCost is the total cost of the Allocation
+// TotalCost is the total cost of the Allocation including adjustments
 func (a *Allocation) TotalCost() float64 {
-	return a.CPUCost + a.GPUCost + a.RAMCost + a.PVCost + a.NetworkCost + a.SharedCost + a.ExternalCost + a.LoadBalancerCost
+	return a.CPUTotalCost() + a.GPUTotalCost() + a.RAMTotalCost() + a.PVTotalCost() + a.NetworkTotalCost() + a.LBTotalCost() + a.SharedTotalCost() + a.ExternalCost
+}
+
+// CPUTotalCost calculates total CPU cost of Allocation including adjustment
+func (a *Allocation) CPUTotalCost() float64 {
+	return a.CPUCost + a.CPUCostAdjustment
+}
+
+// GPUTotalCost calculates total GPU cost of Allocation including adjustment
+func (a *Allocation) GPUTotalCost() float64 {
+	return a.GPUCost + a.GPUCostAdjustment
+}
+
+// RAMTotalCost calculates total RAM cost of Allocation including adjustment
+func (a *Allocation) RAMTotalCost() float64 {
+	return a.RAMCost + a.RAMCostAdjustment
+}
+
+// PVTotalCost calculates total PV cost of Allocation including adjustment
+func (a *Allocation) PVTotalCost() float64 {
+	return a.PVCost() + a.PVCostAdjustment
+}
+
+// NetworkTotalCost calculates total Network cost of Allocation including adjustment
+func (a *Allocation) NetworkTotalCost() float64 {
+	return a.NetworkCost + a.NetworkCostAdjustment
+}
+
+// LBTotalCost calculates total LB cost of Allocation including adjustment
+func (a *Allocation) LBTotalCost() float64 {
+	return a.LoadBalancerCost + a.LoadBalancerCostAdjustment
+}
+
+// SharedTotalCost calculates total shared cost of Allocation including adjustment
+func (a *Allocation) SharedTotalCost() float64 {
+	return a.SharedCost + a.SharedCostAdjustment
+}
+
+// PVCost calculate cumulative cost of all PVs that Allocation is attached to
+func (a *Allocation) PVCost() float64 {
+	cost := 0.0
+	for _, pv := range a.PVs {
+		cost += pv.Cost
+	}
+	return cost
+}
+
+// PVByteHours calculate cumulative ByteHours of all PVs that Allocation is attached to
+func (a *Allocation) PVByteHours() float64 {
+	byteHours := 0.0
+	for _, pv := range a.PVs {
+		byteHours += pv.ByteHours
+	}
+	return byteHours
 }
 
 // CPUEfficiency is the ratio of usage to request. If there is no request and
@@ -290,10 +434,10 @@ func (a *Allocation) RAMEfficiency() float64 {
 // TotalEfficiency is the cost-weighted average of CPU and RAM efficiency. If
 // there is no cost at all, then efficiency is zero.
 func (a *Allocation) TotalEfficiency() float64 {
-	if a.CPUCost+a.RAMCost > 0 {
-		ramCostEff := a.RAMEfficiency() * a.RAMCost
-		cpuCostEff := a.CPUEfficiency() * a.CPUCost
-		return (ramCostEff + cpuCostEff) / (a.CPUCost + a.RAMCost)
+	if a.RAMTotalCost()+a.CPUTotalCost() > 0 {
+		ramCostEff := a.RAMEfficiency() * a.RAMTotalCost()
+		cpuCostEff := a.CPUEfficiency() * a.CPUTotalCost()
+		return (ramCostEff + cpuCostEff) / (a.CPUTotalCost() + a.RAMTotalCost())
 	}
 
 	return 0.0
@@ -315,12 +459,20 @@ func (a *Allocation) RAMBytes() float64 {
 	return a.RAMByteHours / (a.Minutes() / 60.0)
 }
 
+// GPUs converts the Allocation's GPUHours into average GPUs
+func (a *Allocation) GPUs() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.GPUHours / (a.Minutes() / 60.0)
+}
+
 // PVBytes converts the Allocation's PVByteHours into average PVBytes
 func (a *Allocation) PVBytes() float64 {
 	if a.Minutes() <= 0.0 {
 		return 0.0
 	}
-	return a.PVByteHours / (a.Minutes() / 60.0)
+	return a.PVByteHours() / (a.Minutes() / 60.0)
 }
 
 // MarshalJSON implements json.Marshaler interface
@@ -337,21 +489,30 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "cpuCoreUsageAverage", a.CPUCoreUsageAverage, ",")
 	jsonEncodeFloat64(buffer, "cpuCoreHours", a.CPUCoreHours, ",")
 	jsonEncodeFloat64(buffer, "cpuCost", a.CPUCost, ",")
+	jsonEncodeFloat64(buffer, "cpuCostAdjustment", a.CPUCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "cpuEfficiency", a.CPUEfficiency(), ",")
+	jsonEncodeFloat64(buffer, "gpuCount", a.GPUs(), ",")
 	jsonEncodeFloat64(buffer, "gpuHours", a.GPUHours, ",")
 	jsonEncodeFloat64(buffer, "gpuCost", a.GPUCost, ",")
+	jsonEncodeFloat64(buffer, "gpuCostAdjustment", a.GPUCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "networkCost", a.NetworkCost, ",")
+	jsonEncodeFloat64(buffer, "networkCostAdjustment", a.NetworkCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "loadBalancerCost", a.LoadBalancerCost, ",")
+	jsonEncodeFloat64(buffer, "loadBalancerCostAdjustment", a.LoadBalancerCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "pvBytes", a.PVBytes(), ",")
-	jsonEncodeFloat64(buffer, "pvByteHours", a.PVByteHours, ",")
-	jsonEncodeFloat64(buffer, "pvCost", a.PVCost, ",")
+	jsonEncodeFloat64(buffer, "pvByteHours", a.PVByteHours(), ",")
+	jsonEncodeFloat64(buffer, "pvCost", a.PVCost(), ",")
+	jsonEncode(buffer, "pvs", a.PVs, ",") // Todo Sean: this does not work properly
+	jsonEncodeFloat64(buffer, "pvCostAdjustment", a.PVCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "ramBytes", a.RAMBytes(), ",")
 	jsonEncodeFloat64(buffer, "ramByteRequestAverage", a.RAMBytesRequestAverage, ",")
 	jsonEncodeFloat64(buffer, "ramByteUsageAverage", a.RAMBytesUsageAverage, ",")
 	jsonEncodeFloat64(buffer, "ramByteHours", a.RAMByteHours, ",")
 	jsonEncodeFloat64(buffer, "ramCost", a.RAMCost, ",")
+	jsonEncodeFloat64(buffer, "ramCostAdjustment", a.RAMCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "ramEfficiency", a.RAMEfficiency(), ",")
 	jsonEncodeFloat64(buffer, "sharedCost", a.SharedCost, ",")
+	jsonEncodeFloat64(buffer, "sharedCostAdjustment", a.SharedCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "externalCost", a.ExternalCost, ",")
 	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), ",")
 	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), ",")
@@ -467,18 +628,28 @@ func (a *Allocation) add(that *Allocation) {
 	a.CPUCoreHours += that.CPUCoreHours
 	a.GPUHours += that.GPUHours
 	a.RAMByteHours += that.RAMByteHours
-	a.PVByteHours += that.PVByteHours
 
 	// Sum all cumulative cost fields
 	a.CPUCost += that.CPUCost
 	a.GPUCost += that.GPUCost
 	a.RAMCost += that.RAMCost
-	a.PVCost += that.PVCost
 	a.NetworkCost += that.NetworkCost
 	a.LoadBalancerCost += that.LoadBalancerCost
 	a.SharedCost += that.SharedCost
 	a.ExternalCost += that.ExternalCost
 
+	// Sum PVAllocations
+	a.PVs = a.PVs.Add(that.PVs)
+
+	// Sum all cumulative adjustment fields
+	a.CPUCostAdjustment += that.CPUCostAdjustment
+	a.RAMCostAdjustment += that.RAMCostAdjustment
+	a.GPUCostAdjustment += that.GPUCostAdjustment
+	a.PVCostAdjustment += that.PVCostAdjustment
+	a.NetworkCostAdjustment += that.NetworkCostAdjustment
+	a.LoadBalancerCostAdjustment += that.LoadBalancerCostAdjustment
+	a.SharedCostAdjustment += that.SharedCostAdjustment
+
 	// Any data that is in a "raw allocation only" is not valid in any
 	// sort of cumulative Allocation (like one that is added).
 	a.RawAllocationOnly = nil
@@ -1095,13 +1266,13 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 				totals[clusterID][r] += 1.0
 			}
 		} else {
-			coeffs[clusterID][name]["cpu"] += alloc.CPUCost
-			coeffs[clusterID][name]["gpu"] += alloc.GPUCost
-			coeffs[clusterID][name]["ram"] += alloc.RAMCost
+			coeffs[clusterID][name]["cpu"] += alloc.CPUTotalCost()
+			coeffs[clusterID][name]["gpu"] += alloc.GPUTotalCost()
+			coeffs[clusterID][name]["ram"] += alloc.RAMTotalCost()
 
-			totals[clusterID]["cpu"] += alloc.CPUCost
-			totals[clusterID]["gpu"] += alloc.GPUCost
-			totals[clusterID]["ram"] += alloc.RAMCost
+			totals[clusterID]["cpu"] += alloc.CPUTotalCost()
+			totals[clusterID]["gpu"] += alloc.GPUTotalCost()
+			totals[clusterID]["ram"] += alloc.RAMTotalCost()
 		}
 	}
 
@@ -1142,13 +1313,13 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 				totals[clusterID][r] += 1.0
 			}
 		} else {
-			coeffs[clusterID][name]["cpu"] += alloc.CPUCost
-			coeffs[clusterID][name]["gpu"] += alloc.GPUCost
-			coeffs[clusterID][name]["ram"] += alloc.RAMCost
+			coeffs[clusterID][name]["cpu"] += alloc.CPUTotalCost()
+			coeffs[clusterID][name]["gpu"] += alloc.GPUTotalCost()
+			coeffs[clusterID][name]["ram"] += alloc.RAMTotalCost()
 
-			totals[clusterID]["cpu"] += alloc.CPUCost
-			totals[clusterID]["gpu"] += alloc.GPUCost
-			totals[clusterID]["ram"] += alloc.RAMCost
+			totals[clusterID]["cpu"] += alloc.CPUTotalCost()
+			totals[clusterID]["gpu"] += alloc.GPUTotalCost()
+			totals[clusterID]["ram"] += alloc.RAMTotalCost()
 		}
 	}
 
@@ -1418,9 +1589,9 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 			clusterEnds[cluster] = a.End
 		}
 
-		assetClusterResourceCosts[cluster]["cpu"] -= a.CPUCost
-		assetClusterResourceCosts[cluster]["gpu"] -= a.GPUCost
-		assetClusterResourceCosts[cluster]["ram"] -= a.RAMCost
+		assetClusterResourceCosts[cluster]["cpu"] -= a.CPUTotalCost()
+		assetClusterResourceCosts[cluster]["gpu"] -= a.GPUTotalCost()
+		assetClusterResourceCosts[cluster]["ram"] -= a.RAMTotalCost()
 	})
 
 	// Turn remaining un-allocated asset costs into idle allocations
@@ -1460,6 +1631,137 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 	return idleAllocs, nil
 }
 
+// Reconcile calculate the exact cost of Allocation by resource(cpu, ram, gpu etc) based on Asset(s) on which
+// the Allocation depends.
+func (as *AllocationSet) Reconcile(assetSet *AssetSet) error {
+	if as == nil {
+		return fmt.Errorf("cannot reconcile allocation for nil AllocationSet")
+	}
+
+	if assetSet == nil {
+		return fmt.Errorf("cannot reconcile allocation with nil AssetSet")
+	}
+
+	if !as.Window.Equal(assetSet.Window) {
+		return fmt.Errorf("cannot reconcile allocation for sets with mismatched windows: %s != %s", as.Window, assetSet.Window)
+	}
+
+	// Build map of Assets with type Node by their ProviderId so that they can be matched to Allocations to determine
+	// proper CPU GPU and RAM prices
+	nodeByProviderID := map[string]*Node{}
+	diskByName := map[string]*Disk{}
+	assetSet.Each(func(key string, a Asset) {
+		if node, ok := a.(*Node); ok && node.properties.ProviderID != "" {
+			nodeByProviderID[node.properties.ProviderID] = node
+		}
+		if disk, ok := a.(*Disk); ok {
+			diskByName[disk.properties.Name] = disk
+		}
+	})
+
+	// Match Assets against allocations and adjust allocation cost based on the proportion of the asset that they used
+	as.Each(func(name string, a *Allocation) {
+		a.reconcileNodes(nodeByProviderID)
+		a.reconcileDisks(diskByName)
+	})
+
+	return nil
+}
+
+func (a *Allocation) reconcileNodes(nodeByProviderID map[string]*Node) {
+	providerId := a.Properties.ProviderID
+
+	// Reconcile with node Assets
+	node, ok := nodeByProviderID[providerId]
+	if !ok {
+		// Failed to find node for allocation
+		return
+	}
+
+	// adjustmentRate is used to scale resource costs proportionally
+	// by the adjustment. This is necessary because we only get one
+	// adjustment per Node, not one per-resource-per-Node.
+	//
+	// e.g. total cost = $90, adjustment = -$10 => 0.9
+	// e.g. total cost = $150, adjustment = -$300 => 0.3333
+	// e.g. total cost = $150, adjustment = $50 => 1.5
+	adjustmentRate := 1.0
+	if node.TotalCost()-node.Adjustment() == 0 {
+		// If (totalCost - adjustment) is 0.0 then adjustment cancels
+		// the entire node cost and we should make everything 0
+		// without dividing by 0.
+		adjustmentRate = 0.0
+	} else if node.Adjustment() != 0.0 {
+		// adjustmentRate is the ratio of cost-with-adjustment (i.e. TotalCost)
+		// to cost-without-adjustment (i.e. TotalCost - Adjustment).
+		adjustmentRate = node.TotalCost() / (node.TotalCost() - node.Adjustment())
+	}
+
+	// Find total cost of each node resource for the window
+	cpuCost := node.CPUCost * (1.0 - node.Discount) * adjustmentRate
+	ramCost := node.RAMCost * (1.0 - node.Discount) * adjustmentRate
+	gpuCost := node.GPUCost * adjustmentRate
+
+	// Find the proportion of resource hours used by the allocation, checking for 0 denominators
+	cpuUsageProportion := 0.0
+	if node.CPUCoreHours != 0 {
+		cpuUsageProportion = a.CPUCoreHours / node.CPUCoreHours
+	} else {
+		log.Warningf("Missing CPU Hours for node Provider ID: %s", providerId)
+	}
+	ramUsageProportion := 0.0
+	if node.RAMByteHours != 0 {
+		ramUsageProportion = a.RAMByteHours / node.RAMByteHours
+	} else {
+		log.Warningf("Missing Ram Byte Hours for node Provider ID: %s", providerId)
+	}
+	gpuUsageProportion := 0.0
+	if node.GPUHours != 0 {
+		gpuUsageProportion = a.GPUHours / node.GPUHours
+	}
+	// No log for GPU because not all nodes have GPU
+
+	// Calculate the allocation's resource costs by the proportion of resources used and total costs
+	allocCPUCost := cpuUsageProportion * cpuCost
+	allocRAMCost := ramUsageProportion * ramCost
+	allocGPUCost := gpuUsageProportion * gpuCost
+
+	a.CPUCostAdjustment = allocCPUCost - a.CPUCost
+	a.RAMCostAdjustment = allocRAMCost - a.RAMCost
+	a.GPUCostAdjustment = allocGPUCost - a.GPUCost
+}
+
+func (a *Allocation) reconcileDisks(diskByName map[string]*Disk) {
+	pvs := a.PVs
+	if pvs == nil {
+		// No PV usage to reconcile
+		return
+	}
+	// Set PV Adjustment for allocation to 0 for idempotency
+	a.PVCostAdjustment = 0.0
+	for pvKey, pvUsage := range pvs {
+		disk, ok := diskByName[pvKey.Name]
+		if !ok {
+			// Failed to find disk in assets
+			continue
+		}
+		// Check the proportion of disk that is being used by
+		pvUsageProportion := 0.0
+		if disk.ByteHours != 0 {
+			pvUsageProportion = pvUsage.ByteHours / disk.ByteHours
+		} else {
+			log.Warningf("Missing Byte Hours for disk: %s", pvKey)
+		}
+
+		// take proportion of disk adjusted cost
+		allocPVCost := pvUsageProportion * disk.TotalCost()
+
+		// PVCostAdjustment is cumulative as there can be many PVs for each Allocation
+		a.PVCostAdjustment += allocPVCost - pvUsage.Cost
+	}
+
+}
+
 // Delete removes the allocation with the given name from the set
 func (as *AllocationSet) Delete(name string) {
 	if as == nil {

pkg/kubecost/asset.go

@@ -1663,6 +1663,7 @@ type Node struct {
 	NodeType     string
 	CPUCoreHours float64
 	RAMByteHours float64
+	GPUHours     float64
 	CPUBreakdown *Breakdown
 	RAMBreakdown *Breakdown
 	CPUCost      float64
@@ -1889,6 +1890,7 @@ func (n *Node) add(that *Node) {
 
 	n.CPUCoreHours += that.CPUCoreHours
 	n.RAMByteHours += that.RAMByteHours
+	n.GPUHours += that.GPUHours
 
 	n.CPUCost += that.CPUCost
 	n.GPUCost += that.GPUCost
@@ -1912,6 +1914,7 @@ func (n *Node) Clone() Asset {
 		NodeType:     n.NodeType,
 		CPUCoreHours: n.CPUCoreHours,
 		RAMByteHours: n.RAMByteHours,
+		GPUHours:     n.GPUHours,
 		CPUBreakdown: n.CPUBreakdown.Clone(),
 		RAMBreakdown: n.RAMBreakdown.Clone(),
 		CPUCost:      n.CPUCost,
@@ -1960,6 +1963,9 @@ func (n *Node) Equal(a Asset) bool {
 	if n.RAMByteHours != that.RAMByteHours {
 		return false
 	}
+	if n.GPUHours != that.GPUHours {
+		return false
+	}
 	if !n.CPUBreakdown.Equal(that.CPUBreakdown) {
 		return false
 	}
@@ -2000,13 +2006,14 @@ func (n *Node) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "ramBytes", n.RAMBytes(), ",")
 	jsonEncodeFloat64(buffer, "cpuCoreHours", n.CPUCoreHours, ",")
 	jsonEncodeFloat64(buffer, "ramByteHours", n.RAMByteHours, ",")
+	jsonEncodeFloat64(buffer, "GPUHours", n.GPUHours, ",")
 	jsonEncode(buffer, "cpuBreakdown", n.CPUBreakdown, ",")
 	jsonEncode(buffer, "ramBreakdown", n.RAMBreakdown, ",")
 	jsonEncodeFloat64(buffer, "preemptible", n.Preemptible, ",")
 	jsonEncodeFloat64(buffer, "discount", n.Discount, ",")
 	jsonEncodeFloat64(buffer, "cpuCost", n.CPUCost, ",")
 	jsonEncodeFloat64(buffer, "gpuCost", n.GPUCost, ",")
-	jsonEncodeFloat64(buffer, "gpuCount", n.GPUCount, ",")
+	jsonEncodeFloat64(buffer, "gpuCount", n.GPUs(), ",")
 	jsonEncodeFloat64(buffer, "ramCost", n.RAMCost, ",")
 	jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
 	jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
@@ -2047,15 +2054,30 @@ func (n *Node) CPUCores() float64 {
 // and a 16GiB-RAM node running for the last 20 hours of the same 24-hour window
 // would produce:
 //   (12*10 + 16*20) / 24 = 18.333GiB RAM
-// However, any number of cores running for the full span of a window will
-// report the actual number of cores of the static node; e.g. the above
+// However, any number of bytes running for the full span of a window will
+// report the actual number of bytes of the static node; e.g. the above
 // scenario for one entire 24-hour window:
-//   (12*24 + 16*24) / 24 = (12 + 16) = 28 cores
+//   (12*24 + 16*24) / 24 = (12 + 16) = 28GiB RAM
 func (n *Node) RAMBytes() float64 {
 	// [b*hr]*([min/hr]*[1/min]) = [b*hr]/[hr] = b
 	return n.RAMByteHours * (60.0 / n.Minutes())
 }
 
+// GPUs returns the amount of GPUs belonging to the node. This could be
+// fractional because it's the number of gpu*hours divided by the number of
+// hours running; e.g. the sum of a 2 gpu node running for the first 10 hours
+// and a 1 gpu node running for the last 20 hours of the same 24-hour window
+// would produce:
+//   (2*10 + 1*20) / 24 = 1.667 GPUs
+// However, any number of GPUs running for the full span of a window will
+// report the actual number of GPUs of the static node; e.g. the above
+// scenario for one entire 24-hour window:
+//   (2*24 + 1*24) / 24 = (2 + 1) = 3 GPUs
+func (n *Node) GPUs() float64 {
+	// [b*hr]*([min/hr]*[1/min]) = [b*hr]/[hr] = b
+	return n.GPUHours * (60.0 / n.Minutes())
+}
+
 // LoadBalancer is an Asset representing a single load balancer in a cluster
 // TODO: add GB of ingress processed, numForwardingRules once we start recording those to prometheus metric
 type LoadBalancer struct {

pkg/kubecost/asset_test.go

@@ -65,6 +65,7 @@ func generateAssetSet(start time.Time) *AssetSet {
 	node1.Discount = 0.5
 	node1.CPUCoreHours = 2.0 * hours
 	node1.RAMByteHours = 4.0 * gb * hours
+	node1.GPUHours = 1.0 * hours
 	node1.SetAdjustment(1.0)
 	node1.SetLabels(map[string]string{"test": "test"})
 
@@ -75,6 +76,7 @@ func generateAssetSet(start time.Time) *AssetSet {
 	node2.Discount = 0.5
 	node2.CPUCoreHours = 2.0 * hours
 	node2.RAMByteHours = 4.0 * gb * hours
+	node2.GPUHours = 0.0 * hours
 	node2.SetAdjustment(1.5)
 
 	node3 := NewNode("node3", "cluster1", "gcp-node3", *window.Clone().start, *window.Clone().end, window.Clone())
@@ -84,6 +86,7 @@ func generateAssetSet(start time.Time) *AssetSet {
 	node3.Discount = 0.5
 	node3.CPUCoreHours = 2.0 * hours
 	node3.RAMByteHours = 4.0 * gb * hours
+	node3.GPUHours = 2.0 * hours
 	node3.SetAdjustment(-0.5)
 
 	node4 := NewNode("node4", "cluster2", "gcp-node4", *window.Clone().start, *window.Clone().end, window.Clone())
@@ -93,6 +96,7 @@ func generateAssetSet(start time.Time) *AssetSet {
 	node4.Discount = 0.25
 	node4.CPUCoreHours = 4.0 * hours
 	node4.RAMByteHours = 12.0 * gb * hours
+	node4.GPUHours = 0.0 * hours
 	node4.SetAdjustment(-1.0)
 
 	node5 := NewNode("node5", "cluster3", "aws-node5", *window.Clone().start, *window.Clone().end, window.Clone())
@@ -102,6 +106,7 @@ func generateAssetSet(start time.Time) *AssetSet {
 	node5.Discount = 0.0
 	node5.CPUCoreHours = 8.0 * hours
 	node5.RAMByteHours = 24.0 * gb * hours
+	node5.GPUHours = 0.0 * hours
 	node5.SetAdjustment(2.0)
 
 	disk1 := NewDisk("disk1", "cluster1", "gcp-disk1", *window.Clone().start, *window.Clone().end, window.Clone())
@@ -481,6 +486,7 @@ func TestNode_Add(t *testing.T) {
 	node1 := NewNode("node1", "cluster1", "node1", *windows[0].start, *windows[0].end, windows[0])
 	node1.CPUCoreHours = 1.0 * hours
 	node1.RAMByteHours = 2.0 * gb * hours
+	node1.GPUHours = 0.0 * hours
 	node1.GPUCost = 0.0
 	node1.CPUCost = 8.0
 	node1.RAMCost = 4.0
@@ -502,6 +508,7 @@ func TestNode_Add(t *testing.T) {
 	node2 := NewNode("node2", "cluster1", "node2", *windows[0].start, *windows[0].end, windows[0])
 	node2.CPUCoreHours = 1.0 * hours
 	node2.RAMByteHours = 2.0 * gb * hours
+	node2.GPUHours = 0.0 * hours
 	node2.GPUCost = 0.0
 	node2.CPUCost = 3.0
 	node2.RAMCost = 1.0
@@ -566,6 +573,7 @@ func TestNode_Add(t *testing.T) {
 	node3 := NewNode("node3", "cluster1", "node3", *windows[0].start, *windows[0].end, windows[0])
 	node3.CPUCoreHours = 0 * hours
 	node3.RAMByteHours = 0 * hours
+	node3.GPUHours = 0.0 * hours
 	node3.GPUCost = 0
 	node3.CPUCost = 0.0
 	node3.RAMCost = 0.0
@@ -575,6 +583,7 @@ func TestNode_Add(t *testing.T) {
 	node4 := NewNode("node4", "cluster1", "node4", *windows[0].start, *windows[0].end, windows[0])
 	node4.CPUCoreHours = 0 * hours
 	node4.RAMByteHours = 0 * hours
+	node4.GPUHours = 0.0 * hours
 	node4.GPUCost = 0
 	node4.CPUCost = 0.0
 	node4.RAMCost = 0.0
@@ -595,6 +604,7 @@ func TestNode_Add(t *testing.T) {
 	nodeA1 := NewNode("nodeA1", "cluster1", "nodeA1", *windows[0].start, *windows[0].end, windows[0])
 	nodeA1.CPUCoreHours = 1.0 * hours
 	nodeA1.RAMByteHours = 2.0 * gb * hours
+	nodeA1.GPUHours = 0.0 * hours
 	nodeA1.GPUCost = 0.0
 	nodeA1.CPUCost = 8.0
 	nodeA1.RAMCost = 4.0
@@ -604,6 +614,7 @@ func TestNode_Add(t *testing.T) {
 	nodeA2 := NewNode("nodeA2", "cluster1", "nodeA2", *windows[1].start, *windows[1].end, windows[1])
 	nodeA2.CPUCoreHours = 1.0 * hours
 	nodeA2.RAMByteHours = 2.0 * gb * hours
+	nodeA2.GPUHours = 0.0 * hours
 	nodeA2.GPUCost = 0.0
 	nodeA2.CPUCost = 3.0
 	nodeA2.RAMCost = 1.0
@@ -637,6 +648,9 @@ func TestNode_Add(t *testing.T) {
 	if nodeAT.RAMBytes() != 2.0*gb {
 		t.Fatalf("Node.Add: expected %f; got %f", 2.0*gb, nodeAT.RAMBytes())
 	}
+	if nodeAT.GPUs() != 0.0 {
+		t.Fatalf("Node.Add: expected %f; got %f", 0.0, nodeAT.GPUs())
+	}
 
 	// Check that the original assets are unchanged
 	if !util.IsApproximately(nodeA1.TotalCost(), 10.0) {
@@ -664,8 +678,10 @@ func TestNode_MarshalJSON(t *testing.T) {
 	})
 	node.CPUCost = 9.0
 	node.RAMCost = 0.0
+	node.RAMCost = 21.0
 	node.CPUCoreHours = 123.0
 	node.RAMByteHours = 13323.0
+	node.GPUHours = 123.0
 	node.SetAdjustment(1.0)
 
 	_, err := json.Marshal(node)

pkg/kubecost/bingen.go

@@ -25,5 +25,8 @@ package kubecost
 // @bingen:generate:AllocationLabels
 // @bingen:generate:AllocationAnnotations
 // @bingen:generate:RawAllocationOnlyData
+// @bingen:generate:PVAllocations
+// @bingen:generate:PVKey
+// @bingen:generate:PVAllocation
 
-//go:generate bingen -package=kubecost -version=11 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=13 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/kubecost_codecs.go

@@ -25,7 +25,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 11
+	CodecVersion uint8 = 13
 )
 
 //--------------------------------------------------------------------------
@@ -50,6 +50,8 @@ var typeMap map[string]reflect.Type = map[string]reflect.Type{
 	"LoadBalancer":          reflect.TypeOf((*LoadBalancer)(nil)).Elem(),
 	"Network":               reflect.TypeOf((*Network)(nil)).Elem(),
 	"Node":                  reflect.TypeOf((*Node)(nil)).Elem(),
+	"PVAllocation":          reflect.TypeOf((*PVAllocation)(nil)).Elem(),
+	"PVKey":                 reflect.TypeOf((*PVKey)(nil)).Elem(),
 	"RawAllocationOnlyData": reflect.TypeOf((*RawAllocationOnlyData)(nil)).Elem(),
 	"SharedAsset":           reflect.TypeOf((*SharedAsset)(nil)).Elem(),
 	"Window":                reflect.TypeOf((*Window)(nil)).Elem(),
@@ -160,21 +162,65 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteBytes(d)
 	// --- [end][write][reference](time.Time) ---
 
-	buff.WriteFloat64(target.CPUCoreHours)           // write float64
-	buff.WriteFloat64(target.CPUCoreRequestAverage)  // write float64
-	buff.WriteFloat64(target.CPUCoreUsageAverage)    // write float64
-	buff.WriteFloat64(target.CPUCost)                // write float64
-	buff.WriteFloat64(target.GPUHours)               // write float64
-	buff.WriteFloat64(target.GPUCost)                // write float64
-	buff.WriteFloat64(target.NetworkCost)            // write float64
-	buff.WriteFloat64(target.LoadBalancerCost)       // write float64
-	buff.WriteFloat64(target.PVByteHours)            // write float64
-	buff.WriteFloat64(target.PVCost)                 // write float64
+	buff.WriteFloat64(target.CPUCoreHours)               // write float64
+	buff.WriteFloat64(target.CPUCoreRequestAverage)      // write float64
+	buff.WriteFloat64(target.CPUCoreUsageAverage)        // write float64
+	buff.WriteFloat64(target.CPUCost)                    // write float64
+	buff.WriteFloat64(target.CPUCostAdjustment)          // write float64
+	buff.WriteFloat64(target.GPUHours)                   // write float64
+	buff.WriteFloat64(target.GPUCost)                    // write float64
+	buff.WriteFloat64(target.GPUCostAdjustment)          // write float64
+	buff.WriteFloat64(target.NetworkCost)                // write float64
+	buff.WriteFloat64(target.NetworkCostAdjustment)      // write float64
+	buff.WriteFloat64(target.LoadBalancerCost)           // write float64
+	buff.WriteFloat64(target.LoadBalancerCostAdjustment) // write float64
+	// --- [begin][write][alias](PVAllocations) ---
+	if map[PVKey]*PVAllocation(target.PVs) == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		// --- [begin][write][map](map[PVKey]*PVAllocation) ---
+		buff.WriteInt(len(map[PVKey]*PVAllocation(target.PVs))) // map length
+		for v, z := range map[PVKey]*PVAllocation(target.PVs) {
+			// --- [begin][write][struct](PVKey) ---
+			e, errE := v.MarshalBinary()
+			if errE != nil {
+				return nil, errE
+			}
+			buff.WriteInt(len(e))
+			buff.WriteBytes(e)
+			// --- [end][write][struct](PVKey) ---
+
+			if z == nil {
+				buff.WriteUInt8(uint8(0)) // write nil byte
+			} else {
+				buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+				// --- [begin][write][struct](PVAllocation) ---
+				f, errF := z.MarshalBinary()
+				if errF != nil {
+					return nil, errF
+				}
+				buff.WriteInt(len(f))
+				buff.WriteBytes(f)
+				// --- [end][write][struct](PVAllocation) ---
+
+			}
+		}
+		// --- [end][write][map](map[PVKey]*PVAllocation) ---
+
+	}
+	// --- [end][write][alias](PVAllocations) ---
+
+	buff.WriteFloat64(target.PVCostAdjustment)       // write float64
 	buff.WriteFloat64(target.RAMByteHours)           // write float64
 	buff.WriteFloat64(target.RAMBytesRequestAverage) // write float64
 	buff.WriteFloat64(target.RAMBytesUsageAverage)   // write float64
 	buff.WriteFloat64(target.RAMCost)                // write float64
+	buff.WriteFloat64(target.RAMCostAdjustment)      // write float64
 	buff.WriteFloat64(target.SharedCost)             // write float64
+	buff.WriteFloat64(target.SharedCostAdjustment)   // write float64
 	buff.WriteFloat64(target.ExternalCost)           // write float64
 	if target.RawAllocationOnly == nil {
 		buff.WriteUInt8(uint8(0)) // write nil byte
@@ -182,12 +228,12 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 		buff.WriteUInt8(uint8(1)) // write non-nil byte
 
 		// --- [begin][write][struct](RawAllocationOnlyData) ---
-		e, errE := target.RawAllocationOnly.MarshalBinary()
-		if errE != nil {
-			return nil, errE
+		g, errG := target.RawAllocationOnly.MarshalBinary()
+		if errG != nil {
+			return nil, errG
 		}
-		buff.WriteInt(len(e))
-		buff.WriteBytes(e)
+		buff.WriteInt(len(g))
+		buff.WriteBytes(g)
 		// --- [end][write][struct](RawAllocationOnlyData) ---
 
 	}
@@ -282,53 +328,113 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	target.CPUCost = s
 
 	t := buff.ReadFloat64() // read float64
-	target.GPUHours = t
+	target.CPUCostAdjustment = t
 
 	u := buff.ReadFloat64() // read float64
-	target.GPUCost = u
+	target.GPUHours = u
 
 	w := buff.ReadFloat64() // read float64
-	target.NetworkCost = w
+	target.GPUCost = w
 
 	x := buff.ReadFloat64() // read float64
-	target.LoadBalancerCost = x
+	target.GPUCostAdjustment = x
 
 	y := buff.ReadFloat64() // read float64
-	target.PVByteHours = y
+	target.NetworkCost = y
 
 	aa := buff.ReadFloat64() // read float64
-	target.PVCost = aa
+	target.NetworkCostAdjustment = aa
 
 	bb := buff.ReadFloat64() // read float64
-	target.RAMByteHours = bb
+	target.LoadBalancerCost = bb
 
 	cc := buff.ReadFloat64() // read float64
-	target.RAMBytesRequestAverage = cc
+	target.LoadBalancerCostAdjustment = cc
 
-	dd := buff.ReadFloat64() // read float64
-	target.RAMBytesUsageAverage = dd
+	// --- [begin][read][alias](PVAllocations) ---
+	var dd map[PVKey]*PVAllocation
+	if buff.ReadUInt8() == uint8(0) {
+		dd = nil
+	} else {
+		// --- [begin][read][map](map[PVKey]*PVAllocation) ---
+		ff := buff.ReadInt() // map len
+		ee := make(map[PVKey]*PVAllocation, ff)
+		for i := 0; i < ff; i++ {
+			// --- [begin][read][struct](PVKey) ---
+			gg := &PVKey{}
+			hh := buff.ReadInt()     // byte array length
+			kk := buff.ReadBytes(hh) // byte array
+			errE := gg.UnmarshalBinary(kk)
+			if errE != nil {
+				return errE
+			}
+			v := *gg
+			// --- [end][read][struct](PVKey) ---
 
-	ee := buff.ReadFloat64() // read float64
-	target.RAMCost = ee
+			var z *PVAllocation
+			if buff.ReadUInt8() == uint8(0) {
+				z = nil
+			} else {
+				// --- [begin][read][struct](PVAllocation) ---
+				ll := &PVAllocation{}
+				mm := buff.ReadInt()     // byte array length
+				nn := buff.ReadBytes(mm) // byte array
+				errF := ll.UnmarshalBinary(nn)
+				if errF != nil {
+					return errF
+				}
+				z = ll
+				// --- [end][read][struct](PVAllocation) ---
 
-	ff := buff.ReadFloat64() // read float64
-	target.SharedCost = ff
+			}
+			ee[v] = z
+		}
+		dd = ee
+		// --- [end][read][map](map[PVKey]*PVAllocation) ---
 
-	gg := buff.ReadFloat64() // read float64
-	target.ExternalCost = gg
+	}
+	target.PVs = PVAllocations(dd)
+	// --- [end][read][alias](PVAllocations) ---
+
+	oo := buff.ReadFloat64() // read float64
+	target.PVCostAdjustment = oo
+
+	pp := buff.ReadFloat64() // read float64
+	target.RAMByteHours = pp
+
+	qq := buff.ReadFloat64() // read float64
+	target.RAMBytesRequestAverage = qq
+
+	rr := buff.ReadFloat64() // read float64
+	target.RAMBytesUsageAverage = rr
+
+	ss := buff.ReadFloat64() // read float64
+	target.RAMCost = ss
+
+	tt := buff.ReadFloat64() // read float64
+	target.RAMCostAdjustment = tt
+
+	uu := buff.ReadFloat64() // read float64
+	target.SharedCost = uu
+
+	ww := buff.ReadFloat64() // read float64
+	target.SharedCostAdjustment = ww
+
+	xx := buff.ReadFloat64() // read float64
+	target.ExternalCost = xx
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.RawAllocationOnly = nil
 	} else {
 		// --- [begin][read][struct](RawAllocationOnlyData) ---
-		hh := &RawAllocationOnlyData{}
-		kk := buff.ReadInt()     // byte array length
-		ll := buff.ReadBytes(kk) // byte array
-		errE := hh.UnmarshalBinary(ll)
-		if errE != nil {
-			return errE
-		}
-		target.RawAllocationOnly = hh
+		yy := &RawAllocationOnlyData{}
+		aaa := buff.ReadInt()      // byte array length
+		bbb := buff.ReadBytes(aaa) // byte array
+		errG := yy.UnmarshalBinary(bbb)
+		if errG != nil {
+			return errG
+		}
+		target.RawAllocationOnly = yy
 		// --- [end][read][struct](RawAllocationOnlyData) ---
 
 	}
@@ -2658,6 +2764,7 @@ func (target *Node) MarshalBinary() (data []byte, err error) {
 	buff.WriteString(target.NodeType)      // write string
 	buff.WriteFloat64(target.CPUCoreHours) // write float64
 	buff.WriteFloat64(target.RAMByteHours) // write float64
+	buff.WriteFloat64(target.GPUHours)     // write float64
 	if target.CPUBreakdown == nil {
 		buff.WriteUInt8(uint8(0)) // write nil byte
 	} else {
@@ -2807,18 +2914,21 @@ func (target *Node) UnmarshalBinary(data []byte) (err error) {
 	x := buff.ReadFloat64() // read float64
 	target.RAMByteHours = x
 
+	y := buff.ReadFloat64() // read float64
+	target.GPUHours = y
+
 	if buff.ReadUInt8() == uint8(0) {
 		target.CPUBreakdown = nil
 	} else {
 		// --- [begin][read][struct](Breakdown) ---
-		y := &Breakdown{}
-		aa := buff.ReadInt()     // byte array length
-		bb := buff.ReadBytes(aa) // byte array
-		errE := y.UnmarshalBinary(bb)
+		aa := &Breakdown{}
+		bb := buff.ReadInt()     // byte array length
+		cc := buff.ReadBytes(bb) // byte array
+		errE := aa.UnmarshalBinary(cc)
 		if errE != nil {
 			return errE
 		}
-		target.CPUBreakdown = y
+		target.CPUBreakdown = aa
 		// --- [end][read][struct](Breakdown) ---
 
 	}
@@ -2826,34 +2936,156 @@ func (target *Node) UnmarshalBinary(data []byte) (err error) {
 		target.RAMBreakdown = nil
 	} else {
 		// --- [begin][read][struct](Breakdown) ---
-		cc := &Breakdown{}
-		dd := buff.ReadInt()     // byte array length
-		ee := buff.ReadBytes(dd) // byte array
-		errF := cc.UnmarshalBinary(ee)
+		dd := &Breakdown{}
+		ee := buff.ReadInt()     // byte array length
+		ff := buff.ReadBytes(ee) // byte array
+		errF := dd.UnmarshalBinary(ff)
 		if errF != nil {
 			return errF
 		}
-		target.RAMBreakdown = cc
+		target.RAMBreakdown = dd
 		// --- [end][read][struct](Breakdown) ---
 
 	}
-	ff := buff.ReadFloat64() // read float64
-	target.CPUCost = ff
-
 	gg := buff.ReadFloat64() // read float64
-	target.GPUCost = gg
+	target.CPUCost = gg
 
 	hh := buff.ReadFloat64() // read float64
-	target.GPUCount = hh
+	target.GPUCost = hh
 
 	kk := buff.ReadFloat64() // read float64
-	target.RAMCost = kk
+	target.GPUCount = kk
 
 	ll := buff.ReadFloat64() // read float64
-	target.Discount = ll
+	target.RAMCost = ll
 
 	mm := buff.ReadFloat64() // read float64
-	target.Preemptible = mm
+	target.Discount = mm
+
+	nn := buff.ReadFloat64() // read float64
+	target.Preemptible = nn
+
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  PVAllocation
+//--------------------------------------------------------------------------
+
+// MarshalBinary serializes the internal properties of this PVAllocation instance
+// into a byte array
+func (target *PVAllocation) MarshalBinary() (data []byte, err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBuffer()
+	buff.WriteUInt8(CodecVersion) // version
+
+	buff.WriteFloat64(target.ByteHours) // write float64
+	buff.WriteFloat64(target.Cost)      // write float64
+	return buff.Bytes(), nil
+}
+
+// UnmarshalBinary uses the data passed byte array to set all the internal properties of
+// the PVAllocation type
+func (target *PVAllocation) UnmarshalBinary(data []byte) (err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBufferFromBytes(data)
+
+	// Codec Version Check
+	version := buff.ReadUInt8()
+	if version != CodecVersion {
+		return fmt.Errorf("Invalid Version Unmarshaling PVAllocation. Expected %d, got %d", CodecVersion, version)
+	}
+
+	a := buff.ReadFloat64() // read float64
+	target.ByteHours = a
+
+	b := buff.ReadFloat64() // read float64
+	target.Cost = b
+
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  PVKey
+//--------------------------------------------------------------------------
+
+// MarshalBinary serializes the internal properties of this PVKey instance
+// into a byte array
+func (target *PVKey) MarshalBinary() (data []byte, err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBuffer()
+	buff.WriteUInt8(CodecVersion) // version
+
+	buff.WriteString(target.Cluster) // write string
+	buff.WriteString(target.Name)    // write string
+	return buff.Bytes(), nil
+}
+
+// UnmarshalBinary uses the data passed byte array to set all the internal properties of
+// the PVKey type
+func (target *PVKey) UnmarshalBinary(data []byte) (err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBufferFromBytes(data)
+
+	// Codec Version Check
+	version := buff.ReadUInt8()
+	if version != CodecVersion {
+		return fmt.Errorf("Invalid Version Unmarshaling PVKey. Expected %d, got %d", CodecVersion, version)
+	}
+
+	a := buff.ReadString() // read string
+	target.Cluster = a
+
+	b := buff.ReadString() // read string
+	target.Name = b
 
 	return nil
 }

pkg/kubecost/kubecost_codecs_test.go

@@ -134,6 +134,8 @@ func TestAllocationSetRange_BinaryEncoding(t *testing.T) {
 				t.Fatalf("AllocationSetRange.Binary: missing Allocation: %s", a0)
 			}
 
+			// TODO Sean: fix JSON marshaling of PVs
+			a1.PVs = a0.PVs
 			if !a0.Equal(a1) {
 				t.Fatalf("AllocationSetRange.Binary: unequal Allocations \"%s\": expected %s; found %s", k, a0, a1)
 			}

pkg/kubecost/window_test.go

@@ -198,7 +198,7 @@ func TestParseWindowUTC(t *testing.T) {
 	if week.Duration().Hours() < hoursThisWeek {
 		t.Fatalf(`expect: window "week" to have at least %f hours; actual: %f hours`, hoursThisWeek, week.Duration().Hours())
 	}
-	if !week.End().Before(time.Now().UTC()) {
+	if week.End().After(time.Now().UTC()) {
 		t.Fatalf(`expect: window "week" to end before now; actual: %s ends after %s`, week, time.Now().UTC())
 	}
 

pkg/util/json/json.go

@@ -1,12 +1,14 @@
 package json
 
 import (
-    "encoding/json"
+	"encoding/json"
 
-    jsoniter "github.com/json-iterator/go"
+	jsoniter "github.com/json-iterator/go"
 )
 
 var Marshal = jsoniter.ConfigCompatibleWithStandardLibrary.Marshal
 var Unmarshal = jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal
+
 type Marshaler json.Marshaler
+
 var NewDecoder = json.NewDecoder