CostModel.ComputeAllocation WIP: first queries rewritten with 'query'; Allocation type updated to match Minutes changes; WIP Window.Overlaps function

pkg/costmodel/costmodel.go

@@ -1543,6 +1543,13 @@ func requestKeyFor(window kubecost.Window, resolution time.Duration, filterNames
 }
 
 func (cm *CostModel) ComputeAllocation(start, end time.Time) (*kubecost.AllocationSet, error) {
+	// Create a window spanning the requested query
+	s, e := start, end
+	window := kubecost.NewWindow(&s, &e)
+
+	// Create an empty AllocationSet. For safety, in the case of an error, we
+	// should prefer to return this empty set with the error. (In the case of
+	// no error, of course we populate the set and return it.)
 	allocSet := kubecost.NewAllocationSet(start, end)
 
 	// Convert window (start, end) to (duration, offset) for querying Prometheus
@@ -1581,42 +1588,84 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time) (*kubecost.Allocati
 	if offset < time.Minute {
 		offStr = ""
 	}
-	// TODO niko/cdmr dynamic resolution?
+
+	// TODO niko/cdmr dynamic resolution? add to ComputeAllocation() in allocation.Source?
 	resStr := "1m"
-	resPerHr := 60
+	// resPerHr := 60
+
+	var ctx *prom.Context
+	if cm.ThanosClient != nil {
+		ctx = prom.NewContext(cm.ThanosClient)
+	} else {
+		ctx = prom.NewContext(cm.PrometheusClient)
+	}
+
+	// TODO niko/cdmr retries? (That should probably go into the Store.)
+
+	queryMinutes := fmt.Sprintf(`
+		avg(kube_pod_container_status_running{}) by (container, pod, namespace, kubernetes_node, cluster_id)[%s:%s]%s
+	`)
+	resChMinutes := ctx.Query(queryMinutes)
 
-	// TODO niko/cdmr retries? That should probably go into the Store.
+	// TODO niko/cmdr check: will multiple Prometheus jobs multiply the totals?
 
-	// label_replace(label_replace(
+	// TOOD niko/cdmr Riemann-style or average*time style?
+	// [byte*res] * [hr/res] = [byte*hr]
+	// queryRAMAlloc := fmt.Sprintf(`
 	// 	sum(
-	// 		sum_over_time(container_memory_allocation_bytes{container!="",container!="POD", node!=""}[%s])
-	// 	) by (namespace,container,pod,node,cluster_id) * %f / 60 / 60
-	// , "container_name","$1","container","(.+)"), "pod_name","$1","pod","(.+)")
+	// 		sum_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s:%s]%s)
+	// 	) by (container, pod, namespace, node, cluster_id) / %f
+	// `, durStr, resStr, offStr, resPerHr)
 
-	// TODO niko/cmdr will multiple jobs multiply the totals here?
+	queryRAMAlloc := fmt.Sprintf(`
+		avg(
+			avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s:%s]%s)
+		) by (container, pod, namespace, node, cluster_id)
+	`, durStr, resStr, offStr)
+	resChRAMAlloc := ctx.Query(queryRAMAlloc)
 
 	// queryRAMRequests := fmt.Sprintf()
 
 	// queryRAMUsage := fmt.Sprintf()
 
-	// [byte*res] * [hr/res] = [byte*hr]
-	queryRAMAlloc := fmt.Sprintf(`
-		sum(
-			sum_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s:%s]%s)
-		) by (container, pod, namespace, node, cluster_id) / %f
-	`, durStr, resStr, offStr, resPerHr)
+	// TOOD niko/cdmr Riemann-style or average*time style?
+	// [cpu*res] * [hr/res] = [cpu*hr]
+	// queryCPUAlloc := fmt.Sprintf(`
+	// 	sum(
+	// 		sum_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s:%s]%s)
+	// 	) by (container, pod, namespace, node, cluster_id) / %f
+	// `, durStr, resStr, offStr, resPerHr)
 
 	queryCPUAlloc := fmt.Sprintf(`
-		sum(
-			sum_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s:%s]%s)
-		) by (container, pod, namespace, node, cluster_id) / %f
-	`, durStr, resStr, offStr, resPerHr)
+		avg(
+			avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s:%s]%s)
+		) by (container, pod, namespace, node, cluster_id)
+	`, durStr, resStr, offStr)
+	resChCPUAlloc := ctx.Query(queryCPUAlloc)
 
 	// queryCPURequests := fmt.Sprintf()
 
 	// queryCPUUsage := fmt.Sprintf()
 
-	// queryGPURequests := fmt.Sprintf()
+	// avg(
+	// 	label_replace(
+	// 		label_replace(
+	// 			avg(
+	// 				count_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s] %s)
+	// 				*
+	// 				avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s] %s)
+	// 				* %f
+	// 			) by (namespace,container,pod,node,cluster_id) , "container_name","$1","container","(.+)"
+	// 		), "pod_name","$1","pod","(.+)"
+	// 	)
+	// ) by (namespace,container_name,pod_name,node,cluster_id)
+	// * on (pod_name, namespace, cluster_id) group_left(container) label_replace(avg(avg_over_time(kube_pod_status_phase{phase="Running"}[%s] %s)) by (pod,namespace,cluster_id), "pod_name","$1","pod","(.+)")
+
+	queryGPURequests := fmt.Sprintf(`
+		avg(
+			avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s:%s]%s)
+		) by (container, pod, namespace, node, cluster_id)
+	`)
 
 	// queryPVRequests := fmt.Sprintf()
 
@@ -1630,11 +1679,101 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time) (*kubecost.Allocati
 
 	// queryNetInternetRequests := fmt.Sprintf()
 
-	// queryNormalization := fmt.Sprintf()
+	resMinutes, _ := resChMinutes.Await()
+
+	// Build out a map of allocations, starting with (start, end) so that we
+	// begin with minutes, from which we compute resource allocation and cost
+	// totals from measured rate data.
+	// TODO niko/cdmr can we start with a reasonable guess at map size?
+	allocMap := map[string]*kubecost.Allocation{}
+
+	// clusterStarts and clusterEnds record the earliest start and latest end
+	// times, respectively, on a cluster-basis. These are used for unmounted
+	// PVs and other "virtual" Allocations so that minutes are maximally
+	// accurate during start-up or spin-down of a cluster
+	clusterStart := map[string]time.Time{}
+	clusterEnd := map[string]time.Time{}
+
+	for _, res := range resMinutes {
+		if len(res.Values) == 0 {
+			log.Warningf("CostModel.ComputeAllocation: empty minutes result")
+			continue
+		}
+
+		container, err := res.GetString("container")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: minutes query result missing 'container'")
+			continue
+		}
+
+		pod, err := res.GetString("pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: minutes query result missing 'pod': %s", container)
+			continue
+		}
+
+		namespace, err := res.GetString("namespace")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: minutes query result missing 'namespace': %s/%s", container, pod)
+			continue
+		}
+
+		node, err := res.GetString("kubernetes_node")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: minutes query result missing 'kubernetes_node': %s/%s/%s", container, pod, namespace)
+			// TODO niko/cdmr can we do without node?
+			// continue
+		}
+
+		cluster, err := res.GetString("cluster_id")
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
 
-	// queryMinutes := fmt.Sprintf()
+		// TODO niko/cdmr do we really need node here?
+		name := fmt.Sprintf("%s/%s/%s/%s/%s", cluster, node, namespace, pod, container)
 
-	// TODO niko/cdmr minutes?
+		// start is the timestamp of the first minute. We subtract 1m because
+		// this point will actually represent the end of the first minute. We
+		// don't subtract from end (timestamp of the last minute) because it's
+		// already the end of the last minute, which is what we want.
+		start := time.Unix(int64(res.Values[0].Timestamp), 0).Add(-1 * time.Minute)
+		end := time.Unix(int64(res.Values[len(res.Values)-1].Timestamp), 0)
+
+		// Set start if unset or this datum's start time is earlier than the
+		// current earliest time.
+		if _, ok := clusterStart[cluster]; !ok || start.Before(clusterStart[cluster]) {
+			clusterStart[cluster] = start
+		}
+
+		// Set end if unset or this datum's end time is later than the
+		// current latest time.
+		if _, ok := clusterEnd[cluster]; !ok || end.After(clusterEnd[cluster]) {
+			clusterEnd[cluster] = end
+		}
+
+		alloc := &kubecost.Allocation{
+			Name:   name,
+			Start:  start,
+			End:    end,
+			Window: window.Clone(),
+		}
+
+		props := kubecost.Properties{}
+		props.SetContainer(container)
+		props.SetPod(pod)
+		props.SetNamespace(namespace)
+		props.SetNode(node)
+		props.SetCluster(cluster)
+
+		// TODO niko/cdmr controller, labels, annotations, etc.
+
+		allocMap[name] = alloc
+	}
+
+	for _, alloc := range allocMap {
+		allocSet.Set(alloc)
+	}
 
 	return allocSet, nil
 }

pkg/kubecost/allocation.go

@@ -45,10 +45,9 @@ const ShareNone = "__none__"
 type Allocation struct {
 	Name            string     `json:"name"`
 	Properties      Properties `json:"properties,omitempty"`
+	Window          Window     `json:"window"`
 	Start           time.Time  `json:"start"`
 	End             time.Time  `json:"end"`
-	Minutes         float64    `json:"minutes"`
-	ActiveStart     time.Time  `json:"-"`
 	CPUCoreHours    float64    `json:"cpuCoreHours"`
 	CPUCost         float64    `json:"cpuCost"`
 	CPUEfficiency   float64    `json:"cpuEfficiency"`
@@ -82,12 +81,13 @@ func (a *Allocation) Add(that *Allocation) (*Allocation, error) {
 		return that.Clone(), nil
 	}
 
-	if !a.Start.Equal(that.Start) || !a.End.Equal(that.End) {
-		return nil, fmt.Errorf("error adding Allocations: mismatched windows")
+	if that == nil {
+		return a.Clone(), nil
 	}
 
+	// Note: no need to clone "that", as add only mutates the receiver
 	agg := a.Clone()
-	agg.add(that, false, false)
+	agg.add(that)
 
 	return agg, nil
 }
@@ -101,10 +101,9 @@ func (a *Allocation) Clone() *Allocation {
 	return &Allocation{
 		Name:            a.Name,
 		Properties:      a.Properties.Clone(),
+		Window:          a.Window.Clone(),
 		Start:           a.Start,
 		End:             a.End,
-		Minutes:         a.Minutes,
-		ActiveStart:     a.ActiveStart,
 		CPUCoreHours:    a.CPUCoreHours,
 		CPUCost:         a.CPUCost,
 		CPUEfficiency:   a.CPUEfficiency,
@@ -133,16 +132,16 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.Name != that.Name {
 		return false
 	}
-	if !a.Start.Equal(that.Start) {
+	if !a.Properties.Equal(&that.Properties) {
 		return false
 	}
-	if !a.End.Equal(that.End) {
+	if !a.Window.Equal(that.Window) {
 		return false
 	}
-	if a.Minutes != that.Minutes {
+	if !a.Start.Equal(that.Start) {
 		return false
 	}
-	if !a.ActiveStart.Equal(that.ActiveStart) {
+	if !a.End.Equal(that.End) {
 		return false
 	}
 	if a.CPUCoreHours != that.CPUCoreHours {
@@ -190,13 +189,34 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if a.TotalEfficiency != that.TotalEfficiency {
 		return false
 	}
-	if !a.Properties.Equal(&that.Properties) {
-		return false
-	}
 
 	return true
 }
 
+// CPUCores converts the Allocation's CPUCoreHours into average CPUCores
+func (a *Allocation) CPUCores() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.CPUCoreHours / (a.Minutes() / 60.0)
+}
+
+// RAMBytes converts the Allocation's RAMByteHours into average RAMBytes
+func (a *Allocation) RAMBytes() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.RAMByteHours / (a.Minutes() / 60.0)
+}
+
+// PVBytes converts the Allocation's PVByteHours into average PVBytes
+func (a *Allocation) PVBytes() float64 {
+	if a.Minutes() <= 0.0 {
+		return 0.0
+	}
+	return a.PVByteHours / (a.Minutes() / 60.0)
+}
+
 // Resolution returns the duration of time covered by the Allocation
 func (a *Allocation) Resolution() time.Duration {
 	return a.End.Sub(a.Start)
@@ -223,22 +243,44 @@ func (a *Allocation) IsUnallocated() bool {
 	return strings.Contains(a.Name, UnallocatedSuffix)
 }
 
+// Minutes returns the number of minutes the Allocation represents, as defined
+// by the difference between the end and start times.
+func (a *Allocation) Minutes() float64 {
+	return a.End.Sub(a.Start).Minutes()
+}
+
 // Share works like Add, but converts the entire cost of the given Allocation
 // to SharedCost, rather than adding to the individual resource costs.
+// TODO niko/cdmr unit test changes!!!
 func (a *Allocation) Share(that *Allocation) (*Allocation, error) {
-	if a == nil {
-		return that.Clone(), nil
-	}
+	if that == nil {
+		return a.Clone(), nil
+	}
+
+	// Convert all costs of shared Allocation to SharedCost, zero out all
+	// non-shared costs, then add.
+	share := that.Clone()
+	share.SharedCost += share.TotalCost
+	share.TotalEfficiency = 1.0
+	share.CPUCost = 0
+	share.CPUCoreHours = 0
+	share.CPUEfficiency = 0
+	share.RAMCost = 0
+	share.RAMByteHours = 0
+	share.RAMEfficiency = 0
+	share.GPUCost = 0
+	share.GPUHours = 0
+	share.PVCost = 0
+	share.PVByteHours = 0
+	share.NetworkCost = 0
+	share.ExternalCost = 0
 
-	if !a.Start.Equal(that.Start) {
-		return nil, fmt.Errorf("mismatched start time: expected %s, received %s", a.Start, that.Start)
-	}
-	if !a.End.Equal(that.End) {
-		return nil, fmt.Errorf("mismatched start time: expected %s, received %s", a.End, that.End)
+	if a == nil {
+		return share, nil
 	}
 
 	agg := a.Clone()
-	agg.add(that, true, false)
+	agg.add(that)
 
 	return agg, nil
 }
@@ -248,7 +290,7 @@ func (a *Allocation) String() string {
 	return fmt.Sprintf("%s%s=%.2f", a.Name, NewWindow(&a.Start, &a.End), a.TotalCost)
 }
 
-func (a *Allocation) add(that *Allocation, isShared, isAccumulating bool) {
+func (a *Allocation) add(that *Allocation) {
 	if a == nil {
 		log.Warningf("Allocation.AggregateBy: trying to add a nil receiver")
 		return
@@ -275,65 +317,64 @@ func (a *Allocation) add(that *Allocation, isShared, isAccumulating bool) {
 		}
 	}
 
-	if that.ActiveStart.Before(a.ActiveStart) {
-		a.ActiveStart = that.ActiveStart
+	// Expand Window, Start, and End to be the "max" of each between the two
+	// given Allocations.
+	a.Window = a.Window.Expand(that.Window)
+
+	if that.Start.Before(a.Start) {
+		a.Start = that.Start
 	}
 
-	if isAccumulating {
-		if a.Start.After(that.Start) {
-			a.Start = that.Start
-		}
+	if that.End.Before(a.End) {
+		a.End = that.End
+	}
 
-		if a.End.Before(that.End) {
-			a.End = that.End
-		}
+	// Note: efficiency numbers are computed the cost-weighted sum of each
+	// Allocation's efficiency.
+	// e.g. ($10 @ 25%) + ($10 @ 75%)  = (2.5+7.5)/20   =  50%
+	// e.g. ($90 @ 10%) + ($10 @ 100%) = (9.0+10.0)/100 =  19%
+	// e.g. ($100 @ 0%) + ($100 @ 0%)  = (0.0+0.0)/200  =   0%
+	// e.g. ($10 @ 150%) + ($10 @ 50%) = (15.0+5.0)/20  = 100%
+	// e.g. ($0 @ 100%) + ($0 @ 50%)                    =   0% (no div by 0)
 
-		a.Minutes += that.Minutes
-	} else if that.Minutes > a.Minutes {
-		a.Minutes = that.Minutes
+	// Compute CPU efficiency (see note above for methodology)
+	aggCPUCost := a.CPUCost + that.CPUCost
+	if aggCPUCost > 0 {
+		a.CPUEfficiency = (a.CPUEfficiency*a.CPUCost + that.CPUEfficiency*that.CPUCost) / aggCPUCost
+	} else {
+		a.CPUEfficiency = 0.0
 	}
 
-	// isShared determines whether the given allocation should be spread evenly
-	// across resources (e.g. sharing idle allocation) or lumped into a shared
-	// cost category (e.g. sharing namespace, labels).
-	if isShared {
-		a.SharedCost += that.TotalCost
+	// Compute RAM efficiency (see note above for methodology)
+	aggRAMCost := a.RAMCost + that.RAMCost
+	if aggRAMCost > 0 {
+		a.RAMEfficiency = (a.RAMEfficiency*a.RAMCost + that.RAMEfficiency*that.RAMCost) / aggRAMCost
 	} else {
-		a.CPUCoreHours += that.CPUCoreHours
-		a.GPUHours += that.GPUHours
-		a.RAMByteHours += that.RAMByteHours
-		a.PVByteHours += that.PVByteHours
-
-		aggCPUCost := a.CPUCost + that.CPUCost
-		if aggCPUCost > 0 {
-			a.CPUEfficiency = (a.CPUEfficiency*a.CPUCost + that.CPUEfficiency*that.CPUCost) / aggCPUCost
-		} else {
-			a.CPUEfficiency = 0.0
-		}
-
-		aggRAMCost := a.RAMCost + that.RAMCost
-		if aggRAMCost > 0 {
-			a.RAMEfficiency = (a.RAMEfficiency*a.RAMCost + that.RAMEfficiency*that.RAMCost) / aggRAMCost
-		} else {
-			a.RAMEfficiency = 0.0
-		}
-
-		aggTotalCost := a.TotalCost + that.TotalCost
-		if aggTotalCost > 0 {
-			a.TotalEfficiency = (a.TotalEfficiency*a.TotalCost + that.TotalEfficiency*that.TotalCost) / aggTotalCost
-		} else {
-			aggTotalCost = 0.0
-		}
-
-		a.SharedCost += that.SharedCost
-		a.ExternalCost += that.ExternalCost
-		a.CPUCost += that.CPUCost
-		a.GPUCost += that.GPUCost
-		a.NetworkCost += that.NetworkCost
-		a.RAMCost += that.RAMCost
-		a.PVCost += that.PVCost
+		a.RAMEfficiency = 0.0
 	}
 
+	// Compute total efficiency (see note above for methodology)
+	aggTotalCost := a.TotalCost + that.TotalCost
+	if aggTotalCost > 0 {
+		a.TotalEfficiency = (a.TotalEfficiency*a.TotalCost + that.TotalEfficiency*that.TotalCost) / aggTotalCost
+	} else {
+		aggTotalCost = 0.0
+	}
+
+	// Sum all cumulative resource fields
+	a.CPUCoreHours += that.CPUCoreHours
+	a.GPUHours += that.GPUHours
+	a.RAMByteHours += that.RAMByteHours
+	a.PVByteHours += that.PVByteHours
+
+	// Sum all cumulative cost fields
+	a.CPUCost += that.CPUCost
+	a.GPUCost += that.GPUCost
+	a.RAMCost += that.RAMCost
+	a.PVCost += that.PVCost
+	a.NetworkCost += that.NetworkCost
+	a.SharedCost += that.SharedCost
+	a.ExternalCost += that.ExternalCost
 	a.TotalCost += that.TotalCost
 }
 
@@ -1196,10 +1237,10 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 
 		idleAlloc := &Allocation{
 			Name:       fmt.Sprintf("%s/%s", cluster, IdleSuffix),
+			Window:     window.Clone(),
 			Properties: Properties{ClusterProp: cluster},
 			Start:      start,
 			End:        end,
-			Minutes:    end.Sub(start).Minutes(), // TODO deprecate w/ niko/allocation-minutes
 			CPUCost:    resources["cpu"],
 			GPUCost:    resources["gpu"],
 			RAMCost:    resources["ram"],
@@ -1358,7 +1399,7 @@ func (as *AllocationSet) insert(that *Allocation, accumulate bool) error {
 	if _, ok := as.allocations[that.Name]; !ok {
 		as.allocations[that.Name] = that
 	} else {
-		as.allocations[that.Name].add(that, false, accumulate)
+		as.allocations[that.Name].add(that)
 	}
 
 	// If the given Allocation is an external one, record that
@@ -1500,11 +1541,10 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 		return as, nil
 	}
 
-	if that.Start().Before(as.End()) {
-		timefmt := "2006-01-02T15:04:05"
-		err := fmt.Sprintf("that [%s, %s); that [%s, %s)\n", as.Start().Format(timefmt), as.End().Format(timefmt), that.Start().Format(timefmt), that.End().Format(timefmt))
-		return nil, fmt.Errorf("error accumulating AllocationSets: overlapping windows: %s", err)
-	}
+	// TODO niko/cdmr implement first
+	// if that.Window.Overlaps(as.Window) {
+	// 	return nil, fmt.Errorf("AllocationSet.accumulate: overlapping windows: %s", that.Window, as.Window)
+	// }
 
 	// Set start, end to min(start), max(end)
 	start := as.Start()
@@ -1525,12 +1565,6 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 	defer that.RUnlock()
 
 	for _, alloc := range as.allocations {
-		// Change Start and End to match the new window. However, do not
-		// change Minutes because that will be accounted for during the
-		// insert step, if in fact there are two allocations to add.
-		alloc.Start = start
-		alloc.End = end
-
 		err := acc.insert(alloc, true)
 		if err != nil {
 			return nil, err
@@ -1538,12 +1572,6 @@ func (as *AllocationSet) accumulate(that *AllocationSet) (*AllocationSet, error)
 	}
 
 	for _, alloc := range that.allocations {
-		// Change Start and End to match the new window. However, do not
-		// change Minutes because that will be accounted for during the
-		// insert step, if in fact there are two allocations to add.
-		alloc.Start = start
-		alloc.End = end
-
 		err := acc.insert(alloc, true)
 		if err != nil {
 			return nil, err

pkg/kubecost/allocation_test.go

@@ -34,9 +34,9 @@ func NewUnitAllocation(name string, start time.Time, resolution time.Duration, p
 	alloc := &Allocation{
 		Name:            name,
 		Properties:      *properties,
+		Window:          NewWindow(&start, &end).Clone(),
 		Start:           start,
 		End:             end,
-		Minutes:         1440,
 		CPUCoreHours:    1,
 		CPUCost:         1,
 		CPUEfficiency:   1,
@@ -305,8 +305,8 @@ func assertAllocationWindow(t *testing.T, as *AllocationSet, msg string, expStar
 		if !a.End.Equal(expEnd) {
 			t.Fatalf("AllocationSet.AggregateBy[%s]: expected end %s, actual %s", msg, expEnd, a.End)
 		}
-		if a.Minutes != expMinutes {
-			t.Fatalf("AllocationSet.AggregateBy[%s]: expected minutes %f, actual %f", msg, expMinutes, a.Minutes)
+		if a.Minutes() != expMinutes {
+			t.Fatalf("AllocationSet.AggregateBy[%s]: expected minutes %f, actual %f", msg, expMinutes, a.Minutes())
 		}
 	})
 }
@@ -1163,8 +1163,8 @@ func TestAllocationSetRange_Accumulate(t *testing.T) {
 	if !alloc.End.Equal(tomorrow) {
 		t.Fatalf("accumulating AllocationSetRange: expected to end %s; actual %s", tomorrow, alloc.End)
 	}
-	if alloc.Minutes != 2880.0 {
-		t.Fatalf("accumulating AllocationSetRange: expected %f minutes; actual %f", 2880.0, alloc.Minutes)
+	if alloc.Minutes() != 2880.0 {
+		t.Fatalf("accumulating AllocationSetRange: expected %f minutes; actual %f", 2880.0, alloc.Minutes())
 	}
 }
 

pkg/kubecost/asset.go

@@ -5,7 +5,6 @@ import (
 	"encoding"
 	"encoding/json"
 	"fmt"
-	"math"
 	"strings"
 	"sync"
 	"time"
@@ -2986,33 +2985,6 @@ func (asr *AssetSetRange) Window() Window {
 	return NewWindow(&start, &end)
 }
 
-// TODO move everything below to a separate package
-
-func jsonEncodeFloat64(buffer *bytes.Buffer, name string, val float64, comma string) {
-	var encoding string
-	if math.IsNaN(val) {
-		encoding = fmt.Sprintf("\"%s\":null%s", name, comma)
-	} else {
-		encoding = fmt.Sprintf("\"%s\":%f%s", name, val, comma)
-	}
-
-	buffer.WriteString(encoding)
-}
-
-func jsonEncodeString(buffer *bytes.Buffer, name, val, comma string) {
-	buffer.WriteString(fmt.Sprintf("\"%s\":\"%s\"%s", name, val, comma))
-}
-
-func jsonEncode(buffer *bytes.Buffer, name string, obj interface{}, comma string) {
-	buffer.WriteString(fmt.Sprintf("\"%s\":", name))
-	if bytes, err := json.Marshal(obj); err != nil {
-		buffer.WriteString("null")
-	} else {
-		buffer.Write(bytes)
-	}
-	buffer.WriteString(comma)
-}
-
 // Returns true if string slices a and b contain all of the same strings, in any order.
 func sameContents(a, b []string) bool {
 	if len(a) != len(b) {

pkg/kubecost/bingen.go

@@ -21,4 +21,4 @@ package kubecost
 // @bingen:generate:AllocationSet
 // @bingen:generate:AllocationSetRange
 
-//go:generate bingen -package=kubecost -version=5 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=7 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/json.go

@@ -0,0 +1,35 @@
+package kubecost
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math"
+)
+
+// TODO move everything below to a separate package
+
+func jsonEncodeFloat64(buffer *bytes.Buffer, name string, val float64, comma string) {
+	var encoding string
+	if math.IsNaN(val) {
+		encoding = fmt.Sprintf("\"%s\":null%s", name, comma)
+	} else {
+		encoding = fmt.Sprintf("\"%s\":%f%s", name, val, comma)
+	}
+
+	buffer.WriteString(encoding)
+}
+
+func jsonEncodeString(buffer *bytes.Buffer, name, val, comma string) {
+	buffer.WriteString(fmt.Sprintf("\"%s\":\"%s\"%s", name, val, comma))
+}
+
+func jsonEncode(buffer *bytes.Buffer, name string, obj interface{}, comma string) {
+	buffer.WriteString(fmt.Sprintf("\"%s\":", name))
+	if bytes, err := json.Marshal(obj); err != nil {
+		buffer.WriteString("null")
+	} else {
+		buffer.Write(bytes)
+	}
+	buffer.WriteString(comma)
+}

pkg/kubecost/kubecost_codecs.go

@@ -25,7 +25,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 6
+	CodecVersion uint8 = 7
 )
 
 //--------------------------------------------------------------------------
@@ -125,17 +125,17 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteBytes(a)
 	// --- [end][write][reference](Properties) ---
 
-	// --- [begin][write][reference](time.Time) ---
-	b, errB := target.Start.MarshalBinary()
+	// --- [begin][write][struct](Window) ---
+	b, errB := target.Window.MarshalBinary()
 	if errB != nil {
 		return nil, errB
 	}
 	buff.WriteInt(len(b))
 	buff.WriteBytes(b)
-	// --- [end][write][reference](time.Time) ---
+	// --- [end][write][struct](Window) ---
 
 	// --- [begin][write][reference](time.Time) ---
-	c, errC := target.End.MarshalBinary()
+	c, errC := target.Start.MarshalBinary()
 	if errC != nil {
 		return nil, errC
 	}
@@ -143,9 +143,8 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteBytes(c)
 	// --- [end][write][reference](time.Time) ---
 
-	buff.WriteFloat64(target.Minutes) // write float64
 	// --- [begin][write][reference](time.Time) ---
-	d, errD := target.ActiveStart.MarshalBinary()
+	d, errD := target.End.MarshalBinary()
 	if errD != nil {
 		return nil, errD
 	}
@@ -156,6 +155,8 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteFloat64(target.CPUCoreHours)    // write float64
 	buff.WriteFloat64(target.CPUCost)         // write float64
 	buff.WriteFloat64(target.CPUEfficiency)   // write float64
+	buff.WriteFloat64(target.CPURequestAvg)   // write float64
+	buff.WriteFloat64(target.CPUUsageAvg)     // write float64
 	buff.WriteFloat64(target.GPUHours)        // write float64
 	buff.WriteFloat64(target.GPUCost)         // write float64
 	buff.WriteFloat64(target.NetworkCost)     // write float64
@@ -164,7 +165,10 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteFloat64(target.RAMByteHours)    // write float64
 	buff.WriteFloat64(target.RAMCost)         // write float64
 	buff.WriteFloat64(target.RAMEfficiency)   // write float64
+	buff.WriteFloat64(target.RAMRequestAvg)   // write float64
+	buff.WriteFloat64(target.RAMUsageAvg)     // write float64
 	buff.WriteFloat64(target.SharedCost)      // write float64
+	buff.WriteFloat64(target.ExternalCost)    // write float64
 	buff.WriteFloat64(target.TotalCost)       // write float64
 	buff.WriteFloat64(target.TotalEfficiency) // write float64
 	return buff.Bytes(), nil
@@ -208,16 +212,16 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	target.Properties = *b
 	// --- [end][read][reference](Properties) ---
 
-	// --- [begin][read][reference](time.Time) ---
-	e := &time.Time{}
+	// --- [begin][read][struct](Window) ---
+	e := &Window{}
 	f := buff.ReadInt()    // byte array length
 	g := buff.ReadBytes(f) // byte array
 	errB := e.UnmarshalBinary(g)
 	if errB != nil {
 		return errB
 	}
-	target.Start = *e
-	// --- [end][read][reference](time.Time) ---
+	target.Window = *e
+	// --- [end][read][struct](Window) ---
 
 	// --- [begin][read][reference](time.Time) ---
 	h := &time.Time{}
@@ -227,64 +231,76 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	if errC != nil {
 		return errC
 	}
-	target.End = *h
+	target.Start = *h
 	// --- [end][read][reference](time.Time) ---
 
-	n := buff.ReadFloat64() // read float64
-	target.Minutes = n
-
 	// --- [begin][read][reference](time.Time) ---
-	o := &time.Time{}
-	p := buff.ReadInt()    // byte array length
-	q := buff.ReadBytes(p) // byte array
-	errD := o.UnmarshalBinary(q)
+	n := &time.Time{}
+	o := buff.ReadInt()    // byte array length
+	p := buff.ReadBytes(o) // byte array
+	errD := n.UnmarshalBinary(p)
 	if errD != nil {
 		return errD
 	}
-	target.ActiveStart = *o
+	target.End = *n
 	// --- [end][read][reference](time.Time) ---
 
+	q := buff.ReadFloat64() // read float64
+	target.CPUCoreHours = q
+
 	r := buff.ReadFloat64() // read float64
-	target.CPUCoreHours = r
+	target.CPUCost = r
 
 	s := buff.ReadFloat64() // read float64
-	target.CPUCost = s
+	target.CPUEfficiency = s
 
 	t := buff.ReadFloat64() // read float64
-	target.CPUEfficiency = t
+	target.CPURequestAvg = t
 
 	u := buff.ReadFloat64() // read float64
-	target.GPUHours = u
+	target.CPUUsageAvg = u
 
 	w := buff.ReadFloat64() // read float64
-	target.GPUCost = w
+	target.GPUHours = w
 
 	x := buff.ReadFloat64() // read float64
-	target.NetworkCost = x
+	target.GPUCost = x
 
 	y := buff.ReadFloat64() // read float64
-	target.PVByteHours = y
+	target.NetworkCost = y
 
 	z := buff.ReadFloat64() // read float64
-	target.PVCost = z
+	target.PVByteHours = z
 
 	aa := buff.ReadFloat64() // read float64
-	target.RAMByteHours = aa
+	target.PVCost = aa
 
 	bb := buff.ReadFloat64() // read float64
-	target.RAMCost = bb
+	target.RAMByteHours = bb
 
 	cc := buff.ReadFloat64() // read float64
-	target.RAMEfficiency = cc
+	target.RAMCost = cc
 
 	dd := buff.ReadFloat64() // read float64
-	target.SharedCost = dd
+	target.RAMEfficiency = dd
 
 	ee := buff.ReadFloat64() // read float64
-	target.TotalCost = ee
+	target.RAMRequestAvg = ee
 
 	ff := buff.ReadFloat64() // read float64
-	target.TotalEfficiency = ff
+	target.RAMUsageAvg = ff
+
+	gg := buff.ReadFloat64() // read float64
+	target.SharedCost = gg
+
+	hh := buff.ReadFloat64() // read float64
+	target.ExternalCost = hh
+
+	ll := buff.ReadFloat64() // read float64
+	target.TotalCost = ll
+
+	mm := buff.ReadFloat64() // read float64
+	target.TotalEfficiency = mm
 
 	return nil
 }
@@ -340,19 +356,33 @@ func (target *AllocationSet) MarshalBinary() (data []byte, err error) {
 		// --- [end][write][map](map[string]*Allocation) ---
 
 	}
-	if target.idleKeys == nil {
+	if target.externalKeys == nil {
 		buff.WriteUInt8(uint8(0)) // write nil byte
 	} else {
 		buff.WriteUInt8(uint8(1)) // write non-nil byte
 
 		// --- [begin][write][map](map[string]bool) ---
-		buff.WriteInt(len(target.idleKeys)) // map length
-		for kk, vv := range target.idleKeys {
+		buff.WriteInt(len(target.externalKeys)) // map length
+		for kk, vv := range target.externalKeys {
 			buff.WriteString(kk) // write string
 			buff.WriteBool(vv)   // write bool
 		}
 		// --- [end][write][map](map[string]bool) ---
 
+	}
+	if target.idleKeys == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		// --- [begin][write][map](map[string]bool) ---
+		buff.WriteInt(len(target.idleKeys)) // map length
+		for kkk, vvv := range target.idleKeys {
+			buff.WriteString(kkk) // write string
+			buff.WriteBool(vvv)   // write bool
+		}
+		// --- [end][write][map](map[string]bool) ---
+
 	}
 	// --- [begin][write][struct](Window) ---
 	b, errB := target.Window.MarshalBinary()
@@ -450,7 +480,7 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 
 	}
 	if buff.ReadUInt8() == uint8(0) {
-		target.idleKeys = nil
+		target.externalKeys = nil
 	} else {
 		// --- [begin][read][map](map[string]bool) ---
 		g := make(map[string]bool)
@@ -466,35 +496,56 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 
 			g[kk] = vv
 		}
-		target.idleKeys = g
+		target.externalKeys = g
+		// --- [end][read][map](map[string]bool) ---
+
+	}
+	if buff.ReadUInt8() == uint8(0) {
+		target.idleKeys = nil
+	} else {
+		// --- [begin][read][map](map[string]bool) ---
+		n := make(map[string]bool)
+		o := buff.ReadInt() // map len
+		for ii := 0; ii < o; ii++ {
+			var kkk string
+			p := buff.ReadString() // read string
+			kkk = p
+
+			var vvv bool
+			q := buff.ReadBool() // read bool
+			vvv = q
+
+			n[kkk] = vvv
+		}
+		target.idleKeys = n
 		// --- [end][read][map](map[string]bool) ---
 
 	}
 	// --- [begin][read][struct](Window) ---
-	n := &Window{}
-	o := buff.ReadInt()    // byte array length
-	p := buff.ReadBytes(o) // byte array
-	errB := n.UnmarshalBinary(p)
+	r := &Window{}
+	s := buff.ReadInt()    // byte array length
+	t := buff.ReadBytes(s) // byte array
+	errB := r.UnmarshalBinary(t)
 	if errB != nil {
 		return errB
 	}
-	target.Window = *n
+	target.Window = *r
 	// --- [end][read][struct](Window) ---
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.Warnings = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		r := buff.ReadInt() // array len
-		q := make([]string, r)
-		for ii := 0; ii < r; ii++ {
-			var s string
-			t := buff.ReadString() // read string
-			s = t
+		w := buff.ReadInt() // array len
+		u := make([]string, w)
+		for jj := 0; jj < w; jj++ {
+			var x string
+			y := buff.ReadString() // read string
+			x = y
 
-			q[ii] = s
+			u[jj] = x
 		}
-		target.Warnings = q
+		target.Warnings = u
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -502,16 +553,16 @@ func (target *AllocationSet) UnmarshalBinary(data []byte) (err error) {
 		target.Errors = nil
 	} else {
 		// --- [begin][read][slice]([]string) ---
-		w := buff.ReadInt() // array len
-		u := make([]string, w)
-		for jj := 0; jj < w; jj++ {
-			var x string
-			y := buff.ReadString() // read string
-			x = y
+		aa := buff.ReadInt() // array len
+		z := make([]string, aa)
+		for iii := 0; iii < aa; iii++ {
+			var bb string
+			cc := buff.ReadString() // read string
+			bb = cc
 
-			u[jj] = x
+			z[iii] = bb
 		}
-		target.Errors = u
+		target.Errors = z
 		// --- [end][read][slice]([]string) ---
 
 	}
@@ -1406,6 +1457,7 @@ func (target *Cloud) MarshalBinary() (data []byte, err error) {
 
 	buff.WriteFloat64(target.adjustment) // write float64
 	buff.WriteFloat64(target.Cost)       // write float64
+	buff.WriteFloat64(target.Credit)     // write float64
 	return buff.Bytes(), nil
 }
 
@@ -1513,6 +1565,9 @@ func (target *Cloud) UnmarshalBinary(data []byte) (err error) {
 	w := buff.ReadFloat64() // read float64
 	target.Cost = w
 
+	x := buff.ReadFloat64() // read float64
+	target.Credit = x
+
 	return nil
 }
 

pkg/kubecost/window.go

@@ -432,6 +432,86 @@ func (w Window) Minutes() float64 {
 	return w.end.Sub(*w.start).Minutes()
 }
 
+// Overlaps returns true iff the two given Windows share and amount of temporal
+// coverage.
+// TODO niko/cdmr return to this, with unit tests, and then implement in
+// AllocationSet.accumulate
+func (w Window) Overlaps(x Window) bool {
+	if (w.start == nil && w.end == nil) || (x.start == nil && x.end == nil) {
+		// one window is completely open, so overlap is guaranteed
+		// <---------->
+		//   ?------?
+		return true
+	}
+
+	// Neither window is completely open (nil, nil), but one or the other might
+	// still be future- or past-open.
+
+	if w.start == nil {
+		// w is past-open, future-closed
+		// <------]
+
+		if x.start != nil && !x.start.Before(*w.end) {
+			// x starts after w ends (or eq)
+			// <------]
+			//          [------?
+			return false
+		}
+
+		// <-----]
+		//    ?-----?
+		return true
+	}
+
+	if w.end == nil {
+		// w is future-open, past-closed
+		// [------>
+
+		if x.end != nil && !x.end.After(*w.end) {
+			// x ends before w begins (or eq)
+			//          [------>
+			// ?------]
+			return false
+		}
+
+		//    [------>
+		// ?------?
+		return true
+	}
+
+	// Now we know w is closed, but we don't know about x
+	//  [------]
+	//     ?------?
+	if x.start == nil {
+		// TODO niko/cdmr
+	}
+
+	if x.end == nil {
+		// TODO niko/cdmr
+	}
+
+	// Both are closed.
+
+	if !x.start.Before(*w.end) && !x.end.Before(*w.end) {
+		// x starts and ends after w ends
+		// [------]
+		//          [------]
+		return false
+	}
+
+	if !x.start.After(*w.start) && !x.end.After(*w.start) {
+		// x starts and ends before w starts
+		//          [------]
+		// [------]
+		return false
+	}
+
+	// w and x must overlap
+	//    [------]
+	// [------]
+	return true
+}
+
 func (w Window) Set(start, end *time.Time) {
 	w.start = start
 	w.end = end

pkg/kubecost/window_test.go

@@ -211,7 +211,7 @@ func TestParseWindowUTC(t *testing.T) {
 		t.Fatalf(`expect: window "month" to end before now; actual: %s ends after %s`, month, time.Now().UTC())
 	}
 
-	// TODO niko/etl lastweek
+	// TODO lastweek
 
 	lastmonth, err := ParseWindowUTC("lastmonth")
 	monthMinHours := float64(24 * 28)
@@ -542,30 +542,6 @@ func TestParseWindowWithOffsetString(t *testing.T) {
 
 }
 
-// TODO niko/etl
-// func TestWindow_Contains(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Duration(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_End(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Equal(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_ExpandStart(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_ExpandEnd(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_Start(t *testing.T) {}
-
-// TODO niko/etl
-// func TestWindow_String(t *testing.T) {}
-
 func TestWindow_DurationOffsetStrings(t *testing.T) {
 	w, err := ParseWindowUTC("1d")
 	if err != nil {
@@ -624,3 +600,31 @@ func TestWindow_DurationOffsetStrings(t *testing.T) {
 		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
 	}
 }
+
+func TestWindow_Overlaps(t *testing.T) {
+	// TODO niko/cdmr
+}
+
+// TODO
+// func TestWindow_Contains(t *testing.T) {}
+
+// TODO
+// func TestWindow_Duration(t *testing.T) {}
+
+// TODO
+// func TestWindow_End(t *testing.T) {}
+
+// TODO
+// func TestWindow_Equal(t *testing.T) {}
+
+// TODO
+// func TestWindow_ExpandStart(t *testing.T) {}
+
+// TODO
+// func TestWindow_ExpandEnd(t *testing.T) {}
+
+// TODO
+// func TestWindow_Start(t *testing.T) {}
+
+// TODO
+// func TestWindow_String(t *testing.T) {}