Allocation usage maxes stored in optional struct

See comment on the RawAllocationOnlyData struct for an explanation.

Bingen info had to be updated and re-run for the new struct.

pkg/costmodel/allocation.go

@@ -714,7 +714,13 @@ func applyCPUCoresUsedMax(podMap map[podKey]*Pod, resCPUCoresUsedMax []*prom.Que
 			pod.AppendContainer(container)
 		}
 
-		pod.Allocations[container].CPUCoreUsageMax = res.Values[0].Value
+		if pod.Allocations[container].RawAllocationOnly == nil {
+			pod.Allocations[container].RawAllocationOnly = &kubecost.RawAllocationOnlyData{
+				CPUCoreUsageMax: res.Values[0].Value,
+			}
+		} else {
+			pod.Allocations[container].RawAllocationOnly.CPUCoreUsageMax = res.Values[0].Value
+		}
 	}
 }
 
@@ -844,7 +850,13 @@ func applyRAMBytesUsedMax(podMap map[podKey]*Pod, resRAMBytesUsedMax []*prom.Que
 			pod.AppendContainer(container)
 		}
 
-		pod.Allocations[container].RAMBytesUsageMax = res.Values[0].Value
+		if pod.Allocations[container].RawAllocationOnly == nil {
+			pod.Allocations[container].RawAllocationOnly = &kubecost.RawAllocationOnlyData{
+				RAMBytesUsageMax: res.Values[0].Value,
+			}
+		} else {
+			pod.Allocations[container].RawAllocationOnly.RAMBytesUsageMax = res.Values[0].Value
+		}
 	}
 }
 

pkg/kubecost/allocation.go

@@ -58,7 +58,6 @@ type Allocation struct {
 	CPUCoreHours           float64    `json:"cpuCoreHours"`
 	CPUCoreRequestAverage  float64    `json:"cpuCoreRequestAverage"`
 	CPUCoreUsageAverage    float64    `json:"cpuCoreUsageAverage"`
-	CPUCoreUsageMax        float64    `json:"cpuCoreUsageMax"`
 	CPUCost                float64    `json:"cpuCost"`
 	GPUHours               float64    `json:"gpuHours"`
 	GPUCost                float64    `json:"gpuCost"`
@@ -69,10 +68,41 @@ type Allocation struct {
 	RAMByteHours           float64    `json:"ramByteHours"`
 	RAMBytesRequestAverage float64    `json:"ramByteRequestAverage"`
 	RAMBytesUsageAverage   float64    `json:"ramByteUsageAverage"`
-	RAMBytesUsageMax       float64    `json:"ramByteUsageMax"`
 	RAMCost                float64    `json:"ramCost"`
 	SharedCost             float64    `json:"sharedCost"`
 	ExternalCost           float64    `json:"externalCost"`
+
+	// RawAllocationOnly is a pointer so if it is not present it will be
+	// marshalled as null rather than as an object with Go default values.
+	RawAllocationOnly *RawAllocationOnlyData `json:"rawAllocationOnly"`
+}
+
+// RawAllocationOnlyData is information that only belong in "raw" Allocations,
+// those which have not undergone aggregation, accumulation, or any other form
+// of combination to produce a new Allocation from other Allocations.
+//
+// Max usage data belongs here because computing the overall maximum from two
+// or more Allocations is a non-trivial operation that cannot be defined without
+// maintaining a large amount of state. Consider the following example:
+// _______________________________________________
+//
+// A1 Using 3 CPU    ----      -----     ------
+// A2 Using 2 CPU      ----      -----      ----
+// A3 Using 1 CPU         ---       --
+// _______________________________________________
+//                   Time ---->
+//
+// The logical maximum CPU usage is 5, but this cannot be calculated iteratively,
+// which is how we calculate aggregations and accumulations of Allocations currently.
+// This becomes a problem I could call "maximum sum of overlapping intervals" and is
+// essentially a variant of an interval scheduling algorithm.
+//
+// If we had types to differentiate between regular Allocations and AggregatedAllocations
+// then this type would be unnecessary and its fields would go into the regular Allocation
+// and not in the AggregatedAllocation.
+type RawAllocationOnlyData struct {
+	CPUCoreUsageMax  float64 `json:"cpuCoreUsageMax"`
+	RAMBytesUsageMax float64 `json:"ramByteUsageMax"`
 }
 
 // AllocationMatchFunc is a function that can be used to match Allocations by
@@ -113,7 +143,6 @@ func (a *Allocation) Clone() *Allocation {
 		CPUCoreHours:           a.CPUCoreHours,
 		CPUCoreRequestAverage:  a.CPUCoreRequestAverage,
 		CPUCoreUsageAverage:    a.CPUCoreUsageAverage,
-		CPUCoreUsageMax:        a.CPUCoreUsageMax,
 		CPUCost:                a.CPUCost,
 		GPUHours:               a.GPUHours,
 		GPUCost:                a.GPUCost,
@@ -124,10 +153,22 @@ func (a *Allocation) Clone() *Allocation {
 		RAMByteHours:           a.RAMByteHours,
 		RAMBytesRequestAverage: a.RAMBytesRequestAverage,
 		RAMBytesUsageAverage:   a.RAMBytesUsageAverage,
-		RAMBytesUsageMax:       a.RAMBytesUsageMax,
 		RAMCost:                a.RAMCost,
 		SharedCost:             a.SharedCost,
 		ExternalCost:           a.ExternalCost,
+		RawAllocationOnly:      a.RawAllocationOnly.Clone(),
+	}
+}
+
+// Clone returns a deep copy of the given RawAllocationOnlyData
+func (r *RawAllocationOnlyData) Clone() *RawAllocationOnlyData {
+	if r == nil {
+		return nil
+	}
+
+	return &RawAllocationOnlyData{
+		CPUCoreUsageMax:  r.CPUCoreUsageMax,
+		RAMBytesUsageMax: r.RAMBytesUsageMax,
 	}
 }
 
@@ -191,13 +232,23 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.ExternalCost, that.ExternalCost) {
 		return false
 	}
-	if !util.IsApproximately(a.CPUCoreUsageMax, that.CPUCoreUsageMax) {
+
+	if a.RawAllocationOnly == nil && that.RawAllocationOnly != nil {
 		return false
 	}
-	if !util.IsApproximately(a.RAMBytesUsageMax, that.RAMBytesUsageMax) {
+	if a.RawAllocationOnly != nil && that.RawAllocationOnly == nil {
 		return false
 	}
 
+	if a.RawAllocationOnly != nil && that.RawAllocationOnly != nil {
+		if !util.IsApproximately(a.RawAllocationOnly.CPUCoreUsageMax, that.RawAllocationOnly.CPUCoreUsageMax) {
+			return false
+		}
+		if !util.IsApproximately(a.RawAllocationOnly.RAMBytesUsageMax, that.RawAllocationOnly.RAMBytesUsageMax) {
+			return false
+		}
+	}
+
 	return true
 }
 
@@ -284,7 +335,6 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "cpuCores", a.CPUCores(), ",")
 	jsonEncodeFloat64(buffer, "cpuCoreRequestAverage", a.CPUCoreRequestAverage, ",")
 	jsonEncodeFloat64(buffer, "cpuCoreUsageAverage", a.CPUCoreUsageAverage, ",")
-	jsonEncodeFloat64(buffer, "cpuCoreUsageMax", a.CPUCoreUsageMax, ",")
 	jsonEncodeFloat64(buffer, "cpuCoreHours", a.CPUCoreHours, ",")
 	jsonEncodeFloat64(buffer, "cpuCost", a.CPUCost, ",")
 	jsonEncodeFloat64(buffer, "cpuEfficiency", a.CPUEfficiency(), ",")
@@ -299,13 +349,13 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "ramByteRequestAverage", a.RAMBytesRequestAverage, ",")
 	jsonEncodeFloat64(buffer, "ramByteUsageAverage", a.RAMBytesUsageAverage, ",")
 	jsonEncodeFloat64(buffer, "ramByteHours", a.RAMByteHours, ",")
-	jsonEncodeFloat64(buffer, "ramByteUsageMax", a.RAMBytesUsageMax, ",")
 	jsonEncodeFloat64(buffer, "ramCost", a.RAMCost, ",")
 	jsonEncodeFloat64(buffer, "ramEfficiency", a.RAMEfficiency(), ",")
 	jsonEncodeFloat64(buffer, "sharedCost", a.SharedCost, ",")
 	jsonEncodeFloat64(buffer, "externalCost", a.ExternalCost, ",")
 	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), ",")
-	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), "")
+	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), ",")
+	jsonEncode(buffer, "rawAllocationOnly", a.RawAllocationOnly, "")
 	buffer.WriteString("}")
 	return buffer.Bytes(), nil
 }
@@ -447,6 +497,10 @@ func (a *Allocation) add(that *Allocation) {
 	a.LoadBalancerCost += that.LoadBalancerCost
 	a.SharedCost += that.SharedCost
 	a.ExternalCost += that.ExternalCost
+
+	// Any data that is in a "raw allocation only" is not valid in any
+	// sort of cumulative Allocation (like one that is added).
+	a.RawAllocationOnly = nil
 }
 
 // AllocationSet stores a set of Allocations, each with a unique name, that share

pkg/kubecost/allocation_test.go

@@ -52,6 +52,10 @@ func NewUnitAllocation(name string, start time.Time, resolution time.Duration, p
 		RAMCost:                1,
 		RAMBytesRequestAverage: 1,
 		RAMBytesUsageAverage:   1,
+		RawAllocationOnly: &RawAllocationOnlyData{
+			CPUCoreUsageMax:  1,
+			RAMBytesUsageMax: 1,
+		},
 	}
 
 	// If idle allocation, remove non-idle costs, but maintain total cost
@@ -117,6 +121,7 @@ func TestAllocation_Add(t *testing.T) {
 		RAMCost:                8.0 * hrs1 * ramPrice,
 		SharedCost:             2.00,
 		ExternalCost:           1.00,
+		RawAllocationOnly:      &RawAllocationOnlyData{},
 	}
 	a1b := a1.Clone()
 
@@ -142,6 +147,7 @@ func TestAllocation_Add(t *testing.T) {
 		LoadBalancerCost:       0.05,
 		SharedCost:             0.00,
 		ExternalCost:           1.00,
+		RawAllocationOnly:      &RawAllocationOnlyData{},
 	}
 	a2b := a2.Clone()
 
@@ -237,6 +243,10 @@ func TestAllocation_Add(t *testing.T) {
 	if !util.IsApproximately(1.6493506, act.TotalEfficiency()) {
 		t.Fatalf("Allocation.Add: expected %f; actual %f", 1.6493506, act.TotalEfficiency())
 	}
+
+	if act.RawAllocationOnly != nil {
+		t.Errorf("Allocation.Add: Raw only data must be nil after an add")
+	}
 }
 
 func TestAllocation_Share(t *testing.T) {
@@ -423,6 +433,7 @@ func TestAllocation_MarshalJSON(t *testing.T) {
 		RAMCost:                8.0 * hrs * ramPrice,
 		SharedCost:             2.00,
 		ExternalCost:           1.00,
+		RawAllocationOnly:      &RawAllocationOnlyData{},
 	}
 
 	data, err := json.Marshal(before)

pkg/kubecost/bingen.go

@@ -20,5 +20,6 @@ package kubecost
 // @bingen:generate:Allocation
 // @bingen:generate:AllocationSet
 // @bingen:generate:AllocationSetRange
+// @bingen:generate:RawAllocationOnlyData
 
-//go:generate bingen -package=kubecost -version=9 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=10 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/kubecost_codecs.go

@@ -25,7 +25,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 9
+	CodecVersion uint8 = 10
 )
 
 //--------------------------------------------------------------------------
@@ -35,22 +35,23 @@ const (
 // Generated type map for resolving interface implementations to
 // to concrete types
 var typeMap map[string]reflect.Type = map[string]reflect.Type{
-	"Allocation":         reflect.TypeOf((*Allocation)(nil)).Elem(),
-	"AllocationSet":      reflect.TypeOf((*AllocationSet)(nil)).Elem(),
-	"AllocationSetRange": reflect.TypeOf((*AllocationSetRange)(nil)).Elem(),
-	"Any":                reflect.TypeOf((*Any)(nil)).Elem(),
-	"AssetProperties":    reflect.TypeOf((*AssetProperties)(nil)).Elem(),
-	"AssetSet":           reflect.TypeOf((*AssetSet)(nil)).Elem(),
-	"AssetSetRange":      reflect.TypeOf((*AssetSetRange)(nil)).Elem(),
-	"Breakdown":          reflect.TypeOf((*Breakdown)(nil)).Elem(),
-	"Cloud":              reflect.TypeOf((*Cloud)(nil)).Elem(),
-	"ClusterManagement":  reflect.TypeOf((*ClusterManagement)(nil)).Elem(),
-	"Disk":               reflect.TypeOf((*Disk)(nil)).Elem(),
-	"LoadBalancer":       reflect.TypeOf((*LoadBalancer)(nil)).Elem(),
-	"Network":            reflect.TypeOf((*Network)(nil)).Elem(),
-	"Node":               reflect.TypeOf((*Node)(nil)).Elem(),
-	"SharedAsset":        reflect.TypeOf((*SharedAsset)(nil)).Elem(),
-	"Window":             reflect.TypeOf((*Window)(nil)).Elem(),
+	"Allocation":            reflect.TypeOf((*Allocation)(nil)).Elem(),
+	"AllocationSet":         reflect.TypeOf((*AllocationSet)(nil)).Elem(),
+	"AllocationSetRange":    reflect.TypeOf((*AllocationSetRange)(nil)).Elem(),
+	"Any":                   reflect.TypeOf((*Any)(nil)).Elem(),
+	"AssetProperties":       reflect.TypeOf((*AssetProperties)(nil)).Elem(),
+	"AssetSet":              reflect.TypeOf((*AssetSet)(nil)).Elem(),
+	"AssetSetRange":         reflect.TypeOf((*AssetSetRange)(nil)).Elem(),
+	"Breakdown":             reflect.TypeOf((*Breakdown)(nil)).Elem(),
+	"Cloud":                 reflect.TypeOf((*Cloud)(nil)).Elem(),
+	"ClusterManagement":     reflect.TypeOf((*ClusterManagement)(nil)).Elem(),
+	"Disk":                  reflect.TypeOf((*Disk)(nil)).Elem(),
+	"LoadBalancer":          reflect.TypeOf((*LoadBalancer)(nil)).Elem(),
+	"Network":               reflect.TypeOf((*Network)(nil)).Elem(),
+	"Node":                  reflect.TypeOf((*Node)(nil)).Elem(),
+	"RawAllocationOnlyData": reflect.TypeOf((*RawAllocationOnlyData)(nil)).Elem(),
+	"SharedAsset":           reflect.TypeOf((*SharedAsset)(nil)).Elem(),
+	"Window":                reflect.TypeOf((*Window)(nil)).Elem(),
 }
 
 //--------------------------------------------------------------------------
@@ -168,6 +169,21 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteFloat64(target.RAMCost)                // write float64
 	buff.WriteFloat64(target.SharedCost)             // write float64
 	buff.WriteFloat64(target.ExternalCost)           // write float64
+	if target.RawAllocationOnly == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		// --- [begin][write][struct](RawAllocationOnlyData) ---
+		e, errE := target.RawAllocationOnly.MarshalBinary()
+		if errE != nil {
+			return nil, errE
+		}
+		buff.WriteInt(len(e))
+		buff.WriteBytes(e)
+		// --- [end][write][struct](RawAllocationOnlyData) ---
+
+	}
 	return buff.Bytes(), nil
 }
 
@@ -290,6 +306,21 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	gg := buff.ReadFloat64() // read float64
 	target.ExternalCost = gg
 
+	if buff.ReadUInt8() == uint8(0) {
+		target.RawAllocationOnly = nil
+	} else {
+		// --- [begin][read][struct](RawAllocationOnlyData) ---
+		hh := &RawAllocationOnlyData{}
+		kk := buff.ReadInt()     // byte array length
+		ll := buff.ReadBytes(kk) // byte array
+		errE := hh.UnmarshalBinary(ll)
+		if errE != nil {
+			return errE
+		}
+		target.RawAllocationOnly = hh
+		// --- [end][read][struct](RawAllocationOnlyData) ---
+
+	}
 	return nil
 }
 
@@ -2447,6 +2478,7 @@ func (target *Node) MarshalBinary() (data []byte, err error) {
 	}
 	buff.WriteFloat64(target.CPUCost)     // write float64
 	buff.WriteFloat64(target.GPUCost)     // write float64
+	buff.WriteFloat64(target.GPUCount)    // write float64
 	buff.WriteFloat64(target.RAMCost)     // write float64
 	buff.WriteFloat64(target.Discount)    // write float64
 	buff.WriteFloat64(target.Preemptible) // write float64
@@ -2600,13 +2632,77 @@ func (target *Node) UnmarshalBinary(data []byte) (err error) {
 	target.GPUCost = gg
 
 	hh := buff.ReadFloat64() // read float64
-	target.RAMCost = hh
+	target.GPUCount = hh
 
 	kk := buff.ReadFloat64() // read float64
-	target.Discount = kk
+	target.RAMCost = kk
 
 	ll := buff.ReadFloat64() // read float64
-	target.Preemptible = ll
+	target.Discount = ll
+
+	mm := buff.ReadFloat64() // read float64
+	target.Preemptible = mm
+
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  RawAllocationOnlyData
+//--------------------------------------------------------------------------
+
+// MarshalBinary serializes the internal properties of this RawAllocationOnlyData instance
+// into a byte array
+func (target *RawAllocationOnlyData) MarshalBinary() (data []byte, err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBuffer()
+	buff.WriteUInt8(CodecVersion) // version
+
+	buff.WriteFloat64(target.CPUCoreUsageMax)  // write float64
+	buff.WriteFloat64(target.RAMBytesUsageMax) // write float64
+	return buff.Bytes(), nil
+}
+
+// UnmarshalBinary uses the data passed byte array to set all the internal properties of
+// the RawAllocationOnlyData type
+func (target *RawAllocationOnlyData) UnmarshalBinary(data []byte) (err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBufferFromBytes(data)
+
+	// Codec Version Check
+	version := buff.ReadUInt8()
+	if version != CodecVersion {
+		return fmt.Errorf("Invalid Version Unmarshaling RawAllocationOnlyData. Expected %d, got %d", CodecVersion, version)
+	}
+
+	a := buff.ReadFloat64() // read float64
+	target.CPUCoreUsageMax = a
+
+	b := buff.ReadFloat64() // read float64
+	target.RAMBytesUsageMax = b
 
 	return nil
 }