Merge branch 'develop' into sean/refactor-properties

# Conflicts:
#	pkg/kubecost/allocation.go
#	pkg/kubecost/bingen.go
#	pkg/kubecost/kubecost_codecs.go

pkg/cloud/awsprovider.go

@@ -191,15 +191,28 @@ type AWSOfferTerm struct {
 	PriceDimensions map[string]*AWSRateCode `json:"priceDimensions"`
 }
 
+func (ot *AWSOfferTerm) String() string {
+	var strs []string
+	for k, rc := range ot.PriceDimensions {
+		strs = append(strs, fmt.Sprintf("%s:%s", k, rc.String()))
+	}
+	return fmt.Sprintf("%s:%s", ot.Sku, strings.Join(strs, ","))
+}
+
 // AWSRateCode encodes data about the price of a product
 type AWSRateCode struct {
 	Unit         string          `json:"unit"`
 	PricePerUnit AWSCurrencyCode `json:"pricePerUnit"`
 }
 
+func (rc *AWSRateCode) String() string {
+	return fmt.Sprintf("{unit: %s, pricePerUnit: %v", rc.Unit, rc.PricePerUnit)
+}
+
 // AWSCurrencyCode is the localized currency. (TODO: support non-USD)
 type AWSCurrencyCode struct {
-	USD string `json:"USD"`
+	USD string `json:"USD,omitempty"`
+	CNY string `json:"CNY,omitempty"`
 }
 
 // AWSProductTerms represents the full terms of the product
@@ -219,12 +232,14 @@ const ClusterIdEnvVar = "AWS_CLUSTER_ID"
 
 // OnDemandRateCode is appended to an node sku
 const OnDemandRateCode = ".JRTCKXETXF"
+const OnDemandRateCodeCn = ".99YE2YK9UR"
 
 // ReservedRateCode is appended to a node sku
 const ReservedRateCode = ".38NPMPTW36"
 
 // HourlyRateCode is appended to a node sku
 const HourlyRateCode = ".6YS6EN2CT7"
+const HourlyRateCodeCn = ".Q7UJUT2CE6"
 
 // volTypes are used to map between AWS UsageTypes and
 // EBS volume types, as they would appear in K8s storage class
@@ -565,7 +580,6 @@ func (aws *AWS) ClusterManagementPricing() (string, float64, error) {
 func (aws *AWS) getRegionPricing(nodeList []*v1.Node) (*http.Response, string, error) {
 
 	pricingURL := "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/"
-
 	region := ""
 	multiregion := false
 	for _, n := range nodeList {
@@ -573,6 +587,10 @@ func (aws *AWS) getRegionPricing(nodeList []*v1.Node) (*http.Response, string, e
 		currentNodeRegion := ""
 		if r, ok := util.GetRegion(labels); ok {
 			currentNodeRegion = r
+			// Switch to Chinese endpoint for regions with the Chinese prefix
+			if strings.HasPrefix(currentNodeRegion, "cn-") {
+				pricingURL = "https://pricing.cn-north-1.amazonaws.com.cn/offers/v1.0/cn/AmazonEC2/current/"
+			}
 		} else {
 			multiregion = true // We weren't able to detect the node's region, so pull all data.
 			break
@@ -585,6 +603,7 @@ func (aws *AWS) getRegionPricing(nodeList []*v1.Node) (*http.Response, string, e
 		}
 	}
 
+	// Chinese multiregion endpoint only contains data for Chinese regions and Chinese regions are excluded from other endpoint
 	if region != "" && !multiregion {
 		pricingURL += region + "/"
 	}
@@ -725,6 +744,9 @@ func (aws *AWS) DownloadPricingData() error {
 		if err == io.EOF {
 			klog.V(2).Infof("done loading \"%s\"\n", pricingURL)
 			break
+		} else if err != nil {
+			klog.V(2).Infof("error parsing response json %v", resp.Body)
+			break
 		}
 		if t == "products" {
 			_, err := dec.Token() // this should parse the opening "{""
@@ -819,28 +841,33 @@ func (aws *AWS) DownloadPricingData() error {
 					if err != nil {
 						klog.V(1).Infof("Error decoding AWS Offer Term: " + err.Error())
 					}
-					if sku.(string)+OnDemandRateCode == skuOnDemand {
-						key, ok := skusToKeys[sku.(string)]
-						spotKey := key + ",preemptible"
-						if ok {
-							aws.Pricing[key].OnDemand = offerTerm
-							aws.Pricing[spotKey].OnDemand = offerTerm
-							if strings.Contains(key, "EBS:VolumeP-IOPS.piops") {
-								// If the specific UsageType is the per IO cost used on io1 volumes
-								// we need to add the per IO cost to the io1 PV cost
-								cost := offerTerm.PriceDimensions[sku.(string)+OnDemandRateCode+HourlyRateCode].PricePerUnit.USD
-								// Add the per IO cost to the PV object for the io1 volume type
-								aws.Pricing[key].PV.CostPerIO = cost
-							} else if strings.Contains(key, "EBS:Volume") {
-								// If volume, we need to get hourly cost and add it to the PV object
-								cost := offerTerm.PriceDimensions[sku.(string)+OnDemandRateCode+HourlyRateCode].PricePerUnit.USD
-								costFloat, _ := strconv.ParseFloat(cost, 64)
-								hourlyPrice := costFloat / 730
-
-								aws.Pricing[key].PV.Cost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
-							}
+
+					key, ok := skusToKeys[sku.(string)]
+					spotKey := key + ",preemptible"
+					if ok {
+						aws.Pricing[key].OnDemand = offerTerm
+						aws.Pricing[spotKey].OnDemand = offerTerm
+						var cost string
+						if sku.(string)+OnDemandRateCode == skuOnDemand {
+							cost = offerTerm.PriceDimensions[sku.(string)+OnDemandRateCode+HourlyRateCode].PricePerUnit.USD
+						} else if sku.(string)+OnDemandRateCodeCn == skuOnDemand {
+							cost = offerTerm.PriceDimensions[sku.(string)+OnDemandRateCodeCn+HourlyRateCodeCn].PricePerUnit.CNY
+						}
+						if strings.Contains(key, "EBS:VolumeP-IOPS.piops") {
+						// If the specific UsageType is the per IO cost used on io1 volumes
+						// we need to add the per IO cost to the io1 PV cost
+
+						// Add the per IO cost to the PV object for the io1 volume type
+						aws.Pricing[key].PV.CostPerIO = cost
+						} else if strings.Contains(key, "EBS:Volume") {
+							// If volume, we need to get hourly cost and add it to the PV object
+							costFloat, _ := strconv.ParseFloat(cost, 64)
+							hourlyPrice := costFloat / 730
+
+							aws.Pricing[key].PV.Cost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 						}
 					}
+
 					_, err = dec.Token()
 					if err != nil {
 						return err

pkg/costmodel/allocation.go

@@ -17,13 +17,24 @@ import (
 )
 
 const (
-	queryFmtPods                  = `avg(kube_pod_container_status_running{}) by (pod, namespace, cluster_id)[%s:%s]%s`
-	queryFmtRAMBytesAllocated     = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
-	queryFmtRAMRequests           = `avg(avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
-	queryFmtRAMUsage              = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
-	queryFmtCPUCoresAllocated     = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
-	queryFmtCPURequests           = `avg(avg_over_time(kube_pod_container_resource_requests_cpu_cores{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
-	queryFmtCPUUsage              = `avg(rate(container_cpu_usage_seconds_total{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtPods              = `avg(kube_pod_container_status_running{}) by (pod, namespace, cluster_id)[%s:%s]%s`
+	queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMRequests       = `avg(avg_over_time(kube_pod_container_resource_requests_memory_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtRAMUsageAvg       = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtRAMUsageMax       = `max(max_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+	queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtCPURequests       = `avg(avg_over_time(kube_pod_container_resource_requests_cpu_cores{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
+	queryFmtCPUUsageAvg       = `avg(rate(container_cpu_usage_seconds_total{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
+
+	// This query could be written without the recording rule
+	// "kubecost_savings_container_cpu_usage_seconds", but we should
+	// only do that when we're ready to incur the performance tradeoffs
+	// with subqueries which would probably be in the world of hourly
+	// ETL.
+	//
+	// See PromQL subquery documentation for a rate example:
+	// https://prometheus.io/blog/2019/01/28/subquery-support/#examples
+	queryFmtCPUUsageMax           = `max(max_over_time(kubecost_savings_container_cpu_usage_seconds[%s]%s)) by (container_name, pod_name, namespace, instance, cluster_id)`
 	queryFmtGPUsRequested         = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, cluster_id)`
 	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
 	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, cluster_id, instance_type)`
@@ -107,8 +118,11 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, durStr, offStr)
 	resChRAMRequests := ctx.Query(queryRAMRequests)
 
-	queryRAMUsage := fmt.Sprintf(queryFmtRAMUsage, durStr, offStr)
-	resChRAMUsage := ctx.Query(queryRAMUsage)
+	queryRAMUsageAvg := fmt.Sprintf(queryFmtRAMUsageAvg, durStr, offStr)
+	resChRAMUsageAvg := ctx.Query(queryRAMUsageAvg)
+
+	queryRAMUsageMax := fmt.Sprintf(queryFmtRAMUsageMax, durStr, offStr)
+	resChRAMUsageMax := ctx.Query(queryRAMUsageMax)
 
 	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, durStr, offStr)
 	resChCPUCoresAllocated := ctx.Query(queryCPUCoresAllocated)
@@ -116,8 +130,11 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, durStr, offStr)
 	resChCPURequests := ctx.Query(queryCPURequests)
 
-	queryCPUUsage := fmt.Sprintf(queryFmtCPUUsage, durStr, offStr)
-	resChCPUUsage := ctx.Query(queryCPUUsage)
+	queryCPUUsageAvg := fmt.Sprintf(queryFmtCPUUsageAvg, durStr, offStr)
+	resChCPUUsageAvg := ctx.Query(queryCPUUsageAvg)
+
+	queryCPUUsageMax := fmt.Sprintf(queryFmtCPUUsageMax, durStr, offStr)
+	resChCPUUsageMax := ctx.Query(queryCPUUsageMax)
 
 	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, durStr, offStr)
 	resChGPUsRequested := ctx.Query(queryGPUsRequested)
@@ -202,10 +219,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 
 	resCPUCoresAllocated, _ := resChCPUCoresAllocated.Await()
 	resCPURequests, _ := resChCPURequests.Await()
-	resCPUUsage, _ := resChCPUUsage.Await()
+	resCPUUsageAvg, _ := resChCPUUsageAvg.Await()
+	resCPUUsageMax, _ := resChCPUUsageMax.Await()
 	resRAMBytesAllocated, _ := resChRAMBytesAllocated.Await()
 	resRAMRequests, _ := resChRAMRequests.Await()
-	resRAMUsage, _ := resChRAMUsage.Await()
+	resRAMUsageAvg, _ := resChRAMUsageAvg.Await()
+	resRAMUsageMax, _ := resChRAMUsageMax.Await()
 	resGPUsRequested, _ := resChGPUsRequested.Await()
 
 	resNodeCostPerCPUHr, _ := resChNodeCostPerCPUHr.Await()
@@ -252,10 +271,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	// or equal to request.
 	applyCPUCoresAllocated(podMap, resCPUCoresAllocated)
 	applyCPUCoresRequested(podMap, resCPURequests)
-	applyCPUCoresUsed(podMap, resCPUUsage)
+	applyCPUCoresUsedAvg(podMap, resCPUUsageAvg)
+	applyCPUCoresUsedMax(podMap, resCPUUsageMax)
 	applyRAMBytesAllocated(podMap, resRAMBytesAllocated)
 	applyRAMBytesRequested(podMap, resRAMRequests)
-	applyRAMBytesUsed(podMap, resRAMUsage)
+	applyRAMBytesUsedAvg(podMap, resRAMUsageAvg)
+	applyRAMBytesUsedMax(podMap, resRAMUsageMax)
 	applyGPUsRequested(podMap, resGPUsRequested)
 	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB)
 	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB)
@@ -637,11 +658,11 @@ func applyCPUCoresRequested(podMap map[podKey]*Pod, resCPUCoresRequested []*prom
 	}
 }
 
-func applyCPUCoresUsed(podMap map[podKey]*Pod, resCPUCoresUsed []*prom.QueryResult) {
-	for _, res := range resCPUCoresUsed {
+func applyCPUCoresUsedAvg(podMap map[podKey]*Pod, resCPUCoresUsedAvg []*prom.QueryResult) {
+	for _, res := range resCPUCoresUsedAvg {
 		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
 		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage result missing field: %s", err)
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage avg result missing field: %s", err)
 			continue
 		}
 
@@ -652,7 +673,7 @@ func applyCPUCoresUsed(podMap map[podKey]*Pod, resCPUCoresUsed []*prom.QueryResu
 
 		container, err := res.GetString("container_name")
 		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage query result missing 'container': %s", key)
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage avg query result missing 'container': %s", key)
 			continue
 		}
 
@@ -664,6 +685,39 @@ func applyCPUCoresUsed(podMap map[podKey]*Pod, resCPUCoresUsed []*prom.QueryResu
 	}
 }
 
+func applyCPUCoresUsedMax(podMap map[podKey]*Pod, resCPUCoresUsedMax []*prom.QueryResult) {
+	for _, res := range resCPUCoresUsedMax {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage max result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage max query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		if pod.Allocations[container].RawAllocationOnly == nil {
+			pod.Allocations[container].RawAllocationOnly = &kubecost.RawAllocationOnlyData{
+				CPUCoreUsageMax: res.Values[0].Value,
+			}
+		} else {
+			pod.Allocations[container].RawAllocationOnly.CPUCoreUsageMax = res.Values[0].Value
+		}
+	}
+}
+
 func applyRAMBytesAllocated(podMap map[podKey]*Pod, resRAMBytesAllocated []*prom.QueryResult) {
 	for _, res := range resRAMBytesAllocated {
 		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")
@@ -740,11 +794,11 @@ func applyRAMBytesRequested(podMap map[podKey]*Pod, resRAMBytesRequested []*prom
 	}
 }
 
-func applyRAMBytesUsed(podMap map[podKey]*Pod, resRAMBytesUsed []*prom.QueryResult) {
-	for _, res := range resRAMBytesUsed {
+func applyRAMBytesUsedAvg(podMap map[podKey]*Pod, resRAMBytesUsedAvg []*prom.QueryResult) {
+	for _, res := range resRAMBytesUsedAvg {
 		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
 		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage result missing field: %s", err)
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM avg usage result missing field: %s", err)
 			continue
 		}
 
@@ -755,7 +809,7 @@ func applyRAMBytesUsed(podMap map[podKey]*Pod, resRAMBytesUsed []*prom.QueryResu
 
 		container, err := res.GetString("container_name")
 		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage query result missing 'container': %s", key)
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage avg query result missing 'container': %s", key)
 			continue
 		}
 
@@ -767,6 +821,39 @@ func applyRAMBytesUsed(podMap map[podKey]*Pod, resRAMBytesUsed []*prom.QueryResu
 	}
 }
 
+func applyRAMBytesUsedMax(podMap map[podKey]*Pod, resRAMBytesUsedMax []*prom.QueryResult) {
+	for _, res := range resRAMBytesUsedMax {
+		key, err := resultPodKey(res, "cluster_id", "namespace", "pod_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage max result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[key]
+		if !ok {
+			continue
+		}
+
+		container, err := res.GetString("container_name")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage max query result missing 'container': %s", key)
+			continue
+		}
+
+		if _, ok := pod.Allocations[container]; !ok {
+			pod.AppendContainer(container)
+		}
+
+		if pod.Allocations[container].RawAllocationOnly == nil {
+			pod.Allocations[container].RawAllocationOnly = &kubecost.RawAllocationOnlyData{
+				RAMBytesUsageMax: res.Values[0].Value,
+			}
+		} else {
+			pod.Allocations[container].RawAllocationOnly.RAMBytesUsageMax = res.Values[0].Value
+		}
+	}
+}
+
 func applyGPUsRequested(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult) {
 	for _, res := range resGPUsRequested {
 		key, err := resultPodKey(res, "cluster_id", "namespace", "pod")

pkg/env/costmodelenv.go

@@ -72,7 +72,7 @@ const (
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAppVersion() string {
-	return Get(AppVersionEnvVar, "1.77.1")
+	return Get(AppVersionEnvVar, "1.78.0")
 }
 
 // IsEmitNamespaceAnnotationsMetric returns true if cost-model is configured to emit the kube_namespace_annotations metric

pkg/kubecost/allocation.go

@@ -71,8 +71,39 @@ type Allocation struct {
 	RAMCost                float64               `json:"ramCost"`
 	SharedCost             float64               `json:"sharedCost"`
 	ExternalCost           float64               `json:"externalCost"`
-}
 
+	// RawAllocationOnly is a pointer so if it is not present it will be
+	// marshalled as null rather than as an object with Go default values.
+	RawAllocationOnly *RawAllocationOnlyData `json:"rawAllocationOnly"`
+}
+
+// RawAllocationOnlyData is information that only belong in "raw" Allocations,
+// those which have not undergone aggregation, accumulation, or any other form
+// of combination to produce a new Allocation from other Allocations.
+//
+// Max usage data belongs here because computing the overall maximum from two
+// or more Allocations is a non-trivial operation that cannot be defined without
+// maintaining a large amount of state. Consider the following example:
+// _______________________________________________
+//
+// A1 Using 3 CPU    ----      -----     ------
+// A2 Using 2 CPU      ----      -----      ----
+// A3 Using 1 CPU         ---       --
+// _______________________________________________
+//                   Time ---->
+//
+// The logical maximum CPU usage is 5, but this cannot be calculated iteratively,
+// which is how we calculate aggregations and accumulations of Allocations currently.
+// This becomes a problem I could call "maximum sum of overlapping intervals" and is
+// essentially a variant of an interval scheduling algorithm.
+//
+// If we had types to differentiate between regular Allocations and AggregatedAllocations
+// then this type would be unnecessary and its fields would go into the regular Allocation
+// and not in the AggregatedAllocation.
+type RawAllocationOnlyData struct {
+	CPUCoreUsageMax  float64 `json:"cpuCoreUsageMax"`
+	RAMBytesUsageMax float64 `json:"ramByteUsageMax"`
+}
 
 
 // AllocationMatchFunc is a function that can be used to match Allocations by
@@ -126,6 +157,19 @@ func (a *Allocation) Clone() *Allocation {
 		RAMCost:                a.RAMCost,
 		SharedCost:             a.SharedCost,
 		ExternalCost:           a.ExternalCost,
+		RawAllocationOnly:      a.RawAllocationOnly.Clone(),
+	}
+}
+
+// Clone returns a deep copy of the given RawAllocationOnlyData
+func (r *RawAllocationOnlyData) Clone() *RawAllocationOnlyData {
+	if r == nil {
+		return nil
+	}
+
+	return &RawAllocationOnlyData{
+		CPUCoreUsageMax:  r.CPUCoreUsageMax,
+		RAMBytesUsageMax: r.RAMBytesUsageMax,
 	}
 }
 
@@ -190,6 +234,22 @@ func (a *Allocation) Equal(that *Allocation) bool {
 		return false
 	}
 
+	if a.RawAllocationOnly == nil && that.RawAllocationOnly != nil {
+		return false
+	}
+	if a.RawAllocationOnly != nil && that.RawAllocationOnly == nil {
+		return false
+	}
+
+	if a.RawAllocationOnly != nil && that.RawAllocationOnly != nil {
+		if !util.IsApproximately(a.RawAllocationOnly.CPUCoreUsageMax, that.RawAllocationOnly.CPUCoreUsageMax) {
+			return false
+		}
+		if !util.IsApproximately(a.RawAllocationOnly.RAMBytesUsageMax, that.RawAllocationOnly.RAMBytesUsageMax) {
+			return false
+		}
+	}
+
 	return true
 }
 
@@ -295,7 +355,8 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "sharedCost", a.SharedCost, ",")
 	jsonEncodeFloat64(buffer, "externalCost", a.ExternalCost, ",")
 	jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), ",")
-	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), "")
+	jsonEncodeFloat64(buffer, "totalEfficiency", a.TotalEfficiency(), ",")
+	jsonEncode(buffer, "rawAllocationOnly", a.RawAllocationOnly, "")
 	buffer.WriteString("}")
 	return buffer.Bytes(), nil
 }
@@ -441,6 +502,10 @@ func (a *Allocation) add(that *Allocation) {
 	a.LoadBalancerCost += that.LoadBalancerCost
 	a.SharedCost += that.SharedCost
 	a.ExternalCost += that.ExternalCost
+
+	// Any data that is in a "raw allocation only" is not valid in any
+	// sort of cumulative Allocation (like one that is added).
+	a.RawAllocationOnly = nil
 }
 
 // AllocationSet stores a set of Allocations, each with a unique name, that share

pkg/kubecost/allocation_test.go

@@ -52,6 +52,10 @@ func NewUnitAllocation(name string, start time.Time, resolution time.Duration, p
 		RAMCost:                1,
 		RAMBytesRequestAverage: 1,
 		RAMBytesUsageAverage:   1,
+		RawAllocationOnly: &RawAllocationOnlyData{
+			CPUCoreUsageMax:  1,
+			RAMBytesUsageMax: 1,
+		},
 	}
 
 	// If idle allocation, remove non-idle costs, but maintain total cost
@@ -118,6 +122,7 @@ func TestAllocation_Add(t *testing.T) {
 		RAMCost:                8.0 * hrs1 * ramPrice,
 		SharedCost:             2.00,
 		ExternalCost:           1.00,
+		RawAllocationOnly:      &RawAllocationOnlyData{},
 	}
 	a1b := a1.Clone()
 
@@ -144,6 +149,7 @@ func TestAllocation_Add(t *testing.T) {
 		LoadBalancerCost:       0.05,
 		SharedCost:             0.00,
 		ExternalCost:           1.00,
+		RawAllocationOnly:      &RawAllocationOnlyData{},
 	}
 	a2b := a2.Clone()
 
@@ -239,6 +245,10 @@ func TestAllocation_Add(t *testing.T) {
 	if !util.IsApproximately(1.6493506, act.TotalEfficiency()) {
 		t.Fatalf("Allocation.Add: expected %f; actual %f", 1.6493506, act.TotalEfficiency())
 	}
+
+	if act.RawAllocationOnly != nil {
+		t.Errorf("Allocation.Add: Raw only data must be nil after an add")
+	}
 }
 
 func TestAllocation_Share(t *testing.T) {
@@ -427,6 +437,7 @@ func TestAllocation_MarshalJSON(t *testing.T) {
 		RAMCost:                8.0 * hrs * ramPrice,
 		SharedCost:             2.00,
 		ExternalCost:           1.00,
+		RawAllocationOnly:      &RawAllocationOnlyData{},
 	}
 
 	data, err := json.Marshal(before)

pkg/kubecost/bingen.go

@@ -24,5 +24,6 @@ package kubecost
 // @bingen:generate:AllocationProperty
 // @bingen:generate:AllocationLabels
 // @bingen:generate:AllocationAnnotations
+// @bingen:generate:RawAllocationOnlyData
 
-//go:generate bingen -package=kubecost -version=10 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=11 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/kubecost_codecs.go

@@ -25,7 +25,7 @@ const (
 	GeneratorPackageName string = "kubecost"
 
 	// CodecVersion is the version passed into the generator
-	CodecVersion uint8 = 10
+	CodecVersion uint8 = 11
 )
 
 //--------------------------------------------------------------------------
@@ -35,23 +35,24 @@ const (
 // Generated type map for resolving interface implementations to
 // to concrete types
 var typeMap map[string]reflect.Type = map[string]reflect.Type{
-	"Allocation":           reflect.TypeOf((*Allocation)(nil)).Elem(),
-	"AllocationProperties": reflect.TypeOf((*AllocationProperties)(nil)).Elem(),
-	"AllocationSet":        reflect.TypeOf((*AllocationSet)(nil)).Elem(),
-	"AllocationSetRange":   reflect.TypeOf((*AllocationSetRange)(nil)).Elem(),
-	"Any":                  reflect.TypeOf((*Any)(nil)).Elem(),
-	"AssetProperties":      reflect.TypeOf((*AssetProperties)(nil)).Elem(),
-	"AssetSet":             reflect.TypeOf((*AssetSet)(nil)).Elem(),
-	"AssetSetRange":        reflect.TypeOf((*AssetSetRange)(nil)).Elem(),
-	"Breakdown":            reflect.TypeOf((*Breakdown)(nil)).Elem(),
-	"Cloud":                reflect.TypeOf((*Cloud)(nil)).Elem(),
-	"ClusterManagement":    reflect.TypeOf((*ClusterManagement)(nil)).Elem(),
-	"Disk":                 reflect.TypeOf((*Disk)(nil)).Elem(),
-	"LoadBalancer":         reflect.TypeOf((*LoadBalancer)(nil)).Elem(),
-	"Network":              reflect.TypeOf((*Network)(nil)).Elem(),
-	"Node":                 reflect.TypeOf((*Node)(nil)).Elem(),
-	"SharedAsset":          reflect.TypeOf((*SharedAsset)(nil)).Elem(),
-	"Window":               reflect.TypeOf((*Window)(nil)).Elem(),
+	"Allocation":            reflect.TypeOf((*Allocation)(nil)).Elem(),
+	"AllocationProperties":  reflect.TypeOf((*AllocationProperties)(nil)).Elem(),
+	"AllocationSet":         reflect.TypeOf((*AllocationSet)(nil)).Elem(),
+	"AllocationSetRange":    reflect.TypeOf((*AllocationSetRange)(nil)).Elem(),
+	"Any":                   reflect.TypeOf((*Any)(nil)).Elem(),
+	"AssetProperties":       reflect.TypeOf((*AssetProperties)(nil)).Elem(),
+	"AssetSet":              reflect.TypeOf((*AssetSet)(nil)).Elem(),
+	"AssetSetRange":         reflect.TypeOf((*AssetSetRange)(nil)).Elem(),
+	"Breakdown":             reflect.TypeOf((*Breakdown)(nil)).Elem(),
+	"Cloud":                 reflect.TypeOf((*Cloud)(nil)).Elem(),
+	"ClusterManagement":     reflect.TypeOf((*ClusterManagement)(nil)).Elem(),
+	"Disk":                  reflect.TypeOf((*Disk)(nil)).Elem(),
+	"LoadBalancer":          reflect.TypeOf((*LoadBalancer)(nil)).Elem(),
+	"Network":               reflect.TypeOf((*Network)(nil)).Elem(),
+	"Node":                  reflect.TypeOf((*Node)(nil)).Elem(),
+	"RawAllocationOnlyData": reflect.TypeOf((*RawAllocationOnlyData)(nil)).Elem(),
+	"SharedAsset":           reflect.TypeOf((*SharedAsset)(nil)).Elem(),
+	"Window":                reflect.TypeOf((*Window)(nil)).Elem(),
 }
 
 //--------------------------------------------------------------------------
@@ -175,6 +176,21 @@ func (target *Allocation) MarshalBinary() (data []byte, err error) {
 	buff.WriteFloat64(target.RAMCost)                // write float64
 	buff.WriteFloat64(target.SharedCost)             // write float64
 	buff.WriteFloat64(target.ExternalCost)           // write float64
+	if target.RawAllocationOnly == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		// --- [begin][write][struct](RawAllocationOnlyData) ---
+		e, errE := target.RawAllocationOnly.MarshalBinary()
+		if errE != nil {
+			return nil, errE
+		}
+		buff.WriteInt(len(e))
+		buff.WriteBytes(e)
+		// --- [end][write][struct](RawAllocationOnlyData) ---
+
+	}
 	return buff.Bytes(), nil
 }
 
@@ -301,6 +317,21 @@ func (target *Allocation) UnmarshalBinary(data []byte) (err error) {
 	gg := buff.ReadFloat64() // read float64
 	target.ExternalCost = gg
 
+	if buff.ReadUInt8() == uint8(0) {
+		target.RawAllocationOnly = nil
+	} else {
+		// --- [begin][read][struct](RawAllocationOnlyData) ---
+		hh := &RawAllocationOnlyData{}
+		kk := buff.ReadInt()     // byte array length
+		ll := buff.ReadBytes(kk) // byte array
+		errE := hh.UnmarshalBinary(ll)
+		if errE != nil {
+			return errE
+		}
+		target.RawAllocationOnly = hh
+		// --- [end][read][struct](RawAllocationOnlyData) ---
+
+	}
 	return nil
 }
 
@@ -2827,6 +2858,67 @@ func (target *Node) UnmarshalBinary(data []byte) (err error) {
 	return nil
 }
 
+//--------------------------------------------------------------------------
+//  RawAllocationOnlyData
+//--------------------------------------------------------------------------
+
+// MarshalBinary serializes the internal properties of this RawAllocationOnlyData instance
+// into a byte array
+func (target *RawAllocationOnlyData) MarshalBinary() (data []byte, err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBuffer()
+	buff.WriteUInt8(CodecVersion) // version
+
+	buff.WriteFloat64(target.CPUCoreUsageMax)  // write float64
+	buff.WriteFloat64(target.RAMBytesUsageMax) // write float64
+	return buff.Bytes(), nil
+}
+
+// UnmarshalBinary uses the data passed byte array to set all the internal properties of
+// the RawAllocationOnlyData type
+func (target *RawAllocationOnlyData) UnmarshalBinary(data []byte) (err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := util.NewBufferFromBytes(data)
+
+	// Codec Version Check
+	version := buff.ReadUInt8()
+	if version != CodecVersion {
+		return fmt.Errorf("Invalid Version Unmarshaling RawAllocationOnlyData. Expected %d, got %d", CodecVersion, version)
+	}
+
+	a := buff.ReadFloat64() // read float64
+	target.CPUCoreUsageMax = a
+
+	b := buff.ReadFloat64() // read float64
+	target.RAMBytesUsageMax = b
+
+	return nil
+}
+
 //--------------------------------------------------------------------------
 //  SharedAsset
 //--------------------------------------------------------------------------