summary allocation open source lift

pkg/costmodel/aggregation.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/kubecost/cost-model/pkg/util/httputil"
@@ -2125,6 +2126,205 @@ func ParseAggregationProperties(qp httputil.QueryParams, key string) ([]string,
 	return aggregateBy, nil
 }
 
+type SummaryAllocation struct {
+	Name                   string                         `json:"name"`
+	Properties             *kubecost.AllocationProperties `json:"-"`
+	Start                  time.Time                      `json:"start"`
+	End                    time.Time                      `json:"end"`
+	CPUCoreRequestAverage  float64                        `json:"cpuCoreRequestAverage"`
+	CPUCoreUsageAverage    float64                        `json:"cpuCoreUsageAverage"`
+	CPUCost                float64                        `json:"cpuCost"`
+	GPUCost                float64                        `json:"gpuCost"`
+	NetworkCost            float64                        `json:"networkCost"`
+	LoadBalancerCost       float64                        `json:"loadBalancerCost"`
+	PVCost                 float64                        `json:"pvCost"`
+	RAMBytesRequestAverage float64                        `json:"ramByteRequestAverage"`
+	RAMBytesUsageAverage   float64                        `json:"ramByteUsageAverage"`
+	RAMCost                float64                        `json:"ramCost"`
+	SharedCost             float64                        `json:"sharedCost"`
+	ExternalCost           float64                        `json:"externalCost"`
+	Share                  bool                           `json:"-"`
+}
+
+func NewSummaryAllocation(alloc *kubecost.Allocation) *SummaryAllocation {
+	if alloc == nil {
+		return nil
+	}
+
+	return &SummaryAllocation{
+		Name:                   alloc.Name,
+		Properties:             alloc.Properties.Clone(),
+		Start:                  alloc.Start,
+		End:                    alloc.End,
+		CPUCoreRequestAverage:  alloc.CPUCoreRequestAverage,
+		CPUCoreUsageAverage:    alloc.CPUCoreUsageAverage,
+		CPUCost:                alloc.CPUCost + alloc.CPUCostAdjustment,
+		GPUCost:                alloc.GPUCost + alloc.GPUCostAdjustment,
+		NetworkCost:            alloc.NetworkCost + alloc.NetworkCostAdjustment,
+		LoadBalancerCost:       alloc.LoadBalancerCost + alloc.LoadBalancerCostAdjustment,
+		PVCost:                 alloc.PVCost() + alloc.PVCostAdjustment,
+		RAMBytesRequestAverage: alloc.RAMBytesRequestAverage,
+		RAMBytesUsageAverage:   alloc.RAMBytesUsageAverage,
+		RAMCost:                alloc.RAMCost + alloc.RAMCostAdjustment,
+		SharedCost:             alloc.SharedCost,
+		ExternalCost:           alloc.ExternalCost,
+	}
+}
+
+type SummaryAllocationSet struct {
+	sync.RWMutex
+	externalKeys       map[string]bool
+	idleKeys           map[string]bool
+	SummaryAllocations map[string]*SummaryAllocation `json:"allocations"`
+	Window             kubecost.Window               `json:"window"`
+}
+
+func NewSummaryAllocationSet(as *kubecost.AllocationSet) *SummaryAllocationSet {
+	if as == nil {
+		return nil
+	}
+	asMap := as.Map()
+	var sasMap map[string]*SummaryAllocation
+
+	sasMap = make(map[string]*SummaryAllocation, len(asMap))
+
+	sas := &SummaryAllocationSet{
+		SummaryAllocations: sasMap,
+		Window:             as.Window.Clone(),
+	}
+	sas.externalKeys = map[string]bool{}
+	sas.idleKeys = map[string]bool{}
+
+	for _, alloc := range as.Map() {
+		sa := NewSummaryAllocation(alloc)
+		sas.SummaryAllocations[sa.Name] = sa
+
+		if alloc.IsExternal() {
+			sas.externalKeys[sa.Name] = true
+		}
+
+		if alloc.IsIdle() {
+			sas.idleKeys[sa.Name] = true
+		}
+
+	}
+
+	return sas
+}
+
+type SummaryAllocationSetRange struct {
+	sync.RWMutex
+	Step                  time.Duration           `json:"step"`
+	SummaryAllocationSets []*SummaryAllocationSet `json:"sets"`
+	Window                kubecost.Window         `json:"window"`
+}
+
+func NewSummaryAllocationSetRange(sass ...*SummaryAllocationSet) *SummaryAllocationSetRange {
+	var step time.Duration
+	window := kubecost.NewWindow(nil, nil)
+
+	for _, sas := range sass {
+		if window.Start() == nil || (sas.Window.Start() != nil && sas.Window.Start().Before(*window.Start())) {
+			window.Expand(sas.Window)
+		}
+		if window.End() == nil || (sas.Window.End() != nil && sas.Window.End().After(*window.End())) {
+			window.Expand(sas.Window)
+		}
+		if step == 0 {
+			step = sas.Window.Duration()
+		} else if step != sas.Window.Duration() {
+			log.Warningf("instantiating range with step %s using set of step %s is illegal", step, sas.Window.Duration())
+		}
+	}
+
+	return &SummaryAllocationSetRange{
+		Step:                  step,
+		SummaryAllocationSets: sass,
+		Window:                window,
+	}
+}
+
+func (a *Accesses) ComputeAllocationHandlerSummary(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+
+	qp := httputil.NewQueryParams(r.URL.Query())
+
+	// Window is a required field describing the window of time over which to
+	// compute allocation data.
+	window, err := kubecost.ParseWindowWithOffset(qp.Get("window", ""), env.GetParsedUTCOffset())
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Invalid 'window' parameter: %s", err), http.StatusBadRequest)
+	}
+
+	// Step is an optional parameter that defines the duration per-set, i.e.
+	// the window for an AllocationSet, of the AllocationSetRange to be
+	// computed. Defaults to the window size, making one set.
+	step := qp.GetDuration("step", window.Duration())
+
+	// Resolution is an optional parameter, defaulting to the configured ETL
+	// resolution.
+	resolution := qp.GetDuration("resolution", env.GetETLResolution())
+
+	// Aggregation is a required comma-separated list of fields by which to
+	// aggregate results. Some fields allow a sub-field, which is distinguished
+	// with a colon; e.g. "label:app".
+	// Examples: "namespace", "namespace,label:app"
+	aggregateBy, err := ParseAggregationProperties(qp, "aggregate")
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Invalid 'aggregate' parameter: %s", err), http.StatusBadRequest)
+	}
+
+	// Accumulate is an optional parameter, defaulting to false, which if true
+	// sums each Set in the Range, producing one Set.
+	accumulate := qp.GetBool("accumulate", false)
+
+	// Query for AllocationSets in increments of the given step duration,
+	// appending each to the AllocationSetRange.
+	asr := kubecost.NewAllocationSetRange()
+	stepStart := *window.Start()
+	for window.End().After(stepStart) {
+		stepEnd := stepStart.Add(step)
+		stepWindow := kubecost.NewWindow(&stepStart, &stepEnd)
+
+		as, err := a.Model.ComputeAllocation(*stepWindow.Start(), *stepWindow.End(), resolution)
+		if err != nil {
+			WriteError(w, InternalServerError(err.Error()))
+			return
+		}
+		asr.Append(as)
+
+		stepStart = stepEnd
+	}
+
+	// Aggregate, if requested
+	if len(aggregateBy) > 0 {
+		err = asr.AggregateBy(aggregateBy, nil)
+		if err != nil {
+			WriteError(w, InternalServerError(err.Error()))
+			return
+		}
+	}
+
+	// Accumulate, if requested
+	if accumulate {
+		as, err := asr.Accumulate()
+		if err != nil {
+			WriteError(w, InternalServerError(err.Error()))
+			return
+		}
+		asr = kubecost.NewAllocationSetRange(as)
+	}
+
+	sasl := []*SummaryAllocationSet{}
+	for _, as := range asr.Slice() {
+		sas := NewSummaryAllocationSet(as)
+		sasl = append(sasl, sas)
+	}
+	sasr := NewSummaryAllocationSetRange(sasl...)
+
+	w.Write(WrapData(sasr, nil))
+}
+
 // ComputeAllocationHandler computes an AllocationSetRange from the CostModel.
 func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")

pkg/costmodel/router.go

@@ -1626,6 +1626,7 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 	a.Router.GET("/costDataModelRange", a.CostDataModelRange)
 	a.Router.GET("/aggregatedCostModel", a.AggregateCostModelHandler)
 	a.Router.GET("/allocation/compute", a.ComputeAllocationHandler)
+	a.Router.GET("/allocation/compute/summary", a.ComputeAllocationHandlerSummary)
 	a.Router.GET("/outOfClusterCosts", a.OutOfClusterCostsWithCache)
 	a.Router.GET("/allNodePricing", a.GetAllNodePricing)
 	a.Router.POST("/refreshPricing", a.RefreshPricingData)