Merge pull request #2204 from opencost/v1.106-pick-sharingfilters

Convert share funcs to filters v2.1

pkg/costmodel/aggregation.go

@@ -2215,7 +2215,7 @@ func (a *Accesses) ComputeAllocationHandlerSummary(w http.ResponseWriter, r *htt
 
 	sasl := []*kubecost.SummaryAllocationSet{}
 	for _, as := range asr.Slice() {
-		sas := kubecost.NewSummaryAllocationSet(as, nil, []kubecost.AllocationMatchFunc{}, false, false)
+		sas := kubecost.NewSummaryAllocationSet(as, nil, nil, false, false)
 		sasl = append(sasl, sas)
 	}
 	sasr := kubecost.NewSummaryAllocationSetRange(sasl...)

pkg/filter21/ast/walker.go

@@ -2,6 +2,7 @@ package ast
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/opencost/opencost/pkg/filter21/util"
@@ -367,3 +368,42 @@ func indent(depth int) string {
 	}
 	return strings.Repeat("  ", depth)
 }
+
+func Fields(filter FilterNode) []Field {
+	fields := map[Field]bool{}
+
+	PreOrderTraversal(filter, func(fn FilterNode, state TraversalState) {
+		if fn == nil {
+			return
+		}
+		switch n := fn.(type) {
+		case *EqualOp:
+			if n.Left.Field != nil {
+				fields[*n.Left.Field] = true
+			}
+		case *ContainsOp:
+			if n.Left.Field != nil {
+				fields[*n.Left.Field] = true
+			}
+		case *ContainsPrefixOp:
+			if n.Left.Field != nil {
+				fields[*n.Left.Field] = true
+			}
+		case *ContainsSuffixOp:
+			if n.Left.Field != nil {
+				fields[*n.Left.Field] = true
+			}
+		}
+	})
+
+	response := make([]Field, 0, len(fields))
+	for field := range fields {
+		response = append(response, field)
+	}
+
+	sort.Slice(response, func(i, j int) bool {
+		return response[i].Name < response[j].Name
+	})
+
+	return response
+}

pkg/filter21/ast/walker_test.go

@@ -2,6 +2,8 @@ package ast
 
 import (
 	"fmt"
+	"reflect"
+	"testing"
 )
 
 func ExampleTransformLeaves() {
@@ -50,3 +52,101 @@ func ExampleTransformLeaves() {
 	//   }
 	// }
 }
+
+func TestFields(t *testing.T) {
+	type testCase struct {
+		name   string
+		filter FilterNode
+		exp    []Field
+	}
+
+	fieldNamespace := *NewField("namespace")
+	fieldCluster := *NewField("cluster")
+	fieldControllerKind := *NewField("controllerKind")
+
+	testCases := []testCase{
+		{
+			name:   ``,
+			filter: &VoidOp{},
+			exp:    []Field{},
+		},
+		{
+			name: `namespace:"kubecost"`,
+			filter: &EqualOp{
+				Left: Identifier{
+					Field: NewField("namespace"),
+					Key:   "",
+				},
+				Right: "kubecost",
+			},
+			exp: []Field{fieldNamespace},
+		},
+		{
+			name: `namespace: "kubecost" | cluster:"cluster-one" | controllerKind:"deployment"`,
+			filter: &OrOp{
+				Operands: []FilterNode{
+					&EqualOp{
+						Left: Identifier{
+							Field: NewField("namespace"),
+							Key:   "",
+						},
+						Right: "kubecost",
+					},
+					&EqualOp{
+						Left: Identifier{
+							Field: NewField("cluster"),
+							Key:   "",
+						},
+						Right: "cluster-one",
+					},
+					&EqualOp{
+						Left: Identifier{
+							Field: NewField("controllerKind"),
+							Key:   "",
+						},
+						Right: "deployment",
+					},
+				},
+			},
+			exp: []Field{fieldCluster, fieldControllerKind, fieldNamespace},
+		},
+		{
+			name: `namespace: "kubecost" | namespace:"kube-system" | namespace:"default"`,
+			filter: &OrOp{
+				Operands: []FilterNode{
+					&EqualOp{
+						Left: Identifier{
+							Field: NewField("namespace"),
+							Key:   "",
+						},
+						Right: "kubecost",
+					},
+					&EqualOp{
+						Left: Identifier{
+							Field: NewField("namespace"),
+							Key:   "",
+						},
+						Right: "kube-system",
+					},
+					&EqualOp{
+						Left: Identifier{
+							Field: NewField("namespace"),
+							Key:   "",
+						},
+						Right: "default",
+					},
+				},
+			},
+			exp: []Field{fieldNamespace},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			act := Fields(tc.filter)
+			if !reflect.DeepEqual(tc.exp, act) {
+				t.Errorf("fields do not match; expected %v; got %v", tc.exp, act)
+			}
+		})
+	}
+}

pkg/kubecost/allocation.go

@@ -1370,7 +1370,7 @@ type AllocationAggregationOptions struct {
 	MergeUnallocated                      bool
 	Reconcile                             bool
 	ReconcileNetwork                      bool
-	ShareFuncs                            []AllocationMatchFunc
+	Share                                 filter21.Filter
 	SharedNamespaces                      []string
 	SharedLabels                          map[string][]string
 	ShareIdle                             string
@@ -1470,6 +1470,16 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		return fmt.Errorf("unexpected nil filter")
 	}
 
+	var sharer AllocationMatcher
+	if options.Share != nil {
+		compiler := NewAllocationMatchCompiler(options.LabelConfig)
+		var err error
+		sharer, err = compiler.Compile(options.Share)
+		if err != nil {
+			return fmt.Errorf("compiling sharer '%s': %w", ast.ToPreOrderShortString(options.Filter), err)
+		}
+	}
+
 	var allocatedTotalsMap map[string]map[string]float64
 
 	// If aggregateBy is nil, we don't aggregate anything. On the other hand,
@@ -1477,7 +1487,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	// generateKey for why that makes sense.
 	shouldAggregate := aggregateBy != nil
 	shouldFilter := !isFilterEmpty(filter)
-	shouldShare := len(options.SharedHourlyCosts) > 0 || len(options.ShareFuncs) > 0
+	shouldShare := len(options.SharedHourlyCosts) > 0 || sharer != nil
 	if !shouldAggregate && !shouldFilter && !shouldShare && options.ShareIdle == ShareNone && !options.IncludeProportionalAssetResourceCosts {
 		// There is nothing for AggregateBy to do, so simply return nil
 		// before returning, set aggregated metadata inclusion in properties
@@ -1559,13 +1569,11 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		// Shared allocations must be identified and separated prior to
 		// aggregation and filtering. That is, if any of the ShareFuncs return
 		// true for the allocation, then move it to shareSet.
-		for _, sf := range options.ShareFuncs {
-			if sf(alloc) {
-				delete(as.IdleKeys, alloc.Name)
-				delete(as.Allocations, alloc.Name)
-				shareSet.Insert(alloc)
-				break
-			}
+		if sharer != nil && sharer.Matches(alloc) {
+			delete(as.IdleKeys, alloc.Name)
+			delete(as.Allocations, alloc.Name)
+			shareSet.Insert(alloc)
+			continue
 		}
 	}
 
@@ -2390,7 +2398,7 @@ func (a *Allocation) determineSharingName(options *AllocationAggregationOptions)
 
 	// grab SharedLabels keys and sort them, to keep this function deterministic
 	var labelKeys []string
-	for labelKey, _ := range options.SharedLabels {
+	for labelKey := range options.SharedLabels {
 		labelKeys = append(labelKeys, labelKey)
 	}
 	slices.Sort(labelKeys)

pkg/kubecost/allocation_test.go

@@ -10,7 +10,9 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	filter21 "github.com/opencost/opencost/pkg/filter21"
+	"github.com/opencost/opencost/pkg/filter21/allocation"
 	afilter "github.com/opencost/opencost/pkg/filter21/allocation"
+	"github.com/opencost/opencost/pkg/filter21/ast"
 	"github.com/opencost/opencost/pkg/filter21/ops"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/util"
@@ -718,25 +720,27 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 	idleTotalCost := 30.0
 	sharedOverheadHourlyCost := 7.0
 
-	// Match Functions
-	isNamespace3 := func(a *Allocation) bool {
-		ns := a.Properties.Namespace
-		return ns == "namespace3"
-	}
+	// Match filters
 
-	isApp1 := func(a *Allocation) bool {
-		ls := a.Properties.Labels
-		if app, ok := ls["app"]; ok && app == "app1" {
-			return true
+	// This is ugly, but required because cannot import filterutil due to import cycle
+	namespaceEquals := func(ns string) *ast.EqualOp {
+		return &ast.EqualOp{
+			Left: ast.Identifier{
+				Field: ast.NewField(allocation.FieldNamespace),
+				Key:   "",
+			},
+			Right: ns,
 		}
-		return false
 	}
 
-	// Filters
-	isNamespace := func(matchNamespace string) func(*Allocation) bool {
-		return func(a *Allocation) bool {
-			namespace := a.Properties.Namespace
-			return namespace == matchNamespace
+	// This is ugly, but required because cannot import filterutil due to import cycle
+	labelEquals := func(name, value string) *ast.EqualOp {
+		return &ast.EqualOp{
+			Left: ast.Identifier{
+				Field: ast.NewField(allocation.FieldLabel),
+				Key:   name,
+			},
+			Right: value,
 		}
 	}
 
@@ -1216,7 +1220,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			start: start,
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
-				ShareFuncs: []AllocationMatchFunc{isNamespace3},
+				Share:      namespaceEquals("namespace3"),
 				ShareSplit: ShareEven,
 			},
 			numResults: numNamespaces,
@@ -1238,7 +1242,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			start: start,
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
-				ShareFuncs:                            []AllocationMatchFunc{isNamespace3},
+				Share:                                 namespaceEquals("namespace3"),
 				ShareSplit:                            ShareWeighted,
 				IncludeProportionalAssetResourceCosts: true,
 			},
@@ -1298,7 +1302,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			start: start,
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
-				ShareFuncs: []AllocationMatchFunc{isApp1},
+				Share:      labelEquals("app", "app1"),
 				ShareSplit: ShareEven,
 			},
 			numResults: numNamespaces + numIdle,
@@ -1486,7 +1490,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
 				Filter:     mustParseFilter(`namespace:"namespace2"`),
-				ShareFuncs: []AllocationMatchFunc{isNamespace("namespace1")},
+				Share:      namespaceEquals("namespace1"),
 				ShareSplit: ShareWeighted,
 			},
 			numResults: 1 + numIdle,
@@ -1505,7 +1509,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
 				Filter:     mustParseFilter(`namespace:"namespace2"`),
-				ShareFuncs: []AllocationMatchFunc{isNamespace("namespace1")},
+				Share:      namespaceEquals("namespace1"),
 				ShareSplit: ShareWeighted,
 				ShareIdle:  ShareWeighted,
 			},
@@ -1556,7 +1560,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			start: start,
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
-				ShareFuncs: []AllocationMatchFunc{isNamespace("namespace1")},
+				Share:      namespaceEquals("namespace1"),
 				ShareSplit: ShareWeighted,
 				ShareIdle:  ShareWeighted,
 			},
@@ -1611,7 +1615,7 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
 				Filter:     mustParseFilter(`namespace:"namespace2"`),
-				ShareFuncs: []AllocationMatchFunc{isNamespace("namespace1")},
+				Share:      namespaceEquals("namespace1"),
 				ShareSplit: ShareWeighted,
 				ShareIdle:  ShareWeighted,
 			},
@@ -1888,6 +1892,9 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 	}
 
 	for name, testcase := range cases {
+		if name != "4a" {
+			continue
+		}
 		t.Run(name, func(t *testing.T) {
 			if testcase.aggOpts != nil && testcase.aggOpts.IdleByNode {
 				as = GenerateMockAllocationSetNodeIdle(testcase.start)
@@ -1895,6 +1902,12 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 				as = GenerateMockAllocationSetClusterIdle(testcase.start)
 			}
 			err = as.AggregateBy(testcase.aggBy, testcase.aggOpts)
+
+			log.Infof("RESULTS")
+			for name, alloc := range as.Allocations {
+				log.Infof("  %s = %f", name, alloc.TotalCost())
+			}
+
 			assertAllocationSetTotals(t, as, name, err, testcase.numResults, testcase.totalCost)
 			assertAllocationTotals(t, as, name, testcase.results)
 			assertParcResults(t, as, name, testcase.expectedParcResults)
@@ -1947,14 +1960,15 @@ func TestAllocationSet_AggregateBy_SharedCostBreakdown(t *testing.T) {
 	end := time.Now().UTC().Truncate(day)
 	start := end.Add(-day)
 
-	isNamespace1 := func(a *Allocation) bool {
-		ns := a.Properties.Namespace
-		return ns == "namespace1"
-	}
-
-	isNamespace3 := func(a *Allocation) bool {
-		ns := a.Properties.Namespace
-		return ns == "namespace3"
+	// This is ugly, but required because cannot import filterutil due to import cycle
+	namespaceEquals := func(ns string) *ast.EqualOp {
+		return &ast.EqualOp{
+			Left: ast.Identifier{
+				Field: ast.NewField(allocation.FieldNamespace),
+				Key:   "",
+			},
+			Right: ns,
+		}
 	}
 
 	cases := map[string]struct {
@@ -1974,9 +1988,7 @@ func TestAllocationSet_AggregateBy_SharedCostBreakdown(t *testing.T) {
 			start: start,
 			aggBy: []string{"namespace"},
 			aggOpts: &AllocationAggregationOptions{
-				ShareFuncs: []AllocationMatchFunc{
-					isNamespace1,
-				},
+				Share:                      namespaceEquals("namespace1"),
 				IncludeSharedCostBreakdown: true,
 			},
 		},
@@ -1984,9 +1996,7 @@ func TestAllocationSet_AggregateBy_SharedCostBreakdown(t *testing.T) {
 			start: start,
 			aggBy: []string{"namespace"},
 			aggOpts: &AllocationAggregationOptions{
-				ShareFuncs: []AllocationMatchFunc{
-					isNamespace3,
-				},
+				Share:                      namespaceEquals("namespace3"),
 				IncludeSharedCostBreakdown: true,
 			},
 		},

pkg/kubecost/summaryallocation.go

@@ -434,7 +434,7 @@ type SummaryAllocationSet struct {
 // This filter is an AllocationMatcher, not an AST, because at this point we
 // already have the data and want to make sure that the filter has already
 // gone through a compile step to deal with things like aliases.
-func NewSummaryAllocationSet(as *AllocationSet, filter AllocationMatcher, kfs []AllocationMatchFunc, reconcile, reconcileNetwork bool) *SummaryAllocationSet {
+func NewSummaryAllocationSet(as *AllocationSet, filter, keep AllocationMatcher, reconcile, reconcileNetwork bool) *SummaryAllocationSet {
 	if as == nil {
 		return nil
 	}
@@ -442,7 +442,7 @@ func NewSummaryAllocationSet(as *AllocationSet, filter AllocationMatcher, kfs []
 	// If we can know the exact size of the map, use it. If filters or sharing
 	// functions are present, we can't know the size, so we make a default map.
 	var sasMap map[string]*SummaryAllocation
-	if filter == nil && len(kfs) == 0 {
+	if filter == nil {
 		// No filters, so make the map of summary allocations exactly the size
 		// of the origin allocation set.
 		sasMap = make(map[string]*SummaryAllocation, len(as.Allocations))
@@ -459,14 +459,7 @@ func NewSummaryAllocationSet(as *AllocationSet, filter AllocationMatcher, kfs []
 	for _, alloc := range as.Allocations {
 		// First, detect if the allocation should be kept. If so, mark it as
 		// such, insert it, and continue.
-		shouldKeep := false
-		for _, kf := range kfs {
-			if kf(alloc) {
-				shouldKeep = true
-				break
-			}
-		}
-		if shouldKeep {
+		if keep != nil && keep.Matches(alloc) {
 			sa := NewSummaryAllocation(alloc, reconcile, reconcileNetwork)
 			sa.Share = true
 			sas.Insert(sa)
@@ -630,7 +623,7 @@ func (sas *SummaryAllocationSet) AggregateBy(aggregateBy []string, options *Allo
 	// an empty slice implies that we should aggregate everything. (See
 	// generateKey for why that makes sense.)
 	shouldAggregate := aggregateBy != nil
-	shouldKeep := len(options.SharedHourlyCosts) > 0 || len(options.ShareFuncs) > 0
+	shouldKeep := len(options.SharedHourlyCosts) > 0 || options.Share != nil
 	if !shouldAggregate && !shouldKeep {
 		return nil
 	}

pkg/util/filterutil/filterutil.go

@@ -355,6 +355,20 @@ func AllocationFilterFromParamsV1(
 	return andFilter
 }
 
+func AllocationSharerFromParamsV1(params AllocationFilterV1) filter.Filter {
+	var filterOps []ast.FilterNode
+
+	if len(params.Namespaces) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(params.Namespaces, afilter.FieldNamespace))
+	}
+
+	if len(params.Labels) > 0 {
+		filterOps = push(filterOps, filterV1DoubleValueFromList(params.Labels, afilter.FieldLabel))
+	}
+
+	return opsToAnd(filterOps)
+}
+
 func AssetFilterFromParamsV1(
 	qp mapper.PrimitiveMapReader,
 	clusterMap clusters.ClusterMap,