Merge pull request #2629 from nik-kc/nik/plugin_aggs_filters

pkg/customcost/matcher.go

@@ -0,0 +1,64 @@
+package customcost
+
+import (
+	"fmt"
+
+	"github.com/opencost/opencost/core/pkg/filter/ast"
+	"github.com/opencost/opencost/core/pkg/filter/matcher"
+	"github.com/opencost/opencost/core/pkg/filter/transform"
+)
+
+func NewCustomCostMatchCompiler() *matcher.MatchCompiler[*CustomCost] {
+	passes := []transform.CompilerPass{
+		transform.UnallocatedReplacementPass(),
+	}
+
+	return matcher.NewMatchCompiler(
+		customCostFieldMap,
+		customCostSliceFieldMap,
+		customCostMapFieldMap,
+		passes...,
+	)
+}
+
+// Maps fields from a custom cost to a string value based on an identifier
+func customCostFieldMap(cc *CustomCost, identifier ast.Identifier) (string, error) {
+	if cc == nil {
+		return "", fmt.Errorf("cannot map to nil custom cost")
+	}
+	if identifier.Field == nil {
+		return "", fmt.Errorf("cannot map field from identifier with nil field")
+	}
+	switch CustomCostProperty(identifier.Field.Name) {
+	case CustomCostZoneProp:
+		return cc.Zone, nil
+	case CustomCostAccountNameProp:
+		return cc.AccountName, nil
+	case CustomCostChargeCategoryProp:
+		return cc.ChargeCategory, nil
+	case CustomCostDescriptionProp:
+		return cc.Description, nil
+	case CustomCostResourceNameProp:
+		return cc.ResourceName, nil
+	case CustomCostResourceTypeProp:
+		return cc.ResourceType, nil
+	case CustomCostProviderIdProp:
+		return cc.ProviderId, nil
+	case CustomCostUsageUnitProp:
+		return cc.UsageUnit, nil
+	case CustomCostDomainProp:
+		return cc.Domain, nil
+	}
+
+	return "", fmt.Errorf("failed to find string identifier on CustomCost: %s", identifier.Field.Name)
+}
+
+// Maps slice fields from an asset to a []string value based on an identifier
+func customCostSliceFieldMap(cc *CustomCost, identifier ast.Identifier) ([]string, error) {
+	return nil, fmt.Errorf("custom costs have no slice fields")
+}
+
+// Maps map fields from a custom cost to a map[string]string value based on an identifier
+func customCostMapFieldMap(cc *CustomCost, identifier ast.Identifier) (map[string]string, error) {
+	return nil, fmt.Errorf("custom costs have no map fields")
+}

pkg/customcost/parser.go

@@ -0,0 +1,23 @@
+package customcost
+
+import "github.com/opencost/opencost/core/pkg/filter/ast"
+
+// a slice of all the custom costs field instances the lexer should recognize as
+// valid left-hand comparators
+var customCostFilterFields = []*ast.Field{
+	ast.NewField(CustomCostZoneProp),
+	ast.NewField(CustomCostAccountNameProp),
+	ast.NewField(CustomCostChargeCategoryProp),
+	ast.NewField(CustomCostDescriptionProp),
+	ast.NewField(CustomCostResourceNameProp),
+	ast.NewField(CustomCostResourceTypeProp),
+	ast.NewField(CustomCostProviderIdProp),
+	ast.NewField(CustomCostUsageUnitProp),
+	ast.NewField(CustomCostDomainProp),
+}
+
+// NewCustomCostFilterParser creates a new `ast.FilterParser` implementation
+// which uses CustomCost specific fields
+func NewCustomCostFilterParser() ast.FilterParser {
+	return ast.NewFilterParser(customCostFilterFields)
+}

pkg/customcost/props.go

@@ -8,22 +8,30 @@ import (
 type CustomCostProperty string
 
 const (
-	CustomCostDomainProp CustomCostProperty = "domain"
+	CustomCostZoneProp           CustomCostProperty = "zone"
+	CustomCostAccountNameProp                       = "accountName"
+	CustomCostChargeCategoryProp                    = "chargeCategory"
+	CustomCostDescriptionProp                       = "description"
+	CustomCostResourceNameProp                      = "resourceName"
+	CustomCostResourceTypeProp                      = "resourceType"
+	CustomCostProviderIdProp                        = "providerId"
+	CustomCostUsageUnitProp                         = "usageUnit"
+	CustomCostDomainProp                            = "domain"
 )
 
-func ParseCustomCostProperties(props []string) ([]string, error) {
-	var properties []string
+func ParseCustomCostProperties(props []string) ([]CustomCostProperty, error) {
+	var properties []CustomCostProperty
 	added := make(map[CustomCostProperty]struct{})
 
 	for _, prop := range props {
 		property, err := ParseCustomCostProperty(prop)
 		if err != nil {
-			return nil, fmt.Errorf("Failed to parse property: %w", err)
+			return nil, fmt.Errorf("failed to parse property: %w", err)
 		}
 
 		if _, ok := added[property]; !ok {
 			added[property] = struct{}{}
-			properties = append(properties, string(property))
+			properties = append(properties, property)
 		}
 	}
 
@@ -32,7 +40,23 @@ func ParseCustomCostProperties(props []string) ([]string, error) {
 
 func ParseCustomCostProperty(text string) (CustomCostProperty, error) {
 	switch strings.TrimSpace(strings.ToLower(text)) {
-	case strings.TrimSpace(strings.ToLower(string(CustomCostDomainProp))):
+	case strings.TrimSpace(strings.ToLower(string(CustomCostZoneProp))):
+		return CustomCostZoneProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostAccountNameProp)):
+		return CustomCostAccountNameProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostChargeCategoryProp)):
+		return CustomCostChargeCategoryProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostDescriptionProp)):
+		return CustomCostDescriptionProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostResourceNameProp)):
+		return CustomCostResourceNameProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostResourceTypeProp)):
+		return CustomCostResourceTypeProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostProviderIdProp)):
+		return CustomCostProviderIdProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostUsageUnitProp)):
+		return CustomCostUsageUnitProp, nil
+	case strings.TrimSpace(strings.ToLower(CustomCostDomainProp)):
 		return CustomCostDomainProp, nil
 	}
 

pkg/customcost/queryservice_helper.go

@@ -29,14 +29,14 @@ func ParseCustomCostTotalRequest(qp httputil.QueryParams) (*CostTotalRequest, er
 	}
 
 	var filter filter.Filter
-	//filterString := qp.Get("filter", "")
-	//if filterString != "" {
-	//	parser := cloudcost.NewCloudCostFilterParser()
-	//	filter, err = parser.Parse(filterString)
-	//	if err != nil {
-	//		return nil, fmt.Errorf("parsing 'filter' parameter: %s", err)
-	//	}
-	//}
+	filterString := qp.Get("filter", "")
+	if filterString != "" {
+		parser := NewCustomCostFilterParser()
+		filter, err = parser.Parse(filterString)
+		if err != nil {
+			return nil, fmt.Errorf("parsing 'filter' parameter: %s", err)
+		}
+	}
 
 	opts := &CostTotalRequest{
 		Start:       *window.Start(),
@@ -71,14 +71,14 @@ func ParseCustomCostTimeseriesRequest(qp httputil.QueryParams) (*CostTimeseriesR
 	accumulate := opencost.ParseAccumulate(qp.Get("accumulate", ""))
 
 	var filter filter.Filter
-	//filterString := qp.Get("filter", "")
-	//if filterString != "" {
-	//	parser := cloudcost.NewCloudCostFilterParser()
-	//	filter, err = parser.Parse(filterString)
-	//	if err != nil {
-	//		return nil, fmt.Errorf("parsing 'filter' parameter: %s", err)
-	//	}
-	//}
+	filterString := qp.Get("filter", "")
+	if filterString != "" {
+		parser := NewCustomCostFilterParser()
+		filter, err = parser.Parse(filterString)
+		if err != nil {
+			return nil, fmt.Errorf("parsing 'filter' parameter: %s", err)
+		}
+	}
 
 	opts := &CostTimeseriesRequest{
 		Start:       *window.Start(),

pkg/customcost/repositoryquerier.go

@@ -39,6 +39,12 @@ func (rq *RepositoryQuerier) QueryTotal(ctx context.Context, request CostTotalRe
 		return nil, fmt.Errorf("QueryTotal: %w", err)
 	}
 
+	compiler := NewCustomCostMatchCompiler()
+	matcher, err := compiler.Compile(request.Filter)
+	if err != nil {
+		return nil, fmt.Errorf("RepositoryQuerier: Query: failed to compile filters: %w", err)
+	}
+
 	requestWindow := opencost.NewClosedWindow(request.Start, request.End)
 	ccs := NewCustomCostSet(requestWindow)
 	queryStart := request.Start
@@ -54,7 +60,11 @@ func (rq *RepositoryQuerier) QueryTotal(ctx context.Context, request CostTotalRe
 			}
 
 			customCosts := ParseCustomCostResponse(ccResponse)
-			ccs.Add(customCosts)
+			for _, customCost := range customCosts {
+				if matcher.Matches(customCost) {
+					ccs.Add(customCost)
+				}
+			}
 		}
 
 		queryStart = queryEnd

pkg/customcost/types.go

@@ -13,7 +13,7 @@ import (
 type CostTotalRequest struct {
 	Start       time.Time
 	End         time.Time
-	AggregateBy []string
+	AggregateBy []CustomCostProperty
 	Accumulate  opencost.AccumulateOption
 	Filter      filter.Filter
 }
@@ -21,7 +21,7 @@ type CostTotalRequest struct {
 type CostTimeseriesRequest struct {
 	Start       time.Time
 	End         time.Time
-	AggregateBy []string
+	AggregateBy []CustomCostProperty
 	Accumulate  opencost.AccumulateOption
 	Filter      filter.Filter
 }
@@ -163,11 +163,11 @@ func NewCustomCostSet(window opencost.Window) *CustomCostSet {
 	}
 }
 
-func (ccs *CustomCostSet) Add(customCosts []*CustomCost) {
-	ccs.CustomCosts = append(ccs.CustomCosts, customCosts...)
+func (ccs *CustomCostSet) Add(customCost *CustomCost) {
+	ccs.CustomCosts = append(ccs.CustomCosts, customCost)
 }
 
-func (ccs *CustomCostSet) Aggregate(aggregateBy []string) error {
+func (ccs *CustomCostSet) Aggregate(aggregateBy []CustomCostProperty) error {
 	// when no aggregation, return the original CustomCostSet
 	if len(aggregateBy) == 0 {
 		return nil
@@ -197,15 +197,36 @@ func (ccs *CustomCostSet) Aggregate(aggregateBy []string) error {
 	return nil
 }
 
-func generateAggKey(cc *CustomCost, aggregateBy []string) (string, error) {
+func generateAggKey(cc *CustomCost, aggregateBy []CustomCostProperty) (string, error) {
 	var aggKeys []string
 	for _, agg := range aggregateBy {
-		// TODO only domain is supported currently
-		if agg == string(CustomCostDomainProp) {
-			aggKeys = append(aggKeys, cc.Domain)
+		var aggKey string
+		if agg == CustomCostZoneProp {
+			aggKey = cc.Zone
+		} else if agg == CustomCostAccountNameProp {
+			aggKey = cc.AccountName
+		} else if agg == CustomCostChargeCategoryProp {
+			aggKey = cc.ChargeCategory
+		} else if agg == CustomCostDescriptionProp {
+			aggKey = cc.Description
+		} else if agg == CustomCostResourceNameProp {
+			aggKey = cc.ResourceName
+		} else if agg == CustomCostResourceTypeProp {
+			aggKey = cc.ResourceType
+		} else if agg == CustomCostProviderIdProp {
+			aggKey = cc.ProviderId
+		} else if agg == CustomCostUsageUnitProp {
+			aggKey = cc.UsageUnit
+		} else if agg == CustomCostDomainProp {
+			aggKey = cc.Domain
 		} else {
 			return "", fmt.Errorf("unsupported aggregation type: %s", agg)
 		}
+
+		if len(aggKey) == 0 {
+			aggKey = opencost.UnallocatedSuffix
+		}
+		aggKeys = append(aggKeys, aggKey)
 	}
 	aggKey := strings.Join(aggKeys, "/")