Kaynağa Gözat

Implement Asset matcher compiler for v2.1 filters

Signed-off-by: Michael Dresser <michaelmdresser@gmail.com>
Michael Dresser 2 yıl önce
ebeveyn
işleme
25747fefb1

+ 8 - 8
pkg/kubecost/allocationmatcher.go

@@ -29,7 +29,7 @@ func NewAllocationMatchCompiler(labelConfig *LabelConfig) *matcher.MatchCompiler
 
 	// The label config pass should be the first pass
 	if labelConfig != nil {
-		passes = append(passes, NewAliasPass(*labelConfig))
+		passes = append(passes, NewAllocationAliasPass(*labelConfig))
 	}
 
 	passes = append(passes,
@@ -96,10 +96,10 @@ func allocationMapFieldMap(a *Allocation, identifier ast.Identifier) (map[string
 	return nil, fmt.Errorf("Failed to find map[string]string identifier on Allocation: %s", identifier.Field.Name)
 }
 
-// aliasPass implements the transform.CompilerPass interface, providing a pass
-// which converts alias nodes to logically-equivalent label/annotation filter
-// nodes based on the label config.
-type aliasPass struct {
+// allocatioAliasPass implements the transform.CompilerPass interface, providing
+// a pass which converts alias nodes to logically-equivalent label/annotation
+// filter nodes based on the label config.
+type allocationAliasPass struct {
 	Config              LabelConfig
 	AliasNameToAliasKey map[afilter.AllocationAlias]string
 }
@@ -118,7 +118,7 @@ type aliasPass struct {
 //	(and (not (contains labels <parseraliaskey>))
 //	     (and (contains annotations departmentkey)
 //	          (<op> annotations[<parseraliaskey>] <filtervalue>))))
-func NewAliasPass(config LabelConfig) transform.CompilerPass {
+func NewAllocationAliasPass(config LabelConfig) transform.CompilerPass {
 	aliasNameToAliasKey := map[afilter.AllocationAlias]string{
 		afilter.AliasDepartment:  config.DepartmentLabel,
 		afilter.AliasEnvironment: config.EnvironmentLabel,
@@ -127,7 +127,7 @@ func NewAliasPass(config LabelConfig) transform.CompilerPass {
 		afilter.AliasTeam:        config.TeamLabel,
 	}
 
-	return &aliasPass{
+	return &allocationAliasPass{
 		Config:              config,
 		AliasNameToAliasKey: aliasNameToAliasKey,
 	}
@@ -135,7 +135,7 @@ func NewAliasPass(config LabelConfig) transform.CompilerPass {
 
 // Exec implements the transform.CompilerPass interface for an alias pass.
 // See aliasPass struct documentation for an explanation.
-func (p *aliasPass) Exec(filter ast.FilterNode) (ast.FilterNode, error) {
+func (p *allocationAliasPass) Exec(filter ast.FilterNode) (ast.FilterNode, error) {
 	if p.AliasNameToAliasKey == nil {
 		return nil, fmt.Errorf("cannot perform alias conversion with nil mapping of alias name -> key")
 	}

+ 24 - 19
pkg/kubecost/asset.go

@@ -8,6 +8,9 @@ import (
 	"strings"
 	"time"
 
+	filter21 "github.com/opencost/opencost/pkg/filter21"
+	"github.com/opencost/opencost/pkg/filter21/ast"
+	"github.com/opencost/opencost/pkg/filter21/matcher"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/util/json"
 	"github.com/opencost/opencost/pkg/util/timeutil"
@@ -422,10 +425,6 @@ func (al AssetLabels) Append(newLabels map[string]string, overwrite bool) {
 	}
 }
 
-// AssetMatchFunc is a function that can be used to match Assets by
-// returning true for any given Asset if a condition is met.
-type AssetMatchFunc func(Asset) bool
-
 // AssetType identifies a type of Asset
 type AssetType int
 
@@ -2745,6 +2744,21 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 		return nil
 	}
 
+	var filter AssetMatcher
+	if opts.Filter == nil {
+		filter = &matcher.AllPass[Asset]{}
+	} else {
+		compiler := NewAssetMatchCompiler(opts.LabelConfig)
+		var err error
+		filter, err = compiler.Compile(opts.Filter)
+		if err != nil {
+			return fmt.Errorf("compiling filter '%s': %w", ast.ToPreOrderShortString(opts.Filter), err)
+		}
+	}
+	if filter == nil {
+		return fmt.Errorf("unexpected nil filter")
+	}
+
 	aggSet := NewAssetSet(as.Start(), as.End())
 	aggSet.AggregationKeys = aggregateBy
 
@@ -2761,15 +2775,8 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 		sa := NewSharedAsset(name, as.Window.Clone())
 		sa.Cost = hourlyCost * hours
 
-		// Insert shared asset if it passes all filters
-		insert := true
-		for _, ff := range opts.FilterFuncs {
-			if !ff(sa) {
-				insert = false
-				break
-			}
-		}
-		if insert {
+		// Insert shared asset if it passes filter
+		if filter.Matches(sa) {
 			err := aggSet.Insert(sa, opts.LabelConfig)
 			if err != nil {
 				return err
@@ -2778,11 +2785,9 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 	}
 
 	// Delete the Assets that don't pass each filter
-	for _, ff := range opts.FilterFuncs {
-		for key, asset := range as.Assets {
-			if !ff(asset) {
-				delete(as.Assets, key)
-			}
+	for key, asset := range as.Assets {
+		if !filter.Matches(asset) {
+			delete(as.Assets, key)
 		}
 	}
 
@@ -3462,7 +3467,7 @@ func (asr *AssetSetRange) newAccumulation() (*AssetSet, error) {
 
 type AssetAggregationOptions struct {
 	SharedHourlyCosts map[string]float64
-	FilterFuncs       []AssetMatchFunc
+	Filter            filter21.Filter
 	LabelConfig       *LabelConfig
 }
 

+ 221 - 0
pkg/kubecost/assetmatcher.go

@@ -0,0 +1,221 @@
+package kubecost
+
+import (
+	"fmt"
+	"strings"
+
+	afilter "github.com/opencost/opencost/pkg/filter21/asset"
+	"github.com/opencost/opencost/pkg/filter21/ast"
+	"github.com/opencost/opencost/pkg/filter21/matcher"
+	"github.com/opencost/opencost/pkg/filter21/ops"
+	"github.com/opencost/opencost/pkg/filter21/transform"
+	"github.com/opencost/opencost/pkg/log"
+)
+
+// AssetMatcher is a matcher implementation for Asset instances,
+// compiled using the matcher.MatchCompiler.
+type AssetMatcher matcher.Matcher[Asset]
+
+// NewAssetMatchCompiler creates a new instance of a
+// matcher.MatchCompiler[Asset] which can be used to compile filter.Filter
+// ASTs into matcher.Matcher[Asset] implementations.
+//
+// If the label config is nil, the compiler will fail to compile alias filters
+// if any are present in the AST.
+//
+// If storage interfaces every support querying natively by alias (e.g. if a
+// data store contained a "product" attribute on an Asset row), that should
+// be handled by a purpose-built AST compiler.
+func NewAssetMatchCompiler(labelConfig *LabelConfig) *matcher.MatchCompiler[Asset] {
+	passes := []transform.CompilerPass{}
+
+	// The label config pass should be the first pass
+	if labelConfig != nil {
+		passes = append(passes, NewAssetAliasPass(*labelConfig))
+	}
+
+	passes = append(passes,
+		transform.PrometheusKeySanitizePass(),
+		transform.UnallocatedReplacementPass(),
+	)
+	return matcher.NewMatchCompiler(
+		assetFieldMap,
+		assetSliceFieldMap,
+		assetMapFieldMap,
+		passes...,
+	)
+}
+
+// Maps fields from an asset to a string value based on an identifier
+func assetFieldMap(a Asset, identifier ast.Identifier) (string, error) {
+	if identifier.Field == nil {
+		return "", fmt.Errorf("cannot map field from identifier with nil field")
+	}
+	if a == nil {
+		return "", fmt.Errorf("cannot map field for nil Asset")
+	}
+
+	// Check special fields before defaulting to properties-based fields
+	switch afilter.AssetField(identifier.Field.Name) {
+	case afilter.FieldType:
+		return strings.ToLower(a.Type().String()), nil
+	case afilter.FieldLabel:
+		labels := a.GetLabels()
+		if labels == nil {
+			return "", nil
+		}
+		return labels[identifier.Key], nil
+	}
+
+	props := a.GetProperties()
+	if props == nil {
+		return "", fmt.Errorf("cannot map field for Asset with nil props")
+	}
+
+	switch afilter.AssetField(identifier.Field.Name) {
+	case afilter.FieldName:
+		return props.Name, nil
+	case afilter.FieldCategory:
+		return props.Category, nil
+	case afilter.FieldClusterID:
+		return props.Cluster, nil
+	case afilter.FieldProject:
+		return props.Project, nil
+	case afilter.FieldProvider:
+		return props.Provider, nil
+	case afilter.FieldProviderID:
+		return props.ProviderID, nil
+	case afilter.FieldAccount:
+		return props.Account, nil
+	case afilter.FieldService:
+		return props.Service, nil
+	}
+
+	return "", fmt.Errorf("Failed to find string identifier on Asset: %s", identifier.Field.Name)
+}
+
+// Maps slice fields from an asset to a []string value based on an identifier
+func assetSliceFieldMap(a Asset, identifier ast.Identifier) ([]string, error) {
+	return nil, fmt.Errorf("Assets have no slice fields")
+}
+
+// Maps map fields from an Asset to a map[string]string value based on an identifier
+func assetMapFieldMap(a Asset, identifier ast.Identifier) (map[string]string, error) {
+	if a == nil {
+		return nil, fmt.Errorf("cannot get map field for nil Asset")
+	}
+	switch afilter.AssetField(identifier.Field.Name) {
+	case afilter.FieldLabel:
+		return a.GetLabels(), nil
+	}
+	return nil, fmt.Errorf("Failed to find map[string]string identifier on Asset: %s", identifier.Field.Name)
+}
+
+// assetAPass implements the transform.CompilerPass interface, providing a pass
+// which converts alias nodes to logically-equivalent label/annotation filter
+// nodes based on the label config.
+type assetAliasPass struct {
+	Config              LabelConfig
+	AliasNameToAliasKey map[afilter.AssetAlias]string
+}
+
+// NewAssetAliasPass creates a compiler pass that converts alias nodes to
+// logically-equivalent label/annotation nodes based on the label config.
+func NewAssetAliasPass(config LabelConfig) transform.CompilerPass {
+	aliasNameToAliasKey := map[afilter.AssetAlias]string{
+		// TODO: is external right?
+		afilter.DepartmentProp:  config.DepartmentExternalLabel,
+		afilter.EnvironmentProp: config.EnvironmentExternalLabel,
+		afilter.OwnerProp:       config.OwnerExternalLabel,
+		afilter.ProductProp:     config.ProductExternalLabel,
+		afilter.TeamProp:        config.TeamExternalLabel,
+	}
+
+	return &assetAliasPass{
+		Config:              config,
+		AliasNameToAliasKey: aliasNameToAliasKey,
+	}
+}
+
+// Exec implements the transform.CompilerPass interface for an alias pass.
+// See aliasPass struct documentation for an explanation.
+func (p *assetAliasPass) Exec(filter ast.FilterNode) (ast.FilterNode, error) {
+	if p.AliasNameToAliasKey == nil {
+		return nil, fmt.Errorf("cannot perform alias conversion with nil mapping of alias name -> key")
+	}
+
+	var transformErr error
+	leafTransformerFunc := func(node ast.FilterNode) ast.FilterNode {
+		if transformErr != nil {
+			return node
+		}
+
+		var field *ast.Field
+		var filterValue string
+		var filterOp ast.FilterOp
+
+		switch concrete := node.(type) {
+		// These ops are not alias ops, alias ops can only be base-level ops
+		// like =, !=, etc. No modification required here.
+		case *ast.AndOp, *ast.OrOp, *ast.NotOp, *ast.VoidOp, *ast.ContradictionOp:
+			return node
+
+		case *ast.EqualOp:
+			field = concrete.Left.Field
+			filterValue = concrete.Right
+			filterOp = ast.FilterOpEquals
+		case *ast.ContainsOp:
+			field = concrete.Left.Field
+			filterValue = concrete.Right
+			filterOp = ast.FilterOpContains
+		case *ast.ContainsPrefixOp:
+			field = concrete.Left.Field
+			filterValue = concrete.Right
+			filterOp = ast.FilterOpContainsPrefix
+		case *ast.ContainsSuffixOp:
+			field = concrete.Left.Field
+			filterValue = concrete.Right
+			filterOp = ast.FilterOpContainsSuffix
+		default:
+			transformErr = fmt.Errorf("unknown op '%s' during alias pass", concrete.Op())
+			return node
+		}
+		if field == nil {
+			return node
+		}
+		if !field.IsAlias() {
+			return node
+		}
+
+		filterFieldAlias := afilter.AssetAlias(field.Name)
+		parserAliasKey, ok := p.AliasNameToAliasKey[filterFieldAlias]
+		if !ok {
+			transformErr = fmt.Errorf("unknown alias field '%s'", filterFieldAlias)
+			return node
+		}
+		labelKey := ops.WithKey(afilter.FieldLabel, parserAliasKey)
+
+		switch filterOp {
+		case ast.FilterOpEquals:
+			return ops.Eq(labelKey, filterValue)
+		case ast.FilterOpContains:
+			return ops.Contains(labelKey, filterValue)
+		case ast.FilterOpContainsPrefix:
+			return ops.ContainsPrefix(labelKey, filterValue)
+		case ast.FilterOpContainsSuffix:
+			return ops.ContainsSuffix(labelKey, filterValue)
+		default:
+			transformErr = fmt.Errorf("unexpected failed case match during Asset alias translation, filterOp %s", filterOp)
+			log.Errorf("Unexpected failed case match for Asset alias translation, filterOp %s", filterOp)
+			return node
+		}
+	}
+
+	newFilter := ast.TransformLeaves(filter, leafTransformerFunc)
+
+	if transformErr != nil {
+		return nil, fmt.Errorf("alias pass transform: %w", transformErr)
+	}
+
+	return newFilter, nil
+}

+ 0 - 3
pkg/kubecost/cloudusage.go

@@ -11,6 +11,3 @@ type CloudUsageSetRange = AssetSetRange
 
 // CloudUsageAggregationOptions is temporarily aliased as the AssetAggregationOptions until further infrastructure and pages can be built to support its usage
 type CloudUsageAggregationOptions = AssetAggregationOptions
-
-// CloudUsageMatchFunc is temporarily aliased as the AssetMatchFunc until further infrastructure and pages can be built to support its usage
-type CloudUsageMatchFunc = AssetMatchFunc

+ 1 - 1
pkg/kubecost/query.go

@@ -87,7 +87,7 @@ type CloudUsageQueryOptions struct {
 	Accumulate   bool
 	AggregateBy  []string
 	Compute      bool
-	FilterFuncs  []CloudUsageMatchFunc
+	Filter       filter21.Filter
 	FilterValues CloudUsageFilter
 	LabelConfig  *LabelConfig
 }

+ 34 - 1
pkg/util/filterutil/filterutil.go

@@ -80,6 +80,24 @@ const (
 	ParamFilterTypes         = "filterTypes"
 )
 
+// ValidAssetFilterParams returns a list of all possible filter parameters
+func ValidAssetFilterParams() []string {
+	return []string{
+		ParamFilterAccounts,
+		ParamFilterCategories,
+		ParamFilterClusters,
+		ParamFilterLabels,
+		ParamFilterNames,
+		ParamFilterProjects,
+		ParamFilterProviders,
+		ParamFilterProviderIDs,
+		ParamFilterProviderIDsV2,
+		ParamFilterRegions,
+		ParamFilterServices,
+		ParamFilterTypes,
+	}
+}
+
 // AllocationPropToV1FilterParamKey maps allocation string property
 // representations to v1 filter param keys for legacy filter config support
 // (e.g. reports). Example mapping: "cluster" -> "filterClusters"
@@ -99,6 +117,22 @@ var AllocationPropToV1FilterParamKey = map[string]string{
 	kubecost.AllocationTeamProp:           ParamFilterTeams,
 }
 
+// Map to store Kubecost Asset property to Asset Filter types.
+// AssetPropToV1FilterParamKey maps asset string property representations to v1
+// filter param keys for legacy filter config support (e.g. reports). Example
+// mapping: "category" -> "filterCategories"
+var AssetPropToV1FilterParamKey = map[kubecost.AssetProperty]string{
+	kubecost.AssetNameProp:       ParamFilterNames,
+	kubecost.AssetTypeProp:       ParamFilterTypes,
+	kubecost.AssetAccountProp:    ParamFilterAccounts,
+	kubecost.AssetCategoryProp:   ParamFilterCategories,
+	kubecost.AssetClusterProp:    ParamFilterClusters,
+	kubecost.AssetProjectProp:    ParamFilterProjects,
+	kubecost.AssetProviderProp:   ParamFilterProviders,
+	kubecost.AssetProviderIDProp: ParamFilterProviderIDs,
+	kubecost.AssetServiceProp:    ParamFilterServices,
+}
+
 // AllHTTPParamKeys returns all HTTP GET parameters used for v1 filters. It is
 // intended to help validate HTTP queries in handlers to help avoid e.g.
 // spelling errors.
@@ -391,7 +425,6 @@ func AssetFilterFromParamsV1(
 			}
 		}
 
-		//
 		clustersOp := opsToOr(ops)
 		filterOps = push(filterOps, clustersOp)
 	}