Merge pull request #1211 from kubecost/mmd/allocationfilter-wildcard-support

Support wildcard (filterNamespaces=kube*) queries with new AllocationFilter types

pkg/kubecost/allocationfilter.go

@@ -1,6 +1,10 @@
 package kubecost
 
-import "github.com/kubecost/cost-model/pkg/log"
+import (
+	"strings"
+
+	"github.com/kubecost/cost-model/pkg/log"
+)
 
 // FilterField is an enum that represents Allocation-specific fields that can be
 // filtered on (namespace, label, etc.)
@@ -35,9 +39,29 @@ type FilterOp string
 // If you add a FilterOp, MAKE SURE TO UPDATE ALL FILTER IMPLEMENTATIONS! Go
 // does not enforce exhaustive pattern matching on "enum" types.
 const (
-	FilterEquals    FilterOp = "equals"
-	FilterNotEquals          = "notequals"
-	FilterContains           = "contains"
+	// FilterEquals is the equality operator
+	// "kube-system" FilterEquals "kube-system" = true
+	// "kube-syste" FilterEquals "kube-system" = false
+	FilterEquals FilterOp = "equals"
+
+	// FilterNotEquals is the inequality operator
+	FilterNotEquals = "notequals"
+
+	// FilterContains is an array/slice membership operator
+	// ["a", "b", "c"] FilterContains "a" = true
+	FilterContains = "contains"
+
+	// FilterStartsWith matches strings with the given prefix.
+	// "kube-system" StartsWith "kube" = true
+	//
+	// When comparing with a field represented by an array/slice, this is like
+	// applying FilterContains to every element of the slice.
+	FilterStartsWith = "startswith"
+
+	// FilterContainsPrefix is like FilterContains, but using StartsWith instead
+	// of Equals.
+	// ["kube-system", "abc123"] ContainsPrefix ["kube"] = true
+	FilterContainsPrefix = "containsprefix"
 )
 
 // AllocationFilter represents anything that can be used to filter an
@@ -189,6 +213,43 @@ func (filter AllocationFilterCondition) Matches(a *Allocation) bool {
 		} else {
 			log.Warnf("Allocation Filter: invalid 'contains' call for non-list filter value")
 		}
+	case FilterStartsWith:
+		if toCompareMissing {
+			return false
+		}
+
+		// We don't need special __unallocated__ logic here because a query
+		// asking for "__unallocated__" won't have a wildcard and unallocated
+		// properties are the empty string.
+
+		s, ok := valueToCompare.(string)
+		if !ok {
+			log.Warnf("Allocation Filter: invalid 'startswith' call for field with unsupported type")
+			return false
+		}
+		return strings.HasPrefix(s, filter.Value)
+	case FilterContainsPrefix:
+		if toCompareMissing {
+			return false
+		}
+
+		// We don't need special __unallocated__ logic here because a query
+		// asking for "__unallocated__" won't have a wildcard and unallocated
+		// properties are the empty string.
+
+		values, ok := valueToCompare.([]string)
+		if !ok {
+			log.Warnf("Allocation Filter: invalid '%s' call for field with unsupported type", FilterContainsPrefix)
+			return false
+		}
+
+		for _, s := range values {
+			if strings.HasPrefix(s, filter.Value) {
+				return true
+			}
+		}
+
+		return false
 	default:
 		log.Errorf("Allocation Filter: Unhandled filter op. This is a filter implementation error and requires immediate patching. Op: %s", filter.Op)
 		return false

pkg/kubecost/allocationfilter_test.go

@@ -27,6 +27,36 @@ func Test_AllocationFilterCondition_Matches(t *testing.T) {
 
 			expected: true,
 		},
+		{
+			name: "ClusterID StartsWith -> true",
+			a: &Allocation{
+				Properties: &AllocationProperties{
+					Cluster: "cluster-one",
+				},
+			},
+			filter: AllocationFilterCondition{
+				Field: FilterClusterID,
+				Op:    FilterStartsWith,
+				Value: "cluster",
+			},
+
+			expected: true,
+		},
+		{
+			name: "ClusterID StartsWith -> false",
+			a: &Allocation{
+				Properties: &AllocationProperties{
+					Cluster: "k8s-one",
+				},
+			},
+			filter: AllocationFilterCondition{
+				Field: FilterClusterID,
+				Op:    FilterStartsWith,
+				Value: "cluster",
+			},
+
+			expected: false,
+		},
 		{
 			name: "Node Equals -> true",
 			a: &Allocation{
@@ -351,6 +381,36 @@ func Test_AllocationFilterCondition_Matches(t *testing.T) {
 
 			expected: false,
 		},
+		{
+			name: `services containsprefix -> true`,
+			a: &Allocation{
+				Properties: &AllocationProperties{
+					Services: []string{"serv1", "serv2"},
+				},
+			},
+			filter: AllocationFilterCondition{
+				Field: FilterServices,
+				Op:    FilterContainsPrefix,
+				Value: "serv",
+			},
+
+			expected: true,
+		},
+		{
+			name: `services containsprefix -> false`,
+			a: &Allocation{
+				Properties: &AllocationProperties{
+					Services: []string{"foo", "bar"},
+				},
+			},
+			filter: AllocationFilterCondition{
+				Field: FilterServices,
+				Op:    FilterContainsPrefix,
+				Value: "serv",
+			},
+
+			expected: false,
+		},
 		{
 			name: `services contains unallocated -> false`,
 			a: &Allocation{

pkg/util/filterutil/allocationfilters.go

@@ -10,6 +10,16 @@ import (
 	"github.com/kubecost/cost-model/pkg/util/httputil"
 )
 
+// parseWildcardEnd checks if the given filter value is wildcarded, meaning
+// it ends in "*". If it does, it removes the suffix and returns the cleaned
+// string and true. Otherwise, it returns the same filter and false.
+//
+// parseWildcardEnd("kube*") = "kube", true
+// parseWildcardEnd("kube") = "kube", false
+func parseWildcardEnd(rawFilterValue string) (string, bool) {
+	return strings.TrimSuffix(rawFilterValue, "*"), strings.HasSuffix(rawFilterValue, "*")
+}
+
 // AllocationFilterFromParamsV1 takes a set of HTTP query parameters and
 // converts them to an AllocationFilter, which is a structured in-Go
 // representation of a set of filters.
@@ -69,8 +79,19 @@ func AllocationFilterFromParamsV1(
 		Filters: []kubecost.AllocationFilter{},
 	}
 	clustersOr.Filters = append(clustersOr.Filters, filterV1SingleValueFromList(filterClusters, kubecost.FilterClusterID))
-	for _, possibleClusterName := range filterClusters {
-		for _, clusterID := range clusterNameToIDs[possibleClusterName] {
+	for _, rawFilterValue := range filterClusters {
+		clusterNameFilter, wildcard := parseWildcardEnd(rawFilterValue)
+
+		clusterIDsToFilter := []string{}
+		for clusterName := range clusterNameToIDs {
+			if wildcard && strings.HasPrefix(clusterName, clusterNameFilter) {
+				clusterIDsToFilter = append(clusterIDsToFilter, clusterNameToIDs[clusterName]...)
+			} else if !wildcard && clusterName == clusterNameFilter {
+				clusterIDsToFilter = append(clusterIDsToFilter, clusterNameToIDs[clusterName]...)
+			}
+		}
+
+		for _, clusterID := range clusterIDsToFilter {
 			clustersOr.Filters = append(clustersOr.Filters,
 				kubecost.AllocationFilterCondition{
 					Field: kubecost.FilterClusterID,
@@ -105,27 +126,39 @@ func AllocationFilterFromParamsV1(
 	for _, rawFilterValue := range filterControllers {
 		split := strings.Split(rawFilterValue, ":")
 		if len(split) == 1 {
-			controllersOr.Filters = append(controllersOr.Filters,
-				kubecost.AllocationFilterCondition{
-					Field: kubecost.FilterControllerName,
-					Op:    kubecost.FilterEquals,
-					Value: split[0],
-				})
+			filterValue, wildcard := parseWildcardEnd(split[0])
+			subFilter := kubecost.AllocationFilterCondition{
+				Field: kubecost.FilterControllerName,
+				Op:    kubecost.FilterEquals,
+				Value: filterValue,
+			}
+
+			if wildcard {
+				subFilter.Op = kubecost.FilterStartsWith
+			}
+			controllersOr.Filters = append(controllersOr.Filters, subFilter)
 		} else if len(split) == 2 {
+			kindFilterVal := split[0]
+			nameFilterVal, wildcard := parseWildcardEnd(split[1])
+
+			kindFilter := kubecost.AllocationFilterCondition{
+				Field: kubecost.FilterControllerKind,
+				Op:    kubecost.FilterEquals,
+				Value: kindFilterVal,
+			}
+			nameFilter := kubecost.AllocationFilterCondition{
+				Field: kubecost.FilterControllerName,
+				Op:    kubecost.FilterEquals,
+				Value: nameFilterVal,
+			}
+
+			if wildcard {
+				nameFilter.Op = kubecost.FilterStartsWith
+			}
+
 			// The controller name AND the controller kind must match
 			multiFilter := kubecost.AllocationFilterAnd{
-				Filters: []kubecost.AllocationFilter{
-					kubecost.AllocationFilterCondition{
-						Field: kubecost.FilterControllerKind,
-						Op:    kubecost.FilterEquals,
-						Value: split[0],
-					},
-					kubecost.AllocationFilterCondition{
-						Field: kubecost.FilterControllerName,
-						Op:    kubecost.FilterEquals,
-						Value: split[1],
-					},
-				},
+				Filters: []kubecost.AllocationFilter{kindFilter, nameFilter},
 			}
 			controllersOr.Filters = append(controllersOr.Filters, multiFilter)
 		} else {
@@ -176,13 +209,17 @@ func AllocationFilterFromParamsV1(
 		Filters: []kubecost.AllocationFilter{},
 	}
 	for _, filterValue := range qp.GetList("filterServices", ",") {
-		servicesFilter.Filters = append(servicesFilter.Filters,
-			kubecost.AllocationFilterCondition{
-				Field: kubecost.FilterServices,
-				Op:    kubecost.FilterContains,
-				Value: filterValue,
-			},
-		)
+		// TODO: wildcard support
+		filterValue, wildcard := parseWildcardEnd(filterValue)
+		subFilter := kubecost.AllocationFilterCondition{
+			Field: kubecost.FilterServices,
+			Op:    kubecost.FilterContains,
+			Value: filterValue,
+		}
+		if wildcard {
+			subFilter.Op = kubecost.FilterContainsPrefix
+		}
+		servicesFilter.Filters = append(servicesFilter.Filters, subFilter)
 	}
 	filter.Filters = append(filter.Filters, servicesFilter)
 
@@ -201,14 +238,20 @@ func filterV1SingleValueFromList(rawFilterValues []string, filterField kubecost.
 
 	for _, filterValue := range rawFilterValues {
 		filterValue = strings.TrimSpace(filterValue)
+		filterValue, wildcard := parseWildcardEnd(filterValue)
 
-		filter.Filters = append(filter.Filters,
-			kubecost.AllocationFilterCondition{
-				Field: filterField,
-				// All v1 filters are equality comparisons
-				Op:    kubecost.FilterEquals,
-				Value: filterValue,
-			})
+		subFilter := kubecost.AllocationFilterCondition{
+			Field: filterField,
+			// All v1 filters are equality comparisons
+			Op:    kubecost.FilterEquals,
+			Value: filterValue,
+		}
+
+		if wildcard {
+			subFilter.Op = kubecost.FilterStartsWith
+		}
+
+		filter.Filters = append(filter.Filters, subFilter)
 	}
 
 	return filter
@@ -224,15 +267,21 @@ func filterV1LabelMappedFromList(rawFilterValues []string, labelName string) kub
 
 	for _, filterValue := range rawFilterValues {
 		filterValue = strings.TrimSpace(filterValue)
+		filterValue, wildcard := parseWildcardEnd(filterValue)
 
-		filter.Filters = append(filter.Filters,
-			kubecost.AllocationFilterCondition{
-				Field: kubecost.FilterLabel,
-				// All v1 filters are equality comparisons
-				Op:    kubecost.FilterEquals,
-				Key:   labelName,
-				Value: filterValue,
-			})
+		subFilter := kubecost.AllocationFilterCondition{
+			Field: kubecost.FilterLabel,
+			// All v1 filters are equality comparisons
+			Op:    kubecost.FilterEquals,
+			Key:   labelName,
+			Value: filterValue,
+		}
+
+		if wildcard {
+			subFilter.Op = kubecost.FilterStartsWith
+		}
+
+		filter.Filters = append(filter.Filters, subFilter)
 	}
 
 	return filter
@@ -257,16 +306,21 @@ func filterV1DoubleValueFromList(rawFilterValuesUnsplit []string, filterField ku
 			}
 			key := prom.SanitizeLabelName(strings.TrimSpace(split[0]))
 			val := strings.TrimSpace(split[1])
+			val, wildcard := parseWildcardEnd(val)
 
-			filter.Filters = append(filter.Filters,
-				kubecost.AllocationFilterCondition{
-					Field: filterField,
-					// All v1 filters are equality comparisons
-					Op:    kubecost.FilterEquals,
-					Key:   key,
-					Value: val,
-				},
-			)
+			subFilter := kubecost.AllocationFilterCondition{
+				Field: filterField,
+				// All v1 filters are equality comparisons
+				Op:    kubecost.FilterEquals,
+				Key:   key,
+				Value: val,
+			}
+
+			if wildcard {
+				subFilter.Op = kubecost.FilterStartsWith
+			}
+
+			filter.Filters = append(filter.Filters, subFilter)
 		}
 	}
 

pkg/util/filterutil/allocationfilters_test.go

@@ -69,6 +69,31 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard cluster ID",
+			qp: map[string]string{
+				"filterClusters": "cluster*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "cluster-one",
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "cluster-two",
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "cluster",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "foo",
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "cluste",
+				}),
+			},
+		},
 		{
 			name: "single cluster name",
 			qp: map[string]string{
@@ -85,6 +110,22 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard cluster name",
+			qp: map[string]string{
+				"filterClusters": "cluster A*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "mapped-cluster-ID-ABC",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster: "cluster-one",
+				}),
+			},
+		},
 		{
 			name: "single node",
 			qp: map[string]string{
@@ -101,6 +142,22 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard node",
+			qp: map[string]string{
+				"filterNodes": "node-1*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Node: "node-123-abc",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Node: "node-456-def",
+				}),
+			},
+		},
 		{
 			name: "single namespace",
 			qp: map[string]string{
@@ -117,6 +174,25 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard namespace",
+			qp: map[string]string{
+				"filterNamespaces": "kube*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Namespace: "kubecost",
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Namespace: "kube-system",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Namespace: "kub",
+				}),
+			},
+		},
 		{
 			name: "single controller kind",
 			qp: map[string]string{
@@ -133,6 +209,22 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard controller kind",
+			qp: map[string]string{
+				"filterControllerKinds": "depl*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					ControllerKind: "deployment",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					ControllerKind: "daemonset",
+				}),
+			},
+		},
 		{
 			name: "single controller name",
 			qp: map[string]string{
@@ -149,6 +241,25 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard controller name",
+			qp: map[string]string{
+				"filterControllers": "kubecost-*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Controller: "kubecost-cost-analyzer",
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Controller: "kubecost-frontend",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Controller: "kube-proxy",
+				}),
+			},
+		},
 		{
 			name: "single controller kind:name combo",
 			qp: map[string]string{
@@ -167,6 +278,28 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard controller kind:name combo",
+			qp: map[string]string{
+				"filterControllers": "deployment:kubecost*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					ControllerKind: "deployment",
+					Controller:     "kubecost-cost-analyzer",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					ControllerKind: "daemonset",
+					Controller:     "kubecost-cost-analyzer",
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					ControllerKind: "deployment",
+					Controller:     "kube-system",
+				}),
+			},
+		},
 		{
 			name: "single pod",
 			qp: map[string]string{
@@ -183,6 +316,22 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard pod",
+			qp: map[string]string{
+				"filterPods": "pod-1*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Pod: "pod-123-abc",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Pod: "pod-456-def",
+				}),
+			},
+		},
 		{
 			name: "single container",
 			qp: map[string]string{
@@ -199,6 +348,22 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard container",
+			qp: map[string]string{
+				"filterContainers": "container-1*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Container: "container-123-abc",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Container: "container-456-def",
+				}),
+			},
+		},
 		{
 			name: "single department",
 			qp: map[string]string{
@@ -219,6 +384,26 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard department",
+			qp: map[string]string{
+				"filterDepartments": "pa*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Labels: map[string]string{
+						"internal-product-umbrella": "pa-1",
+					},
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Labels: map[string]string{
+						"internal-product-umbrella": "ps-N",
+					},
+				}),
+			},
+		},
 		{
 			name: "single label",
 			qp: map[string]string{
@@ -244,6 +429,31 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard label",
+			qp: map[string]string{
+				"filterLabels": "app:cost-*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Labels: map[string]string{
+						"app": "cost-analyzer",
+					},
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Labels: map[string]string{
+						"app": "foo",
+					},
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Labels: map[string]string{
+						"foo": "bar",
+					},
+				}),
+			},
+		},
 		{
 			name: "single annotation",
 			qp: map[string]string{
@@ -269,6 +479,31 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard annotation",
+			qp: map[string]string{
+				"filterAnnotations": "app:cost-*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Annotations: map[string]string{
+						"app": "cost-analyzer",
+					},
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Annotations: map[string]string{
+						"app": "foo",
+					},
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Annotations: map[string]string{
+						"foo": "bar",
+					},
+				}),
+			},
+		},
 		{
 			name: "single service",
 			qp: map[string]string{
@@ -306,6 +541,26 @@ func TestFiltersFromParamsV1(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "wildcard service",
+			qp: map[string]string{
+				"filterServices": "serv*",
+			},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Services: []string{"serv1"},
+				}),
+				allocGenerator(kubecost.AllocationProperties{
+					Services: []string{"serv2"},
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{}),
+				allocGenerator(kubecost.AllocationProperties{
+					Services: []string{"foo"},
+				}),
+			},
+		},
 		{
 			name: "multi: namespaces, labels",
 			qp: map[string]string{