Merge branch 'develop' into swagger

pkg/cloud/awsprovider.go

@@ -65,6 +65,7 @@ var (
 	provIdRx      = regexp.MustCompile("aws:///([^/]+)/([^/]+)")
 	usageTypeRegx = regexp.MustCompile(".*(-|^)(EBS.+)")
 	versionRx     = regexp.MustCompile("^#Version: (\\d+)\\.\\d+$")
+	regionRx      = regexp.MustCompile("([a-z]+-[a-z]+-[0-9])")
 )
 
 func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
@@ -1704,11 +1705,25 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 				volumeSize = int64(*volume.Size)
 			}
 
+			// This is turning us-east-1a into us-east-1
+			var zone string
+			if volume.AvailabilityZone != nil {
+				zone = *volume.AvailabilityZone
+			}
+			var region, url string
+			region = regionRx.FindString(zone)
+			if region != "" {
+				url = "https://console.aws.amazon.com/ec2/home?region=" + region + "#Volumes:sort=desc:createTime"
+			} else {
+				url = "https://console.aws.amazon.com/ec2/home?#Volumes:sort=desc:createTime"
+			}
+
 			or := OrphanedResource{
 				Kind:        "disk",
-				Region:      *volume.AvailabilityZone,
+				Region:      zone,
 				Size:        &volumeSize,
 				DiskName:    *volume.VolumeId,
+				Url:         url,
 				MonthlyCost: cost,
 			}
 
@@ -1720,9 +1735,23 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 		if aws.isAddressOrphaned(address) {
 			cost := AWSHourlyPublicIPCost * timeutil.HoursPerMonth
 
+			desc := map[string]string{}
+			for _, tag := range address.Tags {
+				if tag.Key == nil {
+					continue
+				}
+				if tag.Value == nil {
+					desc[*tag.Key] = ""
+				} else {
+					desc[*tag.Key] = *tag.Value
+				}
+			}
+
 			or := OrphanedResource{
 				Kind:        "address",
 				Address:     *address.PublicIp,
+				Description: desc,
+				Url:         "http://console.aws.amazon.com/ec2/home?#Addresses",
 				MonthlyCost: &cost,
 			}
 

pkg/cloud/azureprovider.go

@@ -1282,13 +1282,19 @@ func (az *Azure) GetOrphanedResources() ([]OrphanedResource, error) {
 				diskSize = int64(*d.DiskSizeGB)
 			}
 
+			desc := map[string]string{}
+			for k, v := range d.Tags {
+				if v == nil {
+					desc[k] = ""
+				} else {
+					desc[k] = *v
+				}
+			}
+
 			or := OrphanedResource{
-				Kind:   "disk",
-				Region: diskRegion,
-				Description: map[string]string{
-					"diskState":   string(d.DiskState),
-					"timeCreated": d.TimeCreated.String(),
-				},
+				Kind:        "disk",
+				Region:      diskRegion,
+				Description: desc,
 				Size:        &diskSize,
 				DiskName:    diskName,
 				MonthlyCost: &cost,

pkg/cloud/gcpprovider.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"net/http"
 	"os"
+	"path"
 	"regexp"
 	"strconv"
 	"strings"
@@ -468,12 +469,26 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 					return nil, err
 				}
 
+				// GCP gives us description as a string formatted as a map[string]string, so we need to
+				// deconstruct it back into a map[string]string to match the OR struct
+				desc := map[string]string{}
+				if err := json.Unmarshal([]byte(disk.Description), &desc); err != nil {
+					return nil, fmt.Errorf("error converting string to map: %s", err)
+				}
+
+				// Converts https://www.googleapis.com/compute/v1/projects/xxxxx/zones/us-central1-c to us-central1-c
+				zone := path.Base(disk.Zone)
+				if zone == "." {
+					zone = ""
+				}
+
 				or := OrphanedResource{
 					Kind:        "disk",
-					Region:      disk.Zone,
-					Description: map[string]string{},
+					Region:      zone,
+					Description: desc,
 					Size:        &disk.SizeGb,
 					DiskName:    disk.Name,
+					Url:         disk.SelfLink,
 					MonthlyCost: cost,
 				}
 				orphanedResources = append(orphanedResources, or)
@@ -491,13 +506,20 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 				//todo: use GCP pricing
 				cost := GCPHourlyPublicIPCost * timeutil.HoursPerMonth
 
+				// Converts https://www.googleapis.com/compute/v1/projects/xxxxx/regions/us-central1 to us-central1
+				region := path.Base(address.Region)
+				if region == "." {
+					region = ""
+				}
+
 				or := OrphanedResource{
 					Kind:   "address",
-					Region: address.Region,
+					Region: region,
 					Description: map[string]string{
 						"type": address.AddressType,
 					},
 					Address:     address.Address,
+					Url:         address.SelfLink,
 					MonthlyCost: &cost,
 				}
 				orphanedResources = append(orphanedResources, or)

pkg/cloud/provider.go

@@ -110,6 +110,7 @@ type OrphanedResource struct {
 	Description map[string]string `json:"description"`
 	Size        *int64            `json:"diskSizeInGB,omitempty"`
 	DiskName    string            `json:"diskName,omitempty"`
+	Url         string            `json:"url"`
 	Address     string            `json:"ipAddress,omitempty"`
 	MonthlyCost *float64          `json:"monthlyCost"`
 }

pkg/env/costmodelenv.go

@@ -481,15 +481,9 @@ func GetETLMaxPrometheusQueryDuration() time.Duration {
 
 // GetETLResolution determines the resolution of ETL queries. The smaller the
 // duration, the higher the resolution; the higher the resolution, the more
-// accurate the query results, but the more computationally expensive. This
-// value is always 1m for Prometheus, but is configurable for Thanos.
+// accurate the query results, but the more computationally expensive.
 func GetETLResolution() time.Duration {
-	// If Thanos is not enabled, hard-code to 1m resolution
-	if !IsThanosEnabled() {
-		return 60 * time.Second
-	}
-
-	// Thanos is enabled, so use the configured ETL resolution, or default to
+	// Use the configured ETL resolution, or default to
 	// 5m (i.e. 300s)
 	secs := time.Duration(GetInt64(ETLResolutionSeconds, 300))
 	return secs * time.Second

pkg/kubecost/allocation_test.go

@@ -413,8 +413,8 @@ func TestAllocationSet_generateKey(t *testing.T) {
 	}
 
 	key = alloc.generateKey(props, nil)
-	if key != "cluster1//app=app1" {
-		t.Fatalf("generateKey: expected \"cluster1//app=app1\"; actual \"%s\"", key)
+	if key != "cluster1//app1" {
+		t.Fatalf("generateKey: expected \"cluster1//app1\"; actual \"%s\"", key)
 	}
 
 	alloc.Properties = &AllocationProperties{
@@ -426,8 +426,8 @@ func TestAllocationSet_generateKey(t *testing.T) {
 		},
 	}
 	key = alloc.generateKey(props, nil)
-	if key != "cluster1/namespace1/app=app1" {
-		t.Fatalf("generateKey: expected \"cluster1/namespace1/app=app1\"; actual \"%s\"", key)
+	if key != "cluster1/namespace1/app1" {
+		t.Fatalf("generateKey: expected \"cluster1/namespace1/app1\"; actual \"%s\"", key)
 	}
 
 	props = []string{
@@ -552,15 +552,15 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 	//     idle:                                  20.00   5.00  15.00   0.00   0.00   0.00   0.00
 	//     namespace1:
 	//       pod1:
-	//         container1: [app=app1, env=env1]   16.00   1.00  11.00   1.00   1.00   1.00   1.00
+	//         container1: [app1, env1]   16.00   1.00  11.00   1.00   1.00   1.00   1.00
 	//       pod-abc: (deployment1)
 	//         container2:                         6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	//       pod-def: (deployment1)
 	//         container3:                         6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	//     namespace2:
 	//       pod-ghi: (deployment2)
-	//         container4: [app=app2, env=env2]    6.00   1.00   1.00   1.00   1.00   1.00   1.00
-	//         container5: [app=app2, env=env2]    6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container4: [app2, env2]    6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container5: [app2, env2]    6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	//       pod-jkl: (daemonset1)
 	//         container6: {service1}              6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	// +-----------------------------------------+------+------+------+------+------+------+------+
@@ -570,16 +570,16 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 	//     idle:                                  10.00   5.00   5.00   0.00   0.00   0.00   0.00
 	//     namespace2:
 	//       pod-mno: (deployment2)
-	//         container4: [app=app2]              6.00   1.00   1.00   1.00   1.00   1.00   1.00
-	//         container5: [app=app2]              6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container4: [app2]              6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container5: [app2]              6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	//       pod-pqr: (daemonset1)
 	//         container6: {service1}              6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	//     namespace3:
 	//       pod-stu: (deployment3)
-	//         container7: an[team=team1]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container7: an[team1]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	//       pod-vwx: (statefulset1)
-	//         container8: an[team=team2]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
-	//         container9: an[team=team1]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container8: an[team2]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container9: an[team1]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
 	// +----------------------------------------+------+------+------+------+------+------+------+
 	//   cluster2 subtotal                        46.00  11.00  11.00   6.00   6.00   6.00   6.00
 	// +----------------------------------------+------+------+------+------+------+------+------+
@@ -843,8 +843,8 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			numResults: numLabelApps + numIdle + numUnallocated,
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
-				"app=app1":        16.00,
-				"app=app2":        24.00,
+				"app1":            16.00,
+				"app2":            24.00,
 				IdleSuffix:        30.00,
 				UnallocatedSuffix: 42.00,
 			},
@@ -878,8 +878,8 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			numResults: 2 + numIdle + numUnallocated,
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
-				"team=team1":      12.00,
-				"team=team2":      6.00,
+				"team1":           12.00,
+				"team2":           6.00,
 				IdleSuffix:        30.00,
 				UnallocatedSuffix: 64.00,
 			},
@@ -933,10 +933,10 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			totalCost:  activeTotalCost + idleTotalCost,
 			// sets should be {idle, unallocated, app1/env1, app2/env2, app2/unallocated}
 			results: map[string]float64{
-				"app=app1/env=env1":                         16.00,
-				"app=app2/env=env2":                         12.00,
-				"app=app2/" + UnallocatedSuffix:             12.00,
-				IdleSuffix:                                  30.00,
+				"app1/env1":                 16.00,
+				"app2/env2":                 12.00,
+				"app2/" + UnallocatedSuffix: 12.00,
+				IdleSuffix:                  30.00,
 				UnallocatedSuffix + "/" + UnallocatedSuffix: 42.00,
 			},
 			windowStart: startYesterday,
@@ -951,11 +951,11 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			numResults: 6,
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
-				"cluster1/app=app2/env=env2": 12.00,
-				"__idle__":                   30.00,
-				"cluster1/app=app1/env=env1": 16.00,
+				"cluster1/app2/env2": 12.00,
+				"__idle__":           30.00,
+				"cluster1/app1/env1": 16.00,
 				"cluster1/" + UnallocatedSuffix + "/" + UnallocatedSuffix: 18.00,
-				"cluster2/app=app2/" + UnallocatedSuffix:                  12.00,
+				"cluster2/app2/" + UnallocatedSuffix:                      12.00,
 				"cluster2/" + UnallocatedSuffix + "/" + UnallocatedSuffix: 24.00,
 			},
 			windowStart: startYesterday,
@@ -971,12 +971,12 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
 				"pod-jkl/" + UnallocatedSuffix: 6.00,
-				"pod-stu/team=team1":           6.00,
+				"pod-stu/team1":                6.00,
 				"pod-abc/" + UnallocatedSuffix: 6.00,
 				"pod-pqr/" + UnallocatedSuffix: 6.00,
 				"pod-def/" + UnallocatedSuffix: 6.00,
-				"pod-vwx/team=team1":           6.00,
-				"pod-vwx/team=team2":           6.00,
+				"pod-vwx/team1":                6.00,
+				"pod-vwx/team2":                6.00,
 				"pod1/" + UnallocatedSuffix:    16.00,
 				"pod-mno/" + UnallocatedSuffix: 12.00,
 				"pod-ghi/" + UnallocatedSuffix: 12.00,

pkg/kubecost/allocationprops.go

@@ -298,7 +298,7 @@ func (p *AllocationProperties) GenerateKey(aggregateBy []string, labelConfig *La
 			} else {
 				labelName := labelConfig.Sanitize(strings.TrimPrefix(agg, "label:"))
 				if labelValue, ok := labels[labelName]; ok {
-					names = append(names, fmt.Sprintf("%s=%s", labelName, labelValue))
+					names = append(names, fmt.Sprintf("%s", labelValue))
 				} else {
 					names = append(names, UnallocatedSuffix)
 				}
@@ -310,7 +310,7 @@ func (p *AllocationProperties) GenerateKey(aggregateBy []string, labelConfig *La
 			} else {
 				annotationName := labelConfig.Sanitize(strings.TrimPrefix(agg, "annotation:"))
 				if annotationValue, ok := annotations[annotationName]; ok {
-					names = append(names, fmt.Sprintf("%s=%s", annotationName, annotationValue))
+					names = append(names, fmt.Sprintf("%s", annotationValue))
 				} else {
 					names = append(names, UnallocatedSuffix)
 				}

pkg/util/allocationfilterutil/v2/parser_test.go

@@ -42,6 +42,66 @@ func TestParse(t *testing.T) {
 				allocGenerator(kubecost.AllocationProperties{Namespace: "kube-system"}),
 			},
 		},
+		{
+			input: `cluster:"cluster-one"+namespace:"kubecost"+controllerKind:"daemonset"+controllerName:"kubecost-network-costs"+container:"kubecost-network-costs"`,
+			expected: kubecost.AllocationFilterAnd{[]kubecost.AllocationFilter{
+				kubecost.AllocationFilterOr{[]kubecost.AllocationFilter{
+					kubecost.AllocationFilterCondition{
+						Field: kubecost.FilterClusterID,
+						Op:    kubecost.FilterEquals,
+						Value: "cluster-one",
+					},
+				}},
+				kubecost.AllocationFilterOr{[]kubecost.AllocationFilter{
+					kubecost.AllocationFilterCondition{
+						Field: kubecost.FilterNamespace,
+						Op:    kubecost.FilterEquals,
+						Value: "kubecost",
+					},
+				}},
+				kubecost.AllocationFilterOr{[]kubecost.AllocationFilter{
+					kubecost.AllocationFilterCondition{
+						Field: kubecost.FilterControllerKind,
+						Op:    kubecost.FilterEquals,
+						Value: "daemonset",
+					},
+				}},
+				kubecost.AllocationFilterOr{[]kubecost.AllocationFilter{
+					kubecost.AllocationFilterCondition{
+						Field: kubecost.FilterControllerName,
+						Op:    kubecost.FilterEquals,
+						Value: "kubecost-network-costs",
+					},
+				}},
+				kubecost.AllocationFilterOr{[]kubecost.AllocationFilter{
+					kubecost.AllocationFilterCondition{
+						Field: kubecost.FilterContainer,
+						Op:    kubecost.FilterEquals,
+						Value: "kubecost-network-costs",
+					},
+				}},
+			}},
+			shouldMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster:        "cluster-one",
+					Namespace:      "kubecost",
+					ControllerKind: "daemonset",
+					Controller:     "kubecost-network-costs",
+					Pod:            "kubecost-network-costs-abc123",
+					Container:      "kubecost-network-costs",
+				}),
+			},
+			shouldNotMatch: []kubecost.Allocation{
+				allocGenerator(kubecost.AllocationProperties{
+					Cluster:        "cluster-one",
+					Namespace:      "default",
+					ControllerKind: "deployment",
+					Controller:     "workload-abc",
+					Pod:            "workload-abc-123abc",
+					Container:      "abc",
+				}),
+			},
+		},
 		{
 			input: `namespace!:"kubecost","kube-system"`,
 			expected: kubecost.AllocationFilterAnd{[]kubecost.AllocationFilter{

ui/src/Reports.js

@@ -155,11 +155,6 @@ const ReportsPage = () => {
         const allocationRange = resp.data
         for (const i in allocationRange) {
           // update cluster aggregations to use clusterName/clusterId names
-	  if (aggregateBy == 'cluster') {
-            for (const k in allocationRange[i]) {
-              allocationRange[i][k].name = 'cluster-one';
-            }
-          }
           allocationRange[i] = sortBy(allocationRange[i], a => a.totalCost)
         }
         setAllocationData(allocationRange)