Merge branch 'develop' into kaelan-prefix-bucket-backup

pkg/cloud/aliyunprovider.go

@@ -118,6 +118,22 @@ var alibabaInstanceFamilies = []string{
 	"se1",
 }
 
+// AlibabaInfo contains configuration for Alibaba's CUR integration
+type AlibabaInfo struct {
+	AlibabaClusterRegion    string `json:"clusterRegion"`
+	AlibabaServiceKeyName   string `json:"serviceKeyName"`
+	AlibabaServiceKeySecret string `json:"serviceKeySecret"`
+	AlibabaAccountID        string `json:"accountID"`
+}
+
+// IsEmpty returns true if all fields in config are empty, false if not.
+func (ai *AlibabaInfo) IsEmpty() bool {
+	return ai.AlibabaClusterRegion == "" &&
+		ai.AlibabaServiceKeyName == "" &&
+		ai.AlibabaServiceKeySecret == "" &&
+		ai.AlibabaAccountID == ""
+}
+
 // AlibabaAccessKey holds Alibaba credentials parsing from the service-key.json file.
 type AlibabaAccessKey struct {
 	AccessKeyID     string `json:"alibaba_access_key_id"`
@@ -355,6 +371,25 @@ func (alibaba *Alibaba) GetAlibabaAccessKey() (*credentials.AccessKeyCredential,
 	return alibaba.accessKey, nil
 }
 
+func (alibaba *Alibaba) GetAlibabaCloudInfo() (*AlibabaInfo, error) {
+	config, err := alibaba.GetConfig()
+	if err != nil {
+		return nil, fmt.Errorf("could not retrieve AlibabaCloudInfo %s", err)
+	}
+
+	aak, err := alibaba.GetAlibabaAccessKey()
+	if err != nil {
+		return nil, err
+	}
+
+	return &AlibabaInfo{
+		AlibabaClusterRegion:    config.AlibabaClusterRegion,
+		AlibabaServiceKeyName:   aak.AccessKeyId,
+		AlibabaServiceKeySecret: aak.AccessKeySecret,
+		AlibabaAccountID:        config.ProjectID,
+	}, nil
+}
+
 // DownloadPricingData satisfies the provider interface and downloads the prices for Node instances and PVs.
 func (alibaba *Alibaba) DownloadPricingData() error {
 	alibaba.DownloadPricingDataLock.Lock()

pkg/cloud/awsprovider.go

@@ -667,10 +667,13 @@ func (k *awsKey) Features() string {
 // If the instance is a spot instance, it will return PreemptibleType
 // Otherwise returns an empty string
 func (k *awsKey) getUsageType(labels map[string]string) string {
-	if label, ok := labels[EKSCapacityTypeLabel]; ok && label == EKSCapacitySpotTypeValue {
+	if eksLabel, ok := labels[EKSCapacityTypeLabel]; ok && eksLabel == EKSCapacitySpotTypeValue {
 		// We currently write out spot instances as "preemptible" in the pricing data, so these need to match
 		return PreemptibleType
 	}
+	if kLabel, ok := labels[KarpenterCapacityTypeLabel]; ok && kLabel == KarpenterCapacitySpotTypeValue {
+		return PreemptibleType
+	}
 	return ""
 }
 

pkg/cloud/awsprovider_test.go

@@ -25,7 +25,7 @@ func Test_awsKey_getUsageType(t *testing.T) {
 			want: "",
 		},
 		{
-			name: "label with a capacityType set to empty string should return empty string",
+			name: "EKS label with a capacityType set to empty string should return empty string",
 			args: args{
 				labels: map[string]string{
 					EKSCapacityTypeLabel: "",
@@ -34,7 +34,7 @@ func Test_awsKey_getUsageType(t *testing.T) {
 			want: "",
 		},
 		{
-			name: "label with capacityType set to a random value should return empty string",
+			name: "EKS label with capacityType set to a random value should return empty string",
 			args: args{
 				labels: map[string]string{
 					EKSCapacityTypeLabel: "TEST_ME",
@@ -43,7 +43,7 @@ func Test_awsKey_getUsageType(t *testing.T) {
 			want: "",
 		},
 		{
-			name: "label with capacityType set to spot should return spot",
+			name: "EKS label with capacityType set to spot should return spot",
 			args: args{
 				labels: map[string]string{
 					EKSCapacityTypeLabel: EKSCapacitySpotTypeValue,
@@ -51,6 +51,33 @@ func Test_awsKey_getUsageType(t *testing.T) {
 			},
 			want: PreemptibleType,
 		},
+		{
+			name: "Karpenter label with a capacityType set to empty string should return empty string",
+			args: args{
+				labels: map[string]string{
+					KarpenterCapacityTypeLabel: "",
+				},
+			},
+			want: "",
+		},
+		{
+			name: "Karpenter label with capacityType set to a random value should return empty string",
+			args: args{
+				labels: map[string]string{
+					KarpenterCapacityTypeLabel: "TEST_ME",
+				},
+			},
+			want: "",
+		},
+		{
+			name: "Karpenter label with capacityType set to spot should return spot",
+			args: args{
+				labels: map[string]string{
+					KarpenterCapacityTypeLabel: KarpenterCapacitySpotTypeValue,
+				},
+			},
+			want: PreemptibleType,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/cloud/gcpprovider.go

@@ -42,6 +42,9 @@ const (
 	GCPMonthlyBasicDiskCost = 0.04
 	GCPMonthlySSDDiskCost   = 0.17
 	GCPMonthlyGP2DiskCost   = 0.1
+
+	GKEPreemptibleLabel = "cloud.google.com/gke-preemptible"
+	GKESpotLabel        = "cloud.google.com/gke-spot"
 )
 
 // List obtained by installing the `gcloud` CLI tool,
@@ -472,8 +475,10 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 				// GCP gives us description as a string formatted as a map[string]string, so we need to
 				// deconstruct it back into a map[string]string to match the OR struct
 				desc := map[string]string{}
-				if err := json.Unmarshal([]byte(disk.Description), &desc); err != nil {
-					return nil, fmt.Errorf("error converting string to map: %s", err)
+				if disk.Description != "" {
+					if err := json.Unmarshal([]byte(disk.Description), &desc); err != nil {
+						return nil, fmt.Errorf("error converting string to map: %s", err)
+					}
 				}
 
 				// Converts https://www.googleapis.com/compute/v1/projects/xxxxx/zones/us-central1-c to us-central1-c
@@ -1561,11 +1566,13 @@ func parseGCPProjectID(id string) string {
 }
 
 func getUsageType(labels map[string]string) string {
-	if t, ok := labels["cloud.google.com/gke-preemptible"]; ok && t == "true" {
+	if t, ok := labels[GKEPreemptibleLabel]; ok && t == "true" {
 		return "preemptible"
-	} else if t, ok := labels["cloud.google.com/gke-spot"]; ok && t == "true" {
+	} else if t, ok := labels[GKESpotLabel]; ok && t == "true" {
 		// https://cloud.google.com/kubernetes-engine/docs/concepts/spot-vms
 		return "preemptible"
+	} else if t, ok := labels[KarpenterCapacityTypeLabel]; ok && t == KarpenterCapacitySpotTypeValue {
+		return "preemptible"
 	}
 	return "ondemand"
 }

pkg/cloud/gcpprovider_test.go

@@ -73,13 +73,19 @@ func TestGetUsageType(t *testing.T) {
 	}{
 		{
 			input: map[string]string{
-				"cloud.google.com/gke-preemptible": "true",
+				GKEPreemptibleLabel: "true",
 			},
 			expected: "preemptible",
 		},
 		{
 			input: map[string]string{
-				"cloud.google.com/gke-spot": "true",
+				GKESpotLabel: "true",
+			},
+			expected: "preemptible",
+		},
+		{
+			input: map[string]string{
+				KarpenterCapacityTypeLabel: KarpenterCapacitySpotTypeValue,
 			},
 			expected: "preemptible",
 		},

pkg/cloud/provider.go

@@ -32,6 +32,9 @@ const authSecretPath = "/var/secrets/service-key.json"
 const storageConfigSecretPath = "/var/azure-storage-config/azure-storage-config.json"
 const defaultShareTenancyCost = "true"
 
+const KarpenterCapacityTypeLabel = "karpenter.sh/capacity-type"
+const KarpenterCapacitySpotTypeValue = "spot"
+
 var createTableStatements = []string{
 	`CREATE TABLE IF NOT EXISTS names (
 		cluster_id VARCHAR(255) NOT NULL,
@@ -174,6 +177,7 @@ type CustomPricing struct {
 	ServiceKeySecret             string `json:"awsServiceKeySecret,omitempty"`
 	AlibabaServiceKeyName        string `json:"alibabaServiceKeyName,omitempty"`
 	AlibabaServiceKeySecret      string `json:"alibabaServiceKeySecret,omitempty"`
+	AlibabaClusterRegion         string `json:"alibabaClusterRegion,omitempty"`
 	SpotDataRegion               string `json:"awsSpotDataRegion,omitempty"`
 	SpotDataBucket               string `json:"awsSpotDataBucket,omitempty"`
 	SpotDataPrefix               string `json:"awsSpotDataPrefix,omitempty"`

pkg/costmodel/aggregation.go

@@ -2231,7 +2231,7 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 	// resolution.
 	resolution := qp.GetDuration("resolution", env.GetETLResolution())
 
-	// Aggregation is a required comma-separated list of fields by which to
+	// Aggregation is an optional comma-separated list of fields by which to
 	// aggregate results. Some fields allow a sub-field, which is distinguished
 	// with a colon; e.g. "label:app".
 	// Examples: "namespace", "namespace,label:app"

pkg/costmodel/router.go

@@ -462,7 +462,7 @@ func (a *Accesses) ClusterCostsOverTime(w http.ResponseWriter, r *http.Request,
 	offset := r.URL.Query().Get("offset")
 
 	if window == "" {
-		w.Write(WrapData(nil, fmt.Errorf("missing window arguement")))
+		w.Write(WrapData(nil, fmt.Errorf("missing window argument")))
 		return
 	}
 	windowDur, err := timeutil.ParseDuration(window)

pkg/kubecost/cloudcostaggregate.go

@@ -35,6 +35,33 @@ func (ccap CloudCostAggregateProperties) Equal(that CloudCostAggregateProperties
 		ccap.LabelValue == that.LabelValue
 }
 
+// Intersection ensure the values of two CloudCostAggregateProperties are maintain only if they are equal
+func (ccap CloudCostAggregateProperties) Intersection(that CloudCostAggregateProperties) CloudCostAggregateProperties {
+	if ccap.Equal(that) {
+		return ccap
+	}
+	intersectionCCAP := CloudCostAggregateProperties{}
+	if ccap == intersectionCCAP || that == intersectionCCAP {
+		return intersectionCCAP
+	}
+
+	if ccap.Provider == that.Provider {
+		intersectionCCAP.Provider = ccap.Provider
+	}
+	if ccap.WorkGroupID == that.WorkGroupID {
+		intersectionCCAP.WorkGroupID = ccap.WorkGroupID
+	}
+	if ccap.BillingID == that.BillingID {
+		intersectionCCAP.BillingID = ccap.BillingID
+	}
+	if ccap.Service == that.Service {
+		intersectionCCAP.Service = ccap.Service
+	}
+	if ccap.LabelValue == that.LabelValue {
+		intersectionCCAP.LabelValue = ccap.LabelValue
+	}
+	return intersectionCCAP
+}
 func (ccap CloudCostAggregateProperties) Key(props []string) string {
 	if len(props) == 0 {
 		return fmt.Sprintf("%s/%s/%s/%s/%s", ccap.Provider, ccap.BillingID, ccap.WorkGroupID, ccap.Service, ccap.LabelValue)
@@ -86,6 +113,15 @@ type CloudCostAggregate struct {
 	NetCost           float64                      `json:"netCost"`
 }
 
+func NewCloudCostAggregate(properties CloudCostAggregateProperties, kubernetesPercent, cost, netCost float64) *CloudCostAggregate {
+	return &CloudCostAggregate{
+		Properties:        properties,
+		KubernetesPercent: kubernetesPercent,
+		Cost:              cost,
+		NetCost:           netCost,
+	}
+}
+
 func (cca *CloudCostAggregate) Clone() *CloudCostAggregate {
 	return &CloudCostAggregate{
 		Properties:        cca.Properties,
@@ -136,6 +172,9 @@ func (cca *CloudCostAggregate) add(that *CloudCostAggregate) {
 		return
 	}
 
+	// Preserve string properties of cloud cost aggregates that are matching between the two CloudCostAggregate
+	cca.Properties = cca.Properties.Intersection(that.Properties)
+
 	// Compute KubernetesPercent for sum
 	k8sPct := 0.0
 	sumCost := cca.Cost + that.Cost

pkg/kubecost/cloudcostaggregate_test.go

@@ -1,9 +1,10 @@
 package kubecost
 
 import (
-	"github.com/opencost/opencost/pkg/util/timeutil"
 	"testing"
 	"time"
+
+	"github.com/opencost/opencost/pkg/util/timeutil"
 )
 
 var ccaProperties1 = CloudCostAggregateProperties{
@@ -14,6 +15,104 @@ var ccaProperties1 = CloudCostAggregateProperties{
 	LabelValue:  "labelValue1",
 }
 
+func TestCloudCostAggregatePropertiesIntersection(t *testing.T) {
+	testCases := map[string]struct {
+		baseCCAP     CloudCostAggregateProperties
+		intCCAP      CloudCostAggregateProperties
+		expectedCCAP CloudCostAggregateProperties
+	}{
+		"When properties match between both CloudCostAggregateProperties": {
+			baseCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "Service1",
+				LabelValue:  "Label1",
+			},
+			intCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "Service1",
+				LabelValue:  "Label1",
+			},
+			expectedCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "Service1",
+				LabelValue:  "Label1",
+			},
+		},
+		"When one of the properties differ in the two CloudCostAggregateProperties": {
+			baseCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "Service1",
+				LabelValue:  "Label1",
+			},
+			intCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "Service2",
+				LabelValue:  "Label1",
+			},
+			expectedCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "",
+				LabelValue:  "Label1",
+			},
+		},
+		"When two of the properties differ in the two CloudCostAggregateProperties": {
+			baseCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID1",
+				BillingID:   "BillingID1",
+				Service:     "Service1",
+				LabelValue:  "Label1",
+			},
+			intCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "WorkGroupID2",
+				BillingID:   "BillingID1",
+				Service:     "Service2",
+				LabelValue:  "Label1",
+			},
+			expectedCCAP: CloudCostAggregateProperties{
+				Provider:    "CustomProvider",
+				WorkGroupID: "",
+				BillingID:   "BillingID1",
+				Service:     "",
+				LabelValue:  "Label1",
+			},
+		},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actualCCAP := tc.baseCCAP.Intersection(tc.intCCAP)
+			if actualCCAP.Provider != tc.expectedCCAP.Provider {
+				t.Errorf("Case %s: Provider properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
+			}
+			if actualCCAP.WorkGroupID != tc.expectedCCAP.WorkGroupID {
+				t.Errorf("Case %s: WorkGroupID properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
+			}
+			if actualCCAP.BillingID != tc.expectedCCAP.BillingID {
+				t.Errorf("Case %s: BillingID properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
+			}
+			if actualCCAP.Service != tc.expectedCCAP.Service {
+				t.Errorf("Case %s: Service properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
+			}
+			if actualCCAP.LabelValue != tc.expectedCCAP.LabelValue {
+				t.Errorf("Case %s: LabelValue properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
+			}
+		})
+	}
+}
+
 // TestCloudCostAggregate_LoadCloudCostAggregate checks that loaded CloudCostAggregates end up in the correct set in the
 // correct proportions
 func TestCloudCostAggregate_LoadCloudCostAggregate(t *testing.T) {
@@ -240,9 +339,9 @@ func TestCloudCostAggregate_LoadCloudCostAggregate(t *testing.T) {
 					},
 				},
 				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[2],
+					Integration:         "integration",
+					LabelName:           "label",
+					Window:              dayWindows[2],
 					CloudCostAggregates: map[string]*CloudCostAggregate{},
 				},
 			},

pkg/kubecost/cloudcostitem.go

@@ -80,6 +80,17 @@ type CloudCostItem struct {
 	NetCost      float64
 }
 
+// NewCloudCostItem instantiates a new CloudCostItem asset
+func NewCloudCostItem(start, end time.Time, cciProperties CloudCostItemProperties, isKubernetes bool, cost, netcost float64) *CloudCostItem {
+	return &CloudCostItem{
+		Properties:   cciProperties,
+		IsKubernetes: isKubernetes,
+		Window:       NewWindow(&start, &end),
+		Cost:         cost,
+		NetCost:      netcost,
+	}
+}
+
 func (cci *CloudCostItem) Clone() *CloudCostItem {
 	return &CloudCostItem{
 		Properties:   cci.Properties.Clone(),

ui/README.md

@@ -1,5 +1,5 @@
 # OpenCost UI
-The preferred install path for Kubecost is via Helm chart, and is explained [here](http://docs.kubecost.com/install)
+The preferred install path for OpenCost is via Helm chart, and is available explained [here](http://github.com/opencost/opencost-helm-chart)
 
 To manually run the OpenCost UI, follow the steps below.