Merge branch 'develop' into kaelan-update-aws-sdk

README.md

@@ -6,13 +6,13 @@ OpenCost models give teams visibility into current and historical Kubernetes spe
 
 OpenCost was originally developed and open sourced by [Kubecost](https://kubecost.com). This project combines a [specification](/spec/) as well as a Golang implementation of these detailed requirements.
 
-![OpenCost allocation UI](/allocation-drilldown.gif)
+![OpenCost allocation UI](./ui/src/opencost-ui.png)
 
 To see the full functionality of OpenCost you can view [OpenCost features](https://opencost.io). Here is a summary of features enabled:
 
-- Real-time cost allocation by Kubernetes service, deployment, namespace, label, statefulset, daemonset, pod, and container
-- Dynamic asset pricing enabled by integrations with AWS, Azure, and GCP billing APIs
-- Supports on-prem k8s clusters with custom pricing sheets
+- Real-time cost allocation by Kubernetes cluster, node, namespace, controller kind, controller, service, or pod
+- Dynamic onDemand asset pricing enabled by integrations with AWS, Azure, and GCP billing APIs
+- Supports on-prem k8s clusters with custom CSV pricing
 - Allocation for in-cluster resources like CPU, GPU, memory, and persistent volumes.
 - Easily export pricing data to Prometheus with /metrics endpoint ([learn more](PROMETHEUS.md))
 - Free and open source distribution (Apache2 license)
@@ -21,7 +21,7 @@ To see the full functionality of OpenCost you can view [OpenCost features](https
 
 You can deploy OpenCost on any Kubernetes 1.8+ cluster in a matter of minutes, if not seconds!
 
-Visit the full documentation for [recommended install options](https://www.opencost.io/docs/install). Compared to building from source, installing from Helm is faster and includes all necessary dependencies.
+Visit the full documentation for [recommended install options](https://www.opencost.io/docs/install).
 
 ## Usage
 

pkg/cloud/aliyunprovider.go

@@ -43,7 +43,6 @@ const (
 	ALIBABA_YEAR_PRICE_UNIT                    = "Year"
 	ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE       = "unknown"
 	ALIBABA_NOT_SUPPORTED_INSTANCE_FAMILY_TYPE = "unsupported"
-	ALIBABA_ENHANCED_GENERAL_PURPOSE_TYPE      = "g6e"
 	ALIBABA_DISK_CLOUD_ESSD_CATEGORY           = "cloud_essd"
 	ALIBABA_DISK_CLOUD_CATEGORY                = "cloud"
 	ALIBABA_DATA_DISK_CATEGORY                 = "data"
@@ -62,6 +61,9 @@ var (
 	sizeRegEx = regexp.MustCompile("(.*?)Gi")
 )
 
+// Variable to keep track of instance families that fail in DescribePrice API due improper defaulting of systemDisk if the information is not available
+var alibabaDefaultToCloudEssd = []string{"g6e", "r6e", "r7", "g7", "g7a", "r7a"}
+
 // Why predefined and dependency on code? Can be converted to API call - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/regions-describeregions
 var alibabaRegions = []string{
 	"cn-qingdao",
@@ -93,13 +95,27 @@ var alibabaRegions = []string{
 }
 
 // To-Do: Convert to API call - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/describeinstancetypefamilies
-// Also first pass only completely tested pricing API for General pupose instances families.
+// Also first pass only completely tested pricing API for General pupose instances families & memory optimized instance families
 var alibabaInstanceFamilies = []string{
+	"g7",
+	"g7a",
 	"g6e",
 	"g6",
 	"g5",
 	"sn2",
 	"sn2ne",
+	"r7",
+	"r7a",
+	"r6e",
+	"r6a",
+	"r6",
+	"r5",
+	"se1",
+	"se1ne",
+	"re6",
+	"re6p",
+	"re4",
+	"se1",
 }
 
 // AlibabaAccessKey holds Alibaba credentials parsing from the service-key.json file.
@@ -873,9 +889,9 @@ func createDescribePriceACSRequest(i interface{}) (*requests.CommonRequest, erro
 				request.QueryParams["SystemDisk.PerformanceLevel"] = node.SystemDisk.PerformanceLevel
 			}
 		} else {
-			// For Enhanced General Purpose Type g6e SystemDisk.Category param doesn't default right,
-			// need it to be specifically assigned to "cloud_ssd" otherwise there's errors
-			if node.InstanceTypeFamily == ALIBABA_ENHANCED_GENERAL_PURPOSE_TYPE {
+			// When System Disk information is not available for instance family g6e, r7 and r6e the defaults in
+			// DescribePrice dont default rightly to cloud_essd for these instances.
+			if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) {
 				request.QueryParams["SystemDisk.Category"] = ALIBABA_DISK_CLOUD_ESSD_CATEGORY
 			}
 		}

pkg/cloud/aliyunprovider_test.go

@@ -83,6 +83,34 @@ func TestProcessDescribePriceAndCreateAlibabaPricing(t *testing.T) {
 		teststruct    interface{}
 		expectedError error
 	}{
+		{
+			name: "test General Purpose Type g7 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.g7.4xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16777216KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-01a",
+				InstanceTypeFamily: "g7",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test General Purpose Type g7a instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.g7a.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-01b",
+				InstanceTypeFamily: "g7a",
+			},
+			expectedError: nil,
+		},
 		{
 			name: "test Enhanced General Purpose Type g6e instance family",
 			teststruct: &SlimK8sNode{
@@ -153,6 +181,174 @@ func TestProcessDescribePriceAndCreateAlibabaPricing(t *testing.T) {
 			},
 			expectedError: nil,
 		},
+		{
+			name: "test Memory Optmized instance type r7 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.r7.6xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "2013265592KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-06",
+				InstanceTypeFamily: "r7",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory Optmized instance type r7a instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.r7a.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-06a",
+				InstanceTypeFamily: "r7a",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Enhanced Memory Optmized instance type r6e instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.r6e.4xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "2013265592KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-07",
+				InstanceTypeFamily: "r6e",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory Optmized instance type r6a instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.r6a.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-07a",
+				InstanceTypeFamily: "r6a",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory Optmized instance type r6 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.r6.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-08",
+				InstanceTypeFamily: "r6",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory type instance and r5 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.r5.xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-09",
+				InstanceTypeFamily: "r5",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory Optmized instance type with se1 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.se1.4xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16777216KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-10",
+				InstanceTypeFamily: "se1",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory Optmized instance type with Enhanced Network Performance se1ne instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.se1ne.3xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "100663296KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-11",
+				InstanceTypeFamily: "se1ne",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test High Memory type with re6 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.re6.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-12",
+				InstanceTypeFamily: "re6",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Persistent Memory Optimized type with re6p instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.re6p.4xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-13",
+				InstanceTypeFamily: "re6p",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory type with re4 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.re4.10xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "41943040KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-14",
+				InstanceTypeFamily: "re4",
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test Memory optimized type with se1 instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.se1.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-15",
+				InstanceTypeFamily: "se1",
+			},
+			expectedError: nil,
+		},
 		{
 			name:          "test for a nil information",
 			teststruct:    nil,

pkg/cloud/azureprovider.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"context"
 	"fmt"
+	"github.com/opencost/opencost/pkg/kubecost"
 	"io"
 	"net/http"
 	"net/url"
@@ -1318,7 +1319,7 @@ func (az *Azure) ClusterInfo() (map[string]string, error) {
 	if c.ClusterName != "" {
 		m["name"] = c.ClusterName
 	}
-	m["provider"] = "Azure"
+	m["provider"] = kubecost.AzureProvider
 	m["account"] = az.clusterAccountId
 	m["region"] = az.clusterRegion
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)

pkg/cloud/customprovider.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"errors"
 	"fmt"
+	"github.com/opencost/opencost/pkg/kubecost"
 	"io"
 	"strconv"
 	"strings"
@@ -108,7 +109,7 @@ func (cp *CustomProvider) ClusterInfo() (map[string]string, error) {
 	if conf.ClusterName != "" {
 		m["name"] = conf.ClusterName
 	}
-	m["provider"] = "custom"
+	m["provider"] = kubecost.CustomProvider
 	m["id"] = env.GetClusterID()
 	return m, nil
 }

pkg/cloud/scalewayprovider.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"errors"
 	"fmt"
+	"github.com/opencost/opencost/pkg/kubecost"
 	"io"
 	"strconv"
 	"strings"
@@ -267,7 +268,7 @@ func (scw *Scaleway) ClusterInfo() (map[string]string, error) {
 	if c.ClusterName != "" {
 		m["name"] = c.ClusterName
 	}
-	m["provider"] = "Scaleway"
+	m["provider"] = kubecost.ScalewayProvider
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 	m["id"] = env.GetClusterID()
 	return m, nil

pkg/kubecost/assetprops.go

@@ -130,6 +130,9 @@ const AlibabaProvider = "Alibaba"
 // CSVProvider describes the provider a CSV
 const CSVProvider = "CSV"
 
+// CustomProvider describes a custom provider
+const CustomProvider = "custom"
+
 // ScalewayProvider describes the provider Scaleway
 const ScalewayProvider = "Scaleway"
 

pkg/kubecost/summaryallocation.go

@@ -1183,6 +1183,9 @@ func (sas *SummaryAllocationSet) RAMEfficiency() float64 {
 	totalRAMBytesRequest := 0.0
 	totalRAMCost := 0.0
 	for _, sa := range sas.SummaryAllocations {
+		if sa.IsIdle() {
+			continue
+		}
 		totalRAMBytesUsage += sa.RAMBytesUsageAverage
 		totalRAMBytesRequest += sa.RAMBytesRequestAverage
 		totalRAMCost += sa.RAMCost
@@ -1212,6 +1215,9 @@ func (sas *SummaryAllocationSet) CPUEfficiency() float64 {
 	totalCPUCoreRequest := 0.0
 	totalCPUCost := 0.0
 	for _, sa := range sas.SummaryAllocations {
+		if sa.IsIdle() {
+			continue
+		}
 		totalCPUCoreUsage += sa.CPUCoreUsageAverage
 		totalCPUCoreRequest += sa.CPUCoreRequestAverage
 		totalCPUCost += sa.CPUCost
@@ -1237,19 +1243,18 @@ func (sas *SummaryAllocationSet) TotalEfficiency() float64 {
 	sas.RLock()
 	defer sas.RUnlock()
 
-	totalRAMCostEff := 0.0
-	totalCPUCostEff := 0.0
 	totalRAMCost := 0.0
 	totalCPUCost := 0.0
 	for _, sa := range sas.SummaryAllocations {
-		totalRAMCostEff += sa.RAMEfficiency() * sa.RAMCost
-		totalCPUCostEff += sa.CPUEfficiency() * sa.CPUCost
+		if sa.IsIdle() {
+			continue
+		}
 		totalRAMCost += sa.RAMCost
 		totalCPUCost += sa.CPUCost
 	}
 
 	if totalRAMCost+totalCPUCost > 0 {
-		return (totalRAMCostEff + totalCPUCostEff) / (totalRAMCost + totalCPUCost)
+		return (totalRAMCost*sas.RAMEfficiency() + totalCPUCost*sas.CPUEfficiency()) / (totalRAMCost + totalCPUCost)
 	}
 
 	return 0.0

pkg/kubecost/summaryallocation_test.go

@@ -213,10 +213,10 @@ func TestSummaryAllocation_Add(t *testing.T) {
 
 func TestSummaryAllocationSet_RAMEfficiency(t *testing.T) {
 	// Generating 6 sample summary allocations for testing
-	var sa1, sa2, sa3, sa4, sa5, sa6 *SummaryAllocation
+	var sa1, sa2, sa3, sa4, sa5, sa6, idlesa *SummaryAllocation
 
 	// Generating accumulated summary allocation sets for testing
-	var sas1, sas2, sas3, sas4, sas5 *SummaryAllocationSet
+	var sas1, sas2, sas3, sas4, sas5, sas6 *SummaryAllocationSet
 
 	window, _ := ParseWindowUTC("7d")
 
@@ -314,6 +314,20 @@ func TestSummaryAllocationSet_RAMEfficiency(t *testing.T) {
 		RAMCost:                0.10,
 	}
 
+	idlesa = &SummaryAllocation{
+		Name: IdleSuffix,
+		Properties: &AllocationProperties{
+			Cluster:   "cluster1",
+			Namespace: "namespace1",
+			Pod:       "pod1",
+			Container: "container7",
+		},
+		Start:   saStart,
+		End:     saEnd,
+		CPUCost: 1.0,
+		RAMCost: 1.0,
+	}
+
 	testcase1Map := map[string]*SummaryAllocation{
 		"cluster1/namespace1/pod1/container1": sa1,
 		"cluster1/namespace1/pod1/container2": sa2,
@@ -340,6 +354,12 @@ func TestSummaryAllocationSet_RAMEfficiency(t *testing.T) {
 		"cluster1/namespace1/pod1/container6": sa6,
 	}
 
+	testcase6Map := map[string]*SummaryAllocation{
+		"cluster1/namespace1/pod1/container1": sa1,
+		"cluster1/namespace1/pod1/container2": sa2,
+		"cluster1/__idle__":                   idlesa,
+	}
+
 	sas1 = &SummaryAllocationSet{
 		SummaryAllocations: testcase1Map,
 		Window:             window,
@@ -365,6 +385,11 @@ func TestSummaryAllocationSet_RAMEfficiency(t *testing.T) {
 		Window:             window,
 	}
 
+	sas6 = &SummaryAllocationSet{
+		SummaryAllocations: testcase6Map,
+		Window:             window,
+	}
+
 	cases := []struct {
 		name               string
 		testsas            *SummaryAllocationSet
@@ -395,6 +420,11 @@ func TestSummaryAllocationSet_RAMEfficiency(t *testing.T) {
 			testsas:            sas5,
 			expectedEfficiency: 0.65,
 		},
+		{
+			name:               "Check RAMEfficiency in presense of n idle allocation",
+			testsas:            sas6,
+			expectedEfficiency: 0.25,
+		},
 	}
 
 	for _, c := range cases {
@@ -410,10 +440,10 @@ func TestSummaryAllocationSet_RAMEfficiency(t *testing.T) {
 
 func TestSummaryAllocationSet_CPUEfficiency(t *testing.T) {
 	// Generating 6 sample summary allocations for testing
-	var sa1, sa2, sa3, sa4, sa5, sa6 *SummaryAllocation
+	var sa1, sa2, sa3, sa4, sa5, sa6, idlesa *SummaryAllocation
 
 	// Generating accumulated summary allocation sets for testing
-	var sas1, sas2, sas3, sas4, sas5 *SummaryAllocationSet
+	var sas1, sas2, sas3, sas4, sas5, sas6 *SummaryAllocationSet
 
 	window, _ := ParseWindowUTC("7d")
 
@@ -511,6 +541,20 @@ func TestSummaryAllocationSet_CPUEfficiency(t *testing.T) {
 		CPUCost:               0.2,
 	}
 
+	idlesa = &SummaryAllocation{
+		Name: IdleSuffix,
+		Properties: &AllocationProperties{
+			Cluster:   "cluster1",
+			Namespace: "namespace1",
+			Pod:       "pod1",
+			Container: "container7",
+		},
+		Start:   saStart,
+		End:     saEnd,
+		CPUCost: 1.0,
+		RAMCost: 1.0,
+	}
+
 	testcase1Map := map[string]*SummaryAllocation{
 		"cluster1/namespace1/pod1/container1": sa1,
 		"cluster1/namespace1/pod1/container2": sa2,
@@ -537,6 +581,12 @@ func TestSummaryAllocationSet_CPUEfficiency(t *testing.T) {
 		"cluster1/namespace1/pod1/container6": sa6,
 	}
 
+	testcase6Map := map[string]*SummaryAllocation{
+		"cluster1/namespace1/pod1/container1": sa1,
+		"cluster1/namespace1/pod1/container2": sa2,
+		"cluster1/__idle__":                   idlesa,
+	}
+
 	sas1 = &SummaryAllocationSet{
 		SummaryAllocations: testcase1Map,
 		Window:             window,
@@ -562,6 +612,11 @@ func TestSummaryAllocationSet_CPUEfficiency(t *testing.T) {
 		Window:             window,
 	}
 
+	sas6 = &SummaryAllocationSet{
+		SummaryAllocations: testcase6Map,
+		Window:             window,
+	}
+
 	cases := []struct {
 		name               string
 		testsas            *SummaryAllocationSet
@@ -592,6 +647,11 @@ func TestSummaryAllocationSet_CPUEfficiency(t *testing.T) {
 			testsas:            sas5,
 			expectedEfficiency: 0.50,
 		},
+		{
+			name:               "Check CPUEfficiency in presence of idle allocation",
+			testsas:            sas6,
+			expectedEfficiency: 0.30,
+		},
 	}
 
 	for _, c := range cases {
@@ -803,7 +863,7 @@ func TestSummaryAllocationSet_TotalEfficiency(t *testing.T) {
 		{
 			name:               "Check TotalEfficiency with idle cost",
 			testsas:            sas4,
-			expectedEfficiency: 0.20,
+			expectedEfficiency: 0.30,
 		},
 	}