Merge branch 'develop' into AjayTripathy-GPU-pricefix

pkg/cloud/awsprovider.go

@@ -55,7 +55,9 @@ const (
 	InUseState    = "in-use"
 	AttachedState = "attached"
 
-	AWSHourlyPublicIPCost = 0.005
+	AWSHourlyPublicIPCost    = 0.005
+	EKSCapacityTypeLabel     = "eks.amazonaws.com/capacityType"
+	EKSCapacitySpotTypeValue = "SPOT"
 )
 
 var (
@@ -63,6 +65,7 @@ var (
 	provIdRx      = regexp.MustCompile("aws:///([^/]+)/([^/]+)")
 	usageTypeRegx = regexp.MustCompile(".*(-|^)(EBS.+)")
 	versionRx     = regexp.MustCompile("^#Version: (\\d+)\\.\\d+$")
+	regionRx      = regexp.MustCompile("([a-z]+-[a-z]+-[0-9])")
 )
 
 func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
@@ -636,6 +639,9 @@ func (k *awsKey) ID() string {
 	return ""
 }
 
+// Features will return a comma seperated list of features for the given node
+// If the node has a spot label, it will be included in the list
+// Otherwise, the list include instance type, operating system, and the region
 func (k *awsKey) Features() string {
 
 	instanceType, _ := util.GetInstanceType(k.Labels)
@@ -643,7 +649,7 @@ func (k *awsKey) Features() string {
 	region, _ := util.GetRegion(k.Labels)
 
 	key := region + "," + instanceType + "," + operatingSystem
-	usageType := PreemptibleType
+	usageType := k.getUsageType(k.Labels)
 	spotKey := key + "," + usageType
 	if l, ok := k.Labels["lifecycle"]; ok && l == "EC2Spot" {
 		return spotKey
@@ -651,9 +657,23 @@ func (k *awsKey) Features() string {
 	if l, ok := k.Labels[k.SpotLabelName]; ok && l == k.SpotLabelValue {
 		return spotKey
 	}
+	if usageType == PreemptibleType {
+		return spotKey
+	}
 	return key
 }
 
+// getUsageType returns the usage type of the instance
+// If the instance is a spot instance, it will return PreemptibleType
+// Otherwise returns an empty string
+func (k *awsKey) getUsageType(labels map[string]string) string {
+	if label, ok := labels[EKSCapacityTypeLabel]; ok && label == EKSCapacitySpotTypeValue {
+		// We currently write out spot instances as "preemptible" in the pricing data, so these need to match
+		return PreemptibleType
+	}
+	return ""
+}
+
 func (aws *AWS) PVPricing(pvk PVKey) (*PV, error) {
 	pricing, ok := aws.Pricing[pvk.Features()]
 	if !ok {
@@ -821,6 +841,7 @@ func (aws *AWS) DownloadPricingData() error {
 
 	inputkeys := make(map[string]bool)
 	for _, n := range nodeList {
+
 		if _, ok := n.Labels["eks.amazonaws.com/nodegroup"]; ok {
 			aws.clusterManagementPrice = 0.10
 			aws.clusterProvisioner = "EKS"
@@ -1684,11 +1705,25 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 				volumeSize = int64(*volume.Size)
 			}
 
+			// This is turning us-east-1a into us-east-1
+			var zone string
+			if volume.AvailabilityZone != nil {
+				zone = *volume.AvailabilityZone
+			}
+			var region, url string
+			region = regionRx.FindString(zone)
+			if region != "" {
+				url = "https://console.aws.amazon.com/ec2/home?region=" + region + "#Volumes:sort=desc:createTime"
+			} else {
+				url = "https://console.aws.amazon.com/ec2/home?#Volumes:sort=desc:createTime"
+			}
+
 			or := OrphanedResource{
 				Kind:        "disk",
-				Region:      *volume.AvailabilityZone,
+				Region:      zone,
 				Size:        &volumeSize,
 				DiskName:    *volume.VolumeId,
+				Url:         url,
 				MonthlyCost: cost,
 			}
 
@@ -1700,9 +1735,23 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 		if aws.isAddressOrphaned(address) {
 			cost := AWSHourlyPublicIPCost * timeutil.HoursPerMonth
 
+			desc := map[string]string{}
+			for _, tag := range address.Tags {
+				if tag.Key == nil {
+					continue
+				}
+				if tag.Value == nil {
+					desc[*tag.Key] = ""
+				} else {
+					desc[*tag.Key] = *tag.Value
+				}
+			}
+
 			or := OrphanedResource{
 				Kind:        "address",
 				Address:     *address.PublicIp,
+				Description: desc,
+				Url:         "http://console.aws.amazon.com/ec2/home?#Addresses",
 				MonthlyCost: &cost,
 			}
 

pkg/cloud/awsprovider_test.go

@@ -0,0 +1,66 @@
+package cloud
+
+import "testing"
+
+func Test_awsKey_getUsageType(t *testing.T) {
+	type fields struct {
+		Labels     map[string]string
+		ProviderID string
+	}
+	type args struct {
+		labels map[string]string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   string
+	}{
+		{
+			// test with no labels should return false
+			name: "Label does not have the capacityType label associated with it",
+			args: args{
+				labels: map[string]string{},
+			},
+			want: "",
+		},
+		{
+			name: "label with a capacityType set to empty string should return empty string",
+			args: args{
+				labels: map[string]string{
+					EKSCapacityTypeLabel: "",
+				},
+			},
+			want: "",
+		},
+		{
+			name: "label with capacityType set to a random value should return empty string",
+			args: args{
+				labels: map[string]string{
+					EKSCapacityTypeLabel: "TEST_ME",
+				},
+			},
+			want: "",
+		},
+		{
+			name: "label with capacityType set to spot should return spot",
+			args: args{
+				labels: map[string]string{
+					EKSCapacityTypeLabel: EKSCapacitySpotTypeValue,
+				},
+			},
+			want: PreemptibleType,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			k := &awsKey{
+				Labels:     tt.fields.Labels,
+				ProviderID: tt.fields.ProviderID,
+			}
+			if got := k.getUsageType(tt.args.labels); got != tt.want {
+				t.Errorf("getUsageType() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/cloud/azureprovider.go

@@ -1282,13 +1282,19 @@ func (az *Azure) GetOrphanedResources() ([]OrphanedResource, error) {
 				diskSize = int64(*d.DiskSizeGB)
 			}
 
+			desc := map[string]string{}
+			for k, v := range d.Tags {
+				if v == nil {
+					desc[k] = ""
+				} else {
+					desc[k] = *v
+				}
+			}
+
 			or := OrphanedResource{
-				Kind:   "disk",
-				Region: diskRegion,
-				Description: map[string]string{
-					"diskState":   string(d.DiskState),
-					"timeCreated": d.TimeCreated.String(),
-				},
+				Kind:        "disk",
+				Region:      diskRegion,
+				Description: desc,
 				Size:        &diskSize,
 				DiskName:    diskName,
 				MonthlyCost: &cost,

pkg/cloud/gcpprovider.go

@@ -468,12 +468,20 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 					return nil, err
 				}
 
+				// GCP gives us description as a string formatted as a map[string]string, so we need to
+				// deconstruct it back into a map[string]string to match the OR struct
+				desc := map[string]string{}
+				if err := json.Unmarshal([]byte(disk.Description), &desc); err != nil {
+					return nil, fmt.Errorf("error converting string to map: %s", err)
+				}
+
 				or := OrphanedResource{
 					Kind:        "disk",
 					Region:      disk.Zone,
-					Description: map[string]string{},
+					Description: desc,
 					Size:        &disk.SizeGb,
 					DiskName:    disk.Name,
+					Url:         disk.SelfLink,
 					MonthlyCost: cost,
 				}
 				orphanedResources = append(orphanedResources, or)
@@ -498,6 +506,7 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 						"type": address.AddressType,
 					},
 					Address:     address.Address,
+					Url:         address.SelfLink,
 					MonthlyCost: &cost,
 				}
 				orphanedResources = append(orphanedResources, or)

pkg/cloud/provider.go

@@ -110,6 +110,7 @@ type OrphanedResource struct {
 	Description map[string]string `json:"description"`
 	Size        *int64            `json:"diskSizeInGB,omitempty"`
 	DiskName    string            `json:"diskName,omitempty"`
+	Url         string            `json:"url"`
 	Address     string            `json:"ipAddress,omitempty"`
 	MonthlyCost *float64          `json:"monthlyCost"`
 }

spec/opencost-specv01.md

@@ -283,6 +283,14 @@ Asset Idle Cost can be calculated by individual assets, groups of assets, cluste
 
 Workload Idle Costs is a cost-weighted measurement of [requested](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container) resources that are unused. Workload Idle Costs can be calculated on any grouping of Kubernetes workloads, e.g. containers, pods, labels, annotations, namespaces, etc.
 
+## Pod States
+
+The state of a pod will affect the ability to assign costs and whether a resource is considered allocated. The OpenCost model does not account for resources allocated to pods with `ImagePullBackOff`.
+
+| State | Cost Allocation | Status |
+|---|---|---|
+| Running | Max (Usage, request) | Implemented |
+| ImagePullBackOff | Request | Currently no charge |
 
 ## Glossary
 

ui/default.nginx.conf

@@ -58,6 +58,7 @@ server {
     add_header ETag "1.96.0";
     listen 9090;
     listen [::]:9090;
+    resolver 127.0.0.1 valid=5s;
     location /healthz {
         return 200 'OK';
     }