|
|
@@ -54,6 +54,7 @@ const (
|
|
|
APIPricingSource = "Public API"
|
|
|
SpotPricingSource = "Spot Data Feed"
|
|
|
ReservedInstancePricingSource = "Savings Plan, Reserved Instance, and Out-Of-Cluster"
|
|
|
+ FargatePricingSource = "Fargate"
|
|
|
|
|
|
InUseState = "in-use"
|
|
|
AttachedState = "attached"
|
|
|
@@ -61,6 +62,13 @@ const (
|
|
|
AWSHourlyPublicIPCost = 0.005
|
|
|
EKSCapacityTypeLabel = "eks.amazonaws.com/capacityType"
|
|
|
EKSCapacitySpotTypeValue = "SPOT"
|
|
|
+
|
|
|
+ // relevant to pricing url
|
|
|
+ awsPricingBaseURL = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/"
|
|
|
+ awsChinaPricingBaseURL = "https://pricing.cn-north-1.amazonaws.com.cn/offers/v1.0/cn/"
|
|
|
+ pricingCurrentPath = "/current/"
|
|
|
+ pricingIndexFile = "index.json"
|
|
|
+ chinaRegionPrefix = "cn-"
|
|
|
)
|
|
|
|
|
|
var (
|
|
|
@@ -81,7 +89,6 @@ var (
|
|
|
)
|
|
|
|
|
|
func (aws *AWS) PricingSourceStatus() map[string]*models.PricingSource {
|
|
|
-
|
|
|
sources := make(map[string]*models.PricingSource)
|
|
|
|
|
|
sps := &models.PricingSource{
|
|
|
@@ -122,8 +129,19 @@ func (aws *AWS) PricingSourceStatus() map[string]*models.PricingSource {
|
|
|
rps.Available = true
|
|
|
}
|
|
|
sources[ReservedInstancePricingSource] = rps
|
|
|
- return sources
|
|
|
|
|
|
+ fs := &models.PricingSource{
|
|
|
+ Name: FargatePricingSource,
|
|
|
+ Enabled: true,
|
|
|
+ Available: true,
|
|
|
+ }
|
|
|
+ if aws.FargatePricingError != nil {
|
|
|
+ fs.Error = aws.FargatePricingError.Error()
|
|
|
+ fs.Available = false
|
|
|
+ }
|
|
|
+ sources[FargatePricingSource] = fs
|
|
|
+
|
|
|
+ return sources
|
|
|
}
|
|
|
|
|
|
// SpotRefreshDuration represents how much time must pass before we refresh
|
|
|
@@ -174,6 +192,8 @@ type AWS struct {
|
|
|
SavingsPlanDataByInstanceID map[string]*SavingsPlanData
|
|
|
SavingsPlanDataRunning bool
|
|
|
SavingsPlanDataLock sync.RWMutex
|
|
|
+ FargatePricing *FargatePricing
|
|
|
+ FargatePricingError error
|
|
|
ValidPricingKeys map[string]bool
|
|
|
Clientset clustercache.ClusterCache
|
|
|
BaseCPUPrice string
|
|
|
@@ -329,8 +349,10 @@ var OnDemandRateCodesCn = map[string]struct{}{
|
|
|
}
|
|
|
|
|
|
// HourlyRateCode is appended to a node sku
|
|
|
-const HourlyRateCode = "6YS6EN2CT7"
|
|
|
-const HourlyRateCodeCn = "Q7UJUT2CE6"
|
|
|
+const (
|
|
|
+ HourlyRateCode = "6YS6EN2CT7"
|
|
|
+ HourlyRateCodeCn = "Q7UJUT2CE6"
|
|
|
+)
|
|
|
|
|
|
// volTypes are used to map between AWS UsageTypes and
|
|
|
// EBS volume types, as they would appear in K8s storage class
|
|
|
@@ -353,8 +375,10 @@ var volTypes = map[string]string{
|
|
|
"io2": "EBS:VolumeUsage.io2",
|
|
|
}
|
|
|
|
|
|
-var loadedAWSSecret bool = false
|
|
|
-var awsSecret *AWSAccessKey = nil
|
|
|
+var (
|
|
|
+ loadedAWSSecret bool = false
|
|
|
+ awsSecret *AWSAccessKey = nil
|
|
|
+)
|
|
|
|
|
|
// KubeAttrConversion maps the k8s labels for region to an AWS key
|
|
|
func (aws *AWS) KubeAttrConversion(region, instanceType, operatingSystem string) string {
|
|
|
@@ -463,7 +487,7 @@ func (aws *AWS) GetAWSAccessKey() (*AWSAccessKey, error) {
|
|
|
if err != nil {
|
|
|
return nil, fmt.Errorf("error configuring Cloud Provider %s", err)
|
|
|
}
|
|
|
- //Look for service key values in env if not present in config
|
|
|
+ // Look for service key values in env if not present in config
|
|
|
if config.ServiceKeyName == "" {
|
|
|
config.ServiceKeyName = env.GetAWSAccessKeyID()
|
|
|
}
|
|
|
@@ -596,6 +620,7 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*models.CustomPric
|
|
|
}
|
|
|
|
|
|
type awsKey struct {
|
|
|
+ Name string
|
|
|
SpotLabelName string
|
|
|
SpotLabelValue string
|
|
|
Labels map[string]string
|
|
|
@@ -624,7 +649,6 @@ func (k *awsKey) ID() string {
|
|
|
// If the node has a spot label, it will be included in the list
|
|
|
// Otherwise, the list include instance type, operating system, and the region
|
|
|
func (k *awsKey) Features() string {
|
|
|
-
|
|
|
instanceType, _ := util.GetInstanceType(k.Labels)
|
|
|
operatingSystem, _ := util.GetOperatingSystem(k.Labels)
|
|
|
region, _ := util.GetRegion(k.Labels)
|
|
|
@@ -644,6 +668,16 @@ func (k *awsKey) Features() string {
|
|
|
return key
|
|
|
}
|
|
|
|
|
|
+const eksComputeTypeLabel = "eks.amazonaws.com/compute-type"
|
|
|
+
|
|
|
+func (k *awsKey) isFargateNode() bool {
|
|
|
+ v := k.Labels[eksComputeTypeLabel]
|
|
|
+ if v == "fargate" {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
// getUsageType returns the usage type of the instance
|
|
|
// If the instance is a spot instance, it will return PreemptibleType
|
|
|
// Otherwise returns an empty string
|
|
|
@@ -751,6 +785,7 @@ func getStorageClassTypeFrom(provisioner string) string {
|
|
|
// GetKey maps node labels to information needed to retrieve pricing data
|
|
|
func (aws *AWS) GetKey(labels map[string]string, n *clustercache.Node) models.Key {
|
|
|
return &awsKey{
|
|
|
+ Name: n.Name,
|
|
|
SpotLabelName: aws.SpotLabelName,
|
|
|
SpotLabelValue: aws.SpotLabelValue,
|
|
|
Labels: labels,
|
|
|
@@ -770,42 +805,50 @@ func (aws *AWS) ClusterManagementPricing() (string, float64, error) {
|
|
|
return aws.clusterProvisioner, aws.clusterManagementPrice, nil
|
|
|
}
|
|
|
|
|
|
-// Use the pricing data from the current region. Fall back to using all region data if needed.
|
|
|
-func (aws *AWS) getRegionPricing(nodeList []*clustercache.Node) (*http.Response, string, error) {
|
|
|
-
|
|
|
- pricingURL := "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/"
|
|
|
+func getPricingListURL(serviceCode string, nodeList []*clustercache.Node) string {
|
|
|
+ // See https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-the-aws-price-list-bulk-api-fetching-price-list-files-manually.html
|
|
|
region := ""
|
|
|
multiregion := false
|
|
|
+ isChina := false
|
|
|
+
|
|
|
for _, n := range nodeList {
|
|
|
- labels := n.Labels
|
|
|
- currentNodeRegion := ""
|
|
|
- if r, ok := util.GetRegion(labels); ok {
|
|
|
- currentNodeRegion = r
|
|
|
- // Switch to Chinese endpoint for regions with the Chinese prefix
|
|
|
- if strings.HasPrefix(currentNodeRegion, "cn-") {
|
|
|
- pricingURL = "https://pricing.cn-north-1.amazonaws.com.cn/offers/v1.0/cn/AmazonEC2/current/"
|
|
|
- }
|
|
|
- } else {
|
|
|
- multiregion = true // We weren't able to detect the node's region, so pull all data.
|
|
|
+ r, ok := util.GetRegion(n.Labels)
|
|
|
+ if !ok {
|
|
|
+ multiregion = true
|
|
|
break
|
|
|
}
|
|
|
- if region == "" { // We haven't set a region yet
|
|
|
- region = currentNodeRegion
|
|
|
- } else if region != "" && currentNodeRegion != region { // If two nodes have different regions here, we'll need to fetch all pricing data.
|
|
|
+ if strings.HasPrefix(r, chinaRegionPrefix) {
|
|
|
+ isChina = true
|
|
|
+ }
|
|
|
+
|
|
|
+ if region == "" {
|
|
|
+ region = r
|
|
|
+ } else if r != region {
|
|
|
multiregion = true
|
|
|
break
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- // Chinese multiregion endpoint only contains data for Chinese regions and Chinese regions are excluded from other endpoint
|
|
|
+ baseURL := awsPricingBaseURL + serviceCode + pricingCurrentPath
|
|
|
+ if isChina {
|
|
|
+ // Chinese regions are isolated and use a different pricing endpoint
|
|
|
+ baseURL = awsChinaPricingBaseURL + serviceCode + pricingCurrentPath
|
|
|
+ }
|
|
|
+
|
|
|
if region != "" && !multiregion {
|
|
|
- pricingURL += region + "/"
|
|
|
+ baseURL += region + "/"
|
|
|
}
|
|
|
|
|
|
- pricingURL += "index.json"
|
|
|
+ return baseURL + pricingIndexFile
|
|
|
+}
|
|
|
|
|
|
+// Use the pricing data from the current region. Fall back to using all region data if needed.
|
|
|
+func (aws *AWS) getRegionPricing(nodeList []*clustercache.Node) (*http.Response, string, error) {
|
|
|
+ var pricingURL string
|
|
|
if env.GetAWSPricingURL() != "" { // Allow override of pricing URL
|
|
|
pricingURL = env.GetAWSPricingURL()
|
|
|
+ } else {
|
|
|
+ pricingURL = getPricingListURL("AmazonEC2", nodeList)
|
|
|
}
|
|
|
|
|
|
log.Infof("starting download of \"%s\", which is quite large ...", pricingURL)
|
|
|
@@ -947,6 +990,15 @@ func (aws *AWS) DownloadPricingData() error {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ // Initialize fargate pricing if it's not initialized yet
|
|
|
+ if aws.FargatePricing == nil {
|
|
|
+ aws.FargatePricing = NewFargatePricing()
|
|
|
+ aws.FargatePricingError = aws.FargatePricing.Initialize(nodeList)
|
|
|
+ if aws.FargatePricingError != nil {
|
|
|
+ log.Errorf("Failed to initialize fargate pricing: %s", aws.FargatePricingError.Error())
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
aws.ValidPricingKeys = make(map[string]bool)
|
|
|
|
|
|
resp, pricingURL, err := aws.getRegionPricing(nodeList)
|
|
|
@@ -1410,6 +1462,75 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
|
|
|
}, meta, nil
|
|
|
}
|
|
|
|
|
|
+func (aws *AWS) getFargatePod(awsKey *awsKey) (*clustercache.Pod, bool) {
|
|
|
+ pods := aws.Clientset.GetAllPods()
|
|
|
+ for _, pod := range pods {
|
|
|
+ if pod.Spec.NodeName == awsKey.Name {
|
|
|
+ return pod, true
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return nil, false
|
|
|
+}
|
|
|
+
|
|
|
+const (
|
|
|
+ nodeOSLabel = "kubernetes.io/os"
|
|
|
+ nodeArchLabel = "kubernetes.io/arch"
|
|
|
+
|
|
|
+ fargatePodCapacityAnnotation = "CapacityProvisioned"
|
|
|
+)
|
|
|
+
|
|
|
+// e.g. "0.25vCPU 0.5GB"
|
|
|
+var fargatePodCapacityRegex = regexp.MustCompile("^([0-9.]+)vCPU ([0-9.]+)GB$")
|
|
|
+
|
|
|
+func (aws *AWS) createFargateNode(awsKey *awsKey, usageType string) (*models.Node, models.PricingMetadata, error) {
|
|
|
+ if aws.FargatePricing == nil {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("fargate pricing not initialized")
|
|
|
+ }
|
|
|
+ pod, ok := aws.getFargatePod(awsKey)
|
|
|
+ if !ok {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("could not find pod for fargate node %s", awsKey.Name)
|
|
|
+ }
|
|
|
+ capacity := pod.Annotations[fargatePodCapacityAnnotation]
|
|
|
+ match := fargatePodCapacityRegex.FindStringSubmatch(capacity)
|
|
|
+ if len(match) == 0 {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("could not parse pod capacity for fargate node %s", awsKey.Name)
|
|
|
+ }
|
|
|
+
|
|
|
+ vCPU, err := strconv.ParseFloat(match[1], 64)
|
|
|
+ if err != nil {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("could not parse vCPU capacity for fargate node %s: %v", awsKey.Name, err)
|
|
|
+ }
|
|
|
+ memory, err := strconv.ParseFloat(match[2], 64)
|
|
|
+ if err != nil {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("could not parse memory capacity for fargate node %s: %v", awsKey.Name, err)
|
|
|
+ }
|
|
|
+
|
|
|
+ region, ok := util.GetRegion(awsKey.Labels)
|
|
|
+ if !ok {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("could not get region for fargate node %s", awsKey.Name)
|
|
|
+ }
|
|
|
+ nodeOS := awsKey.Labels[nodeOSLabel]
|
|
|
+ nodeArch := awsKey.Labels[nodeArchLabel]
|
|
|
+ hourlyCPU, hourlyRAM, err := aws.FargatePricing.GetHourlyPricing(region, nodeOS, nodeArch)
|
|
|
+ if err != nil {
|
|
|
+ return nil, models.PricingMetadata{}, fmt.Errorf("could not get hourly pricing for fargate node %s: %v", awsKey.Name, err)
|
|
|
+ }
|
|
|
+
|
|
|
+ cost := hourlyCPU*vCPU + hourlyRAM*memory
|
|
|
+ return &models.Node{
|
|
|
+ Cost: strconv.FormatFloat(cost, 'f', -1, 64),
|
|
|
+ VCPU: strconv.FormatFloat(vCPU, 'f', -1, 64),
|
|
|
+ RAM: strconv.FormatFloat(memory, 'f', -1, 64),
|
|
|
+ RAMBytes: strconv.FormatFloat(memory*1024*1024*1024, 'f', -1, 64),
|
|
|
+ VCPUCost: strconv.FormatFloat(hourlyCPU, 'f', -1, 64),
|
|
|
+ RAMCost: strconv.FormatFloat(hourlyRAM, 'f', -1, 64),
|
|
|
+ BaseCPUPrice: aws.BaseCPUPrice,
|
|
|
+ BaseRAMPrice: aws.BaseRAMPrice,
|
|
|
+ BaseGPUPrice: aws.BaseGPUPrice,
|
|
|
+ UsageType: usageType,
|
|
|
+ }, models.PricingMetadata{}, nil
|
|
|
+}
|
|
|
+
|
|
|
// NodePricing takes in a key from GetKey and returns a Node object for use in building the cost model.
|
|
|
func (aws *AWS) NodePricing(k models.Key) (*models.Node, models.PricingMetadata, error) {
|
|
|
aws.DownloadPricingDataLock.RLock()
|
|
|
@@ -1455,6 +1576,9 @@ func (aws *AWS) NodePricing(k models.Key) (*models.Node, models.PricingMetadata,
|
|
|
}, meta, fmt.Errorf("Unable to find any Pricing data for \"%s\"", key)
|
|
|
}
|
|
|
return aws.createNode(terms, usageType, k)
|
|
|
+ } else if awsKey, ok := k.(*awsKey); ok && awsKey.isFargateNode() {
|
|
|
+ // Since Fargate pricing is listed at AmazonECS and is different from AmazonEC2, we handle it separately here
|
|
|
+ return aws.createFargateNode(awsKey, usageType)
|
|
|
} else { // Fall back to base pricing if we can't find the key. Base pricing is handled at the costmodel level.
|
|
|
// we seem to have an issue where this error gets thrown during app start.
|
|
|
// somehow the ValidPricingKeys map is being accessed before all the pricing data has been downloaded
|
|
|
@@ -1464,7 +1588,6 @@ func (aws *AWS) NodePricing(k models.Key) (*models.Node, models.PricingMetadata,
|
|
|
|
|
|
// ClusterInfo returns an object that represents the cluster. TODO: actually return the name of the cluster. Blocked on cluster federation.
|
|
|
func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
|
|
|
-
|
|
|
c, err := awsProvider.GetConfig()
|
|
|
if err != nil {
|
|
|
return nil, err
|
|
|
@@ -1673,7 +1796,6 @@ func (aws *AWS) getAllAddresses() ([]*ec2Types.Address, error) {
|
|
|
a := add // duplicate to avoid pointer to iterator
|
|
|
addresses = append(addresses, &a)
|
|
|
}
|
|
|
-
|
|
|
}
|
|
|
|
|
|
var errs []error
|
|
|
@@ -1939,7 +2061,7 @@ func (aws *AWS) GetOrphanedResources() ([]models.OrphanedResource, error) {
|
|
|
}
|
|
|
|
|
|
func (aws *AWS) findCostForDisk(disk *ec2Types.Volume) (*float64, error) {
|
|
|
- //todo: use AWS pricing from all regions
|
|
|
+ // todo: use AWS pricing from all regions
|
|
|
if disk.AvailabilityZone == nil {
|
|
|
return nil, fmt.Errorf("nil region")
|
|
|
}
|
|
|
@@ -2292,7 +2414,6 @@ type spotInfo struct {
|
|
|
}
|
|
|
|
|
|
func (aws *AWS) parseSpotData(bucket string, prefix string, projectID string, region string) (map[string]*spotInfo, error) {
|
|
|
-
|
|
|
aws.ConfigureAuth() // configure aws api authentication by setting env vars
|
|
|
|
|
|
s3Prefix := projectID
|
|
|
@@ -2455,7 +2576,6 @@ func (aws *AWS) parseSpotData(bucket string, prefix string, projectID string, re
|
|
|
|
|
|
// ApplyReservedInstancePricing TODO
|
|
|
func (aws *AWS) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
|
|
|
-
|
|
|
}
|
|
|
|
|
|
func (aws *AWS) ServiceAccountStatus() *models.ServiceAccountStatus {
|
|
|
@@ -2468,7 +2588,6 @@ func (aws *AWS) CombinedDiscountForNode(instanceType string, isPreemptible bool,
|
|
|
|
|
|
// Regions returns a predefined list of AWS regions
|
|
|
func (aws *AWS) Regions() []string {
|
|
|
-
|
|
|
regionOverrides := env.GetRegionOverrideList()
|
|
|
|
|
|
if len(regionOverrides) > 0 {
|