Pricing Module Updates (dedup) & Update of USD/CNY data (#4013)

modules/pricing/public/aws/awspricingsource.go

@@ -33,7 +33,9 @@ func (p *AWSPricingSource) GetPricing() (*pricing.PricingSet, error) {
 		PersistentVolumePricing: []*pricing.PersistentVolumePricing{},
 	}
 	skuToNodeKey := make(map[string]nodeKey)
+	seenNodeKeys := make(map[nodeKey]struct{})
 	skuToVolumeKey := make(map[string]volumeKey)
+	seenVolumeKeys := make(map[volumeKey]struct{})
 
 	var productCount, termCount int
 	const logInterval = 50000
@@ -55,23 +57,48 @@ func (p *AWSPricingSource) GetPricing() (*pricing.PricingSet, error) {
 			return
 		}
 
-		// Handle EC2 instances
+		// Handle EC2 instances.
+		// We only want the base Linux on-demand price:
+		//   - UsageType must be a BoxUsage (compute hour charge)
+		//   - CapacityStatus must be "Used" (not a capacity reservation)
+		//   - MarketOption must be "OnDemand" (not Spot)
+		//   - OperatingSystem must be Linux (or not returned by API)
+		//   - PreInstalledSw must be "NA" (no paid software bundle)
+		// All of these can appear empty when the API omits the field, so we
+		// treat empty as "unknown" and require the affirmative value where it
+		// matters, except OperatingSystem where empty/NA is acceptable.
 		if (strings.HasPrefix(attr.UsageType, "BoxUsage") || strings.Contains(attr.UsageType, "-BoxUsage")) &&
 			(attr.CapacityStatus == "Used" || attr.CapacityStatus == "") &&
 			(attr.MarketOption == "OnDemand" || attr.MarketOption == "") {
 
+			// Skip non-Linux operating systems; allow empty/NA (field may not be returned).
 			if attr.OperatingSystem != "" && attr.OperatingSystem != "NA" && attr.OperatingSystem != "Linux" {
 				return
 			}
 
+			// Skip software bundles (SQL Server, etc.); allow empty (field may not be returned).
+			if attr.PreInstalledSw != "" && attr.PreInstalledSw != "NA" {
+				return
+			}
+
+			// Skip capacity reservations; allow empty (field may not be returned).
+			if attr.CapacityStatus != "" && attr.CapacityStatus != "Used" {
+				return
+			}
+
 			if attr.RegionCode == "" || attr.InstanceType == "" {
 				return
 			}
 
-			skuToNodeKey[product.Sku] = nodeKey{
+			nk := nodeKey{
 				Region:       attr.RegionCode,
 				InstanceType: attr.InstanceType,
 			}
+			if _, seen := seenNodeKeys[nk]; seen {
+				return
+			}
+			seenNodeKeys[nk] = struct{}{}
+			skuToNodeKey[product.Sku] = nk
 			return
 		}
 
@@ -94,11 +121,16 @@ func (p *AWSPricingSource) GetPricing() (*pricing.PricingSet, error) {
 				return
 			}
 
-			skuToVolumeKey[product.Sku] = volumeKey{
+			vk := volumeKey{
 				Region:     attr.RegionCode,
 				VolumeType: volumeType,
 				UsageType:  usageTypeNoRegion,
 			}
+			if _, seen := seenVolumeKeys[vk]; seen {
+				return
+			}
+			seenVolumeKeys[vk] = struct{}{}
+			skuToVolumeKey[product.Sku] = vk
 		}
 	}
 

modules/pricing/public/azure/azurepricingsource.go

@@ -51,6 +51,7 @@ func (a *AzurePricingSource) GetPricing() (*pricing.PricingSet, error) {
 	// Fetch VM pricing
 	url := a.buildVMURL()
 	pageCount := 0
+	seenNodes := make(map[nodeKey]struct{})
 
 	for url != "" {
 		resp, err := azureHTTPClient.Get(url)
@@ -67,7 +68,7 @@ func (a *AzurePricingSource) GetPricing() (*pricing.PricingSet, error) {
 			return nil, fmt.Errorf("PricingSource (Azure): unexpected status %d on VM page %d: %s", resp.StatusCode, pageCount, string(body))
 		}
 
-		next, err := a.parseVMPage(resp.Body, ps)
+		next, err := a.parseVMPage(resp.Body, ps, seenNodes)
 		closeErr := resp.Body.Close()
 		if closeErr != nil {
 			log.Warnf("failed to close response body: %v", closeErr)
@@ -138,7 +139,7 @@ func (a *AzurePricingSource) buildDiskURL() string {
 	return u
 }
 
-func (a *AzurePricingSource) parseVMPage(body io.Reader, ps *pricing.PricingSet) (nextURL string, err error) {
+func (a *AzurePricingSource) parseVMPage(body io.Reader, ps *pricing.PricingSet, seen map[nodeKey]struct{}) (nextURL string, err error) {
 	data, err := io.ReadAll(body)
 	if err != nil {
 		return "", fmt.Errorf("reading response body: %w", err)
@@ -154,6 +155,12 @@ func (a *AzurePricingSource) parseVMPage(body io.Reader, ps *pricing.PricingSet)
 			continue
 		}
 
+		nk := nodeKey{Region: item.ArmRegionName, InstanceType: item.ArmSkuName}
+		if _, ok := seen[nk]; ok {
+			continue
+		}
+		seen[nk] = struct{}{}
+
 		nodePricing := &pricing.NodePricing{
 			Properties: pricing.NodePricingProperties{
 				Provider:     cloud.ProviderAzure,
@@ -232,8 +239,13 @@ func (a *AzurePricingSource) includeItem(item AzurePricingAttributes) bool {
 		return false
 	}
 
+	// The Azure API appends an exact suffix to SkuName for non-on-demand rows.
+	// We only want on-demand Linux pricing, so reject Spot and Low Priority.
 	skuLower := strings.ToLower(item.SkuName)
-	return !strings.Contains(skuLower, "low priority")
+	if strings.HasSuffix(skuLower, " spot") {
+		return false
+	}
+	return !strings.HasSuffix(skuLower, " low priority")
 }
 
 // includeDiskItem filters disk items to include only managed disks.

modules/pricing/public/azure/azurepricingsource_test.go

@@ -102,6 +102,16 @@ func TestIncludeItem(t *testing.T) {
 			},
 			expected: false,
 		},
+		{
+			name: "Spot - excluded",
+			item: AzurePricingAttributes{
+				ArmSkuName:    "Standard_D2s_v3",
+				ArmRegionName: "eastus",
+				ProductName:   "Virtual Machines Dsv3 Series",
+				SkuName:       "D2s v3 Spot",
+			},
+			expected: false,
+		},
 		{
 			name: "Low priority - excluded",
 			item: AzurePricingAttributes{

modules/pricing/public/azure/types.go

@@ -6,6 +6,12 @@ import (
 	"github.com/opencost/opencost/core/pkg/pricing"
 )
 
+// nodeKey is used internally to deduplicate VM pricing entries.
+type nodeKey struct {
+	Region       string
+	InstanceType string
+}
+
 // mapAzureDiskType maps Azure disk SKU names to VolumeType constants
 func mapAzureDiskType(skuName string) pricing.VolumeType {
 	skuLower := strings.ToLower(skuName)

modules/pricing/public/httpclient/httpclient.go

@@ -17,10 +17,10 @@ const (
 
 // retryTransport is an http.RoundTripper that retries requests on 429 and 503s
 type retryTransport struct {
-	wrapped     http.RoundTripper
-	maxRetries  int
-	baseWait    time.Duration
-	maxWait     time.Duration
+	wrapped    http.RoundTripper
+	maxRetries int
+	baseWait   time.Duration
+	maxWait    time.Duration
 }
 
 func (t *retryTransport) RoundTrip(req *http.Request) (*http.Response, error) {