use AWS price index for Network Loadbalancer hourly costs (#2771)

* use AWS price index for Network Loadbalancer hourly costs

Signed-off-by: Reinier Schoof <reinier@skoef.nl>

* move AWS fixtures for testing in separate files

This way it is way easier to check if the JSON is still valid and it doesn't
clutter the code so much.

Signed-off-by: Reinier Schoof <reinier@skoef.nl>

---------

Signed-off-by: Reinier Schoof <reinier@skoef.nl>
Co-authored-by: Cliff Colvin <ccolvin@kubecost.com>

pkg/cloud/aws/provider.go

@@ -250,6 +250,7 @@ type AWSProduct struct {
 type AWSProductAttributes struct {
 	Location        string `json:"location"`
 	RegionCode      string `json:"regionCode"`
+	Operation       string `json:"operation"`
 	InstanceType    string `json:"instanceType"`
 	Memory          string `json:"memory"`
 	Storage         string `json:"storage"`
@@ -302,14 +303,15 @@ type AWSCurrencyCode struct {
 
 // AWSProductTerms represents the full terms of the product
 type AWSProductTerms struct {
-	Sku      string        `json:"sku"`
-	OnDemand *AWSOfferTerm `json:"OnDemand"`
-	Reserved *AWSOfferTerm `json:"Reserved"`
-	Memory   string        `json:"memory"`
-	Storage  string        `json:"storage"`
-	VCpu     string        `json:"vcpu"`
-	GPU      string        `json:"gpu"` // GPU represents the number of GPU on the instance
-	PV       *models.PV    `json:"pv"`
+	Sku          string               `json:"sku"`
+	OnDemand     *AWSOfferTerm        `json:"OnDemand"`
+	Reserved     *AWSOfferTerm        `json:"Reserved"`
+	Memory       string               `json:"memory"`
+	Storage      string               `json:"storage"`
+	VCpu         string               `json:"vcpu"`
+	GPU          string               `json:"gpu"` // GPU represents the number of GPU on the instance
+	PV           *models.PV           `json:"pv"`
+	LoadBalancer *models.LoadBalancer `json:"load_balancer"`
 }
 
 // ClusterIdEnvVar is the environment variable in which one can manually set the ClusterId
@@ -1039,6 +1041,19 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 					skusToKeys[product.Sku] = key
 					aws.ValidPricingKeys[key] = true
 					aws.ValidPricingKeys[spotKey] = true
+				} else if strings.Contains(product.Attributes.UsageType, "LoadBalancerUsage") && product.Attributes.Operation == "LoadBalancing:Network" {
+					// since the costmodel is only using services of type LoadBalancer
+					// (and not ingresses controlled by AWS load balancer controller)
+					// we can safely filter for Network load balancers only
+					productTerms := &AWSProductTerms{
+						Sku:          product.Sku,
+						LoadBalancer: &models.LoadBalancer{},
+					}
+					// there is no spot pricing for load balancers
+					key := product.Attributes.RegionCode + ",LoadBalancerUsage"
+					aws.Pricing[key] = productTerms
+					skusToKeys[product.Sku] = key
+					aws.ValidPricingKeys[key] = true
 				}
 			}
 		}
@@ -1080,7 +1095,9 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 					spotKey := key + ",preemptible"
 					if ok {
 						aws.Pricing[key].OnDemand = offerTerm
-						aws.Pricing[spotKey].OnDemand = offerTerm
+						if _, ok := aws.Pricing[spotKey]; ok {
+							aws.Pricing[spotKey].OnDemand = offerTerm
+						}
 						var cost string
 						if _, isMatch := OnDemandRateCodes[offerTerm.OfferTermCode]; isMatch {
 							priceDimensionKey := strings.Join([]string{sku.(string), offerTerm.OfferTermCode, HourlyRateCode}, ".")
@@ -1135,6 +1152,13 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 							hourlyPrice := costFloat / 730
 
 							aws.Pricing[key].PV.Cost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
+						} else if strings.Contains(key, "LoadBalancerUsage") {
+							costFloat, err := strconv.ParseFloat(cost, 64)
+							if err != nil {
+								return err
+							}
+
+							aws.Pricing[key].LoadBalancer.Cost = costFloat
 						}
 					}
 
@@ -1205,21 +1229,20 @@ func (aws *AWS) NetworkPricing() (*models.Network, error) {
 }
 
 func (aws *AWS) LoadBalancerPricing() (*models.LoadBalancer, error) {
-	fffrc := 0.025
-	afrc := 0.010
-	lbidc := 0.008
+	// TODO: determine key based on function arguments
+	// this is something that should be changed in the Provider interface
 
-	numForwardingRules := 1.0
-	dataIngressGB := 0.0
+	key := aws.ClusterRegion + ",LoadBalancerUsage"
 
-	var totalCost float64
-	if numForwardingRules < 5 {
-		totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB
-	} else {
-		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
+	// set default price
+	hourlyCost := 0.025
+	// use price index when available
+	if terms, ok := aws.Pricing[key]; ok {
+		hourlyCost = terms.LoadBalancer.Cost
 	}
+
 	return &models.LoadBalancer{
-		Cost: totalCost,
+		Cost: hourlyCost,
 	}, nil
 }
 

pkg/cloud/aws/provider_test.go

@@ -1,7 +1,6 @@
 package aws
 
 import (
-	"bytes"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -174,197 +173,19 @@ func Test_PricingData_Regression(t *testing.T) {
 func Test_populate_pricing(t *testing.T) {
 	awsTest := AWS{
 		ValidPricingKeys: map[string]bool{},
+		ClusterRegion:    "us-east-2",
 	}
 	inputkeys := map[string]bool{
 		"us-east-2,m5.large,linux": true,
 	}
-	// Case 0
-	awsUSEastString := `
-	{
-		"formatVersion" : "v1.0",
-		"disclaimer" : "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
-		"offerCode" : "AmazonEC2",
-		"version" : "20230322145651",
-		"publicationDate" : "2023-03-22T14:56:51Z",
-		"products" : {
-			"8D49XP354UEYTHGM" : {
-				"sku" : "8D49XP354UEYTHGM",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "US East (Ohio)",
-				  "locationType" : "AWS Region",
-				  "instanceType" : "m5.large",
-				  "currentGeneration" : "Yes",
-				  "instanceFamily" : "General purpose",
-				  "vcpu" : "2",
-				  "physicalProcessor" : "Intel Xeon Platinum 8175",
-				  "clockSpeed" : "3.1 GHz",
-				  "memory" : "8 GiB",
-				  "storage" : "EBS only",
-				  "networkPerformance" : "Up to 10 Gigabit",
-				  "processorArchitecture" : "64-bit",
-				  "tenancy" : "Shared",
-				  "operatingSystem" : "Linux",
-				  "licenseModel" : "No License required",
-				  "usagetype" : "USE2-BoxUsage:m5.large",
-				  "operation" : "RunInstances",
-				  "availabilityzone" : "NA",
-				  "capacitystatus" : "Used",
-				  "classicnetworkingsupport" : "false",
-				  "dedicatedEbsThroughput" : "Up to 2120 Mbps",
-				  "ecu" : "10",
-				  "enhancedNetworkingSupported" : "Yes",
-				  "gpuMemory" : "NA",
-				  "intelAvxAvailable" : "Yes",
-				  "intelAvx2Available" : "Yes",
-				  "intelTurboAvailable" : "Yes",
-				  "marketoption" : "OnDemand",
-				  "normalizationSizeFactor" : "4",
-				  "preInstalledSw" : "NA",
-				  "processorFeatures" : "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
-				  "regionCode" : "us-east-2",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "vpcnetworkingsupport" : "true"
-				}
-			},
-			"9ZEEN7WWWQKAG292" : {
-				"sku" : "9ZEEN7WWWQKAG292",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "US East (Ohio)",
-				  "locationType" : "AWS Region",
-				  "instanceType" : "p3.8xlarge",
-				  "currentGeneration" : "Yes",
-				  "instanceFamily" : "GPU instance",
-				  "vcpu" : "32",
-				  "physicalProcessor" : "Intel Xeon E5-2686 v4 (Broadwell)",
-				  "clockSpeed" : "2.3 GHz",
-				  "memory" : "244 GiB",
-				  "storage" : "EBS only",
-				  "networkPerformance" : "10 Gigabit",
-				  "processorArchitecture" : "64-bit",
-				  "tenancy" : "Shared",
-				  "operatingSystem" : "Windows",
-				  "licenseModel" : "Bring your own license",
-				  "usagetype" : "USE2-BoxUsage:p3.8xlarge",
-				  "operation" : "RunInstances:0800",
-				  "availabilityzone" : "NA",
-				  "capacitystatus" : "Used",
-				  "classicnetworkingsupport" : "false",
-				  "dedicatedEbsThroughput" : "7000 Mbps",
-				  "ecu" : "97",
-				  "enhancedNetworkingSupported" : "Yes",
-				  "gpu" : "4",
-				  "gpuMemory" : "NA",
-				  "intelAvxAvailable" : "Yes",
-				  "intelAvx2Available" : "Yes",
-				  "intelTurboAvailable" : "Yes",
-				  "marketoption" : "OnDemand",
-				  "normalizationSizeFactor" : "64",
-				  "preInstalledSw" : "NA",
-				  "processorFeatures" : "Intel AVX; Intel AVX2; Intel Turbo",
-				  "regionCode" : "us-east-2",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "vpcnetworkingsupport" : "true"
-				}
-			},
-			"M6UGCCQ3CDJQAA37" : {
-				"sku" : "M6UGCCQ3CDJQAA37",
-				"productFamily" : "Storage",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "US East (Ohio)",
-				  "locationType" : "AWS Region",
-				  "storageMedia" : "SSD-backed",
-				  "volumeType" : "General Purpose",
-				  "maxVolumeSize" : "16 TiB",
-				  "maxIopsvolume" : "16000",
-				  "maxThroughputvolume" : "1000 MiB/s",
-				  "usagetype" : "USE2-EBS:VolumeUsage.gp3",
-				  "operation" : "",
-				  "regionCode" : "us-east-2",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "volumeApiName" : "gp3"
-				}
-			}
-		},
-		"terms" : {
-			"OnDemand" : {
-				"M6UGCCQ3CDJQAA37" : {
-					"M6UGCCQ3CDJQAA37.JRTCKXETXF" : {
-					  "offerTermCode" : "JRTCKXETXF",
-					  "sku" : "M6UGCCQ3CDJQAA37",
-					  "effectiveDate" : "2023-03-01T00:00:00Z",
-					  "priceDimensions" : {
-						"M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7" : {
-						  "rateCode" : "M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7",
-						  "description" : "$0.08 per GB-month of General Purpose (gp3) provisioned storage - US East (Ohio)",
-						  "beginRange" : "0",
-						  "endRange" : "Inf",
-						  "unit" : "GB-Mo",
-						  "pricePerUnit" : {
-							"USD" : "0.0800000000"
-						  },
-						  "appliesTo" : [ ]
-						}
-					  },
-					  "termAttributes" : { }
-					}
-				},
-				"9ZEEN7WWWQKAG292" : {
-					"9ZEEN7WWWQKAG292.JRTCKXETXF" : {
-					  "offerTermCode" : "JRTCKXETXF",
-					  "sku" : "9ZEEN7WWWQKAG292",
-					  "effectiveDate" : "2023-03-01T00:00:00Z",
-					  "priceDimensions" : {
-						"9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7" : {
-						  "rateCode" : "9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7",
-						  "description" : "$12.24 per On Demand Windows BYOL p3.8xlarge Instance Hour",
-						  "beginRange" : "0",
-						  "endRange" : "Inf",
-						  "unit" : "Hrs",
-						  "pricePerUnit" : {
-							"USD" : "12.2400000000"
-						  },
-						  "appliesTo" : [ ]
-						}
-					  },
-					  "termAttributes" : { }
-					}
-				},
-				"8D49XP354UEYTHGM" : {
-					"8D49XP354UEYTHGM.MZU6U2429S" : {
-					  "offerTermCode" : "MZU6U2429S",
-					  "sku" : "8D49XP354UEYTHGM",
-					  "effectiveDate" : "2019-01-01T00:00:00Z",
-					  "priceDimensions" : {
-						"8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U" : {
-						  "rateCode" : "8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U",
-						  "description" : "Upfront Fee",
-						  "unit" : "Quantity",
-						  "pricePerUnit" : {
-							"USD" : "1161"
-						  },
-						  "appliesTo" : [ ]
-						},
-					  },
-					  "termAttributes" : {
-						"LeaseContractLength" : "3yr",
-						"OfferingClass" : "convertible",
-						"PurchaseOption" : "All Upfront"
-					  }
-					}
-				}
-			}
-		},
-		"attributesList" : { }
+
+	fixture, err := os.Open("testdata/pricing-us-east-2.json")
+	if err != nil {
+		t.Fatalf("failed to load pricing fixture: %s", err)
 	}
-	`
 
 	testResponse := http.Response{
-		Body: io.NopCloser(bytes.NewBufferString(awsUSEastString)),
+		Body: io.NopCloser(fixture),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",
@@ -411,9 +232,17 @@ func Test_populate_pricing(t *testing.T) {
 		VCpu:    "2",
 		GPU:     "",
 		OnDemand: &AWSOfferTerm{
-			Sku:             "",
-			OfferTermCode:   "",
-			PriceDimensions: nil,
+			Sku:           "8D49XP354UEYTHGM",
+			OfferTermCode: "MZU6U2429S",
+			PriceDimensions: map[string]*AWSRateCode{
+				"8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U": {
+					Unit: "Quantity",
+					PricePerUnit: AWSCurrencyCode{
+						USD: "1161",
+						CNY: "",
+					},
+				},
+			},
 		},
 	}
 
@@ -424,9 +253,37 @@ func Test_populate_pricing(t *testing.T) {
 		VCpu:    "2",
 		GPU:     "",
 		OnDemand: &AWSOfferTerm{
-			Sku:             "",
-			OfferTermCode:   "",
-			PriceDimensions: nil,
+			Sku:           "8D49XP354UEYTHGM",
+			OfferTermCode: "MZU6U2429S",
+			PriceDimensions: map[string]*AWSRateCode{
+				"8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U": {
+					Unit: "Quantity",
+					PricePerUnit: AWSCurrencyCode{
+						USD: "1161",
+						CNY: "",
+					},
+				},
+			},
+		},
+	}
+
+	expectedProdTermsLoadbalancer := &AWSProductTerms{
+		Sku: "Y9RYMSE644KDSV4S",
+		OnDemand: &AWSOfferTerm{
+			Sku:           "Y9RYMSE644KDSV4S",
+			OfferTermCode: "JRTCKXETXF",
+			PriceDimensions: map[string]*AWSRateCode{
+				"Y9RYMSE644KDSV4S.JRTCKXETXF.6YS6EN2CT7": {
+					Unit: "Hrs",
+					PricePerUnit: AWSCurrencyCode{
+						USD: "0.0225000000",
+						CNY: "",
+					},
+				},
+			},
+		},
+		LoadBalancer: &models.LoadBalancer{
+			Cost: 0.0225,
 		},
 	}
 
@@ -435,161 +292,30 @@ func Test_populate_pricing(t *testing.T) {
 		"us-east-2,EBS:VolumeUsage.gp3,preemptible": expectedProdTermsDisk,
 		"us-east-2,m5.large,linux":                  expectedProdTermsInstanceOndemand,
 		"us-east-2,m5.large,linux,preemptible":      expectedProdTermsInstanceSpot,
+		"us-east-2,LoadBalancerUsage":               expectedProdTermsLoadbalancer,
 	}
 
 	if !reflect.DeepEqual(expectedPricing, awsTest.Pricing) {
-		t.Fatalf("expected parsed pricing did not match actual parsed result (us-east-1)")
+		t.Fatalf("expected parsed pricing did not match actual parsed result (us-east-2)")
+	}
+
+	lbPricing, _ := awsTest.LoadBalancerPricing()
+	if lbPricing.Cost != 0.0225 {
+		t.Fatalf("expected loadbalancer pricing of 0.0225 but got %f (us-east-2)", lbPricing.Cost)
 	}
 
 	// Case 1 - Only accept `"marketoption":"OnDemand"`
 	inputkeysCase1 := map[string]bool{
 		"us-east-1,p4d.24xlarge,linux": true,
 	}
-	pricingCase1 := `
-	{
-		"formatVersion" : "v1.0",
-		"disclaimer" : "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
-		"offerCode" : "AmazonEC2",
-		"version" : "20240528203522",
-		"publicationDate" : "2024-05-28T20:35:22Z",
-		"products" : {
-			"H7NGEAC6UEHNTKSJ" : {
-				"sku" : "H7NGEAC6UEHNTKSJ",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-					"servicecode" : "AmazonEC2",
-					"location" : "US East (N. Virginia)",
-					"locationType" : "AWS Region",
-					"instanceType" : "p4d.24xlarge",
-					"currentGeneration" : "Yes",
-					"instanceFamily" : "GPU instance",
-					"vcpu" : "96",
-					"physicalProcessor" : "Intel Xeon Platinum 8275L",
-					"clockSpeed" : "3 GHz",
-					"memory" : "1152 GiB",
-					"storage" : "8 x 1000 SSD",
-					"networkPerformance" : "400 Gigabit",
-					"processorArchitecture" : "64-bit",
-					"tenancy" : "Shared",
-					"operatingSystem" : "Linux",
-					"licenseModel" : "No License required",
-					"usagetype" : "BoxUsage:p4d.24xlarge",
-					"operation" : "RunInstances",
-					"availabilityzone" : "NA",
-					"capacitystatus" : "Used",
-					"classicnetworkingsupport" : "false",
-					"dedicatedEbsThroughput" : "19000 Mbps",
-					"ecu" : "345",
-					"enhancedNetworkingSupported" : "No",
-					"gpu" : "8",
-					"gpuMemory" : "NA",
-					"intelAvxAvailable" : "Yes",
-					"intelAvx2Available" : "Yes",
-					"intelTurboAvailable" : "Yes",
-					"marketoption" : "OnDemand",
-					"normalizationSizeFactor" : "192",
-					"preInstalledSw" : "NA",
-					"processorFeatures" : "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
-					"regionCode" : "us-east-1",
-					"servicename" : "Amazon Elastic Compute Cloud",
-					"vpcnetworkingsupport" : "true"
-				}
-			},
-			"YSXJGN78QTXNVGDQ" : {
-				"sku" : "YSXJGN78QTXNVGDQ",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-					"servicecode" : "AmazonEC2",
-					"location" : "US East (N. Virginia)",
-					"locationType" : "AWS Region",
-					"instanceType" : "p4d.24xlarge",
-					"currentGeneration" : "Yes",
-					"instanceFamily" : "GPU instance",
-					"vcpu" : "96",
-					"physicalProcessor" : "Intel Xeon Platinum 8275L",
-					"clockSpeed" : "3 GHz",
-					"memory" : "1152 GiB",
-					"storage" : "8 x 1000 SSD",
-					"networkPerformance" : "400 Gigabit",
-					"processorArchitecture" : "64-bit",
-					"tenancy" : "Shared",
-					"operatingSystem" : "Linux",
-					"licenseModel" : "No License required",
-					"usagetype" : "BoxUsage:p4d.24xlarge",
-					"operation" : "RunInstances:CB",
-					"availabilityzone" : "NA",
-					"capacitystatus" : "Used",
-					"classicnetworkingsupport" : "false",
-					"dedicatedEbsThroughput" : "19000 Mbps",
-					"ecu" : "345",
-					"enhancedNetworkingSupported" : "No",
-					"gpu" : "8",
-					"gpuMemory" : "NA",
-					"intelAvxAvailable" : "Yes",
-					"intelAvx2Available" : "Yes",
-					"intelTurboAvailable" : "Yes",
-					"marketoption" : "CapacityBlock",
-					"normalizationSizeFactor" : "192",
-					"preInstalledSw" : "NA",
-					"processorFeatures" : "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
-					"regionCode" : "us-east-1",
-					"servicename" : "Amazon Elastic Compute Cloud",
-					"vpcnetworkingsupport" : "true"
-				}
-			}
-		},
-		"terms" : {
-			"OnDemand" : {
-				"H7NGEAC6UEHNTKSJ" : {
-					"H7NGEAC6UEHNTKSJ.JRTCKXETXF" : {
-						"offerTermCode" : "JRTCKXETXF",
-						"sku" : "H7NGEAC6UEHNTKSJ",
-						"effectiveDate" : "2024-05-01T00:00:00Z",
-						"priceDimensions" : {
-							"H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7" : {
-								"rateCode" : "H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7",
-								"description" : "$32.7726 per On Demand Linux p4d.24xlarge Instance Hour",
-								"beginRange" : "0",
-								"endRange" : "Inf",
-								"unit" : "Hrs",
-								"pricePerUnit" : {
-									"USD" : "32.7726000000"
-								},
-								"appliesTo" : [ ]
-							}
-						},
-						"termAttributes" : { }
-					}
-				},
-				"YSXJGN78QTXNVGDQ" : {
-					"YSXJGN78QTXNVGDQ.JRTCKXETXF" : {
-						"offerTermCode" : "JRTCKXETXF",
-						"sku" : "YSXJGN78QTXNVGDQ",
-						"effectiveDate" : "2024-05-01T00:00:00Z",
-						"priceDimensions" : {
-							"YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7" : {
-							"rateCode" : "YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7",
-							"description" : "$0.00 per Capacity Block Linux p4d.24xlarge Instance Hour",
-							"beginRange" : "0",
-							"endRange" : "Inf",
-							"unit" : "Hrs",
-							"pricePerUnit" : {
-								"USD" : "0.0000000000"
-							},
-							"appliesTo" : [ ]
-						}
-					},
-					"termAttributes" : { }
-					}
-				},
-			}
-		},
-		"attributesList" : { }
+
+	fixture, err = os.Open("testdata/pricing-us-east-1.json")
+	if err != nil {
+		t.Fatalf("failed to load pricing fixture: %s", err)
 	}
-	`
 
 	testResponseCase1 := http.Response{
-		Body: io.NopCloser(bytes.NewBufferString(pricingCase1)),
+		Body: io.NopCloser(fixture),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",
@@ -634,68 +360,17 @@ func Test_populate_pricing(t *testing.T) {
 	}
 
 	// Case 2
-	awsCnString := `
-	{
-		"formatVersion" : "v1.0",
-		"disclaimer" : "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://www.amazonaws.cn.",
-		"offerCode" : "AmazonEC2",
-		"version" : "20230314154740",
-		"publicationDate" : "2023-03-14T15:47:40Z",
-		"products" : {
-			"R83VXG9NAPDASEGN" : {
-				"sku" : "R83VXG9NAPDASEGN",
-				"productFamily" : "Storage",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "China (Ningxia)",
-				  "locationType" : "AWS Region",
-				  "storageMedia" : "SSD-backed",
-				  "volumeType" : "General Purpose",
-				  "maxVolumeSize" : "16 TiB",
-				  "maxIopsvolume" : "16000",
-				  "maxThroughputvolume" : "1000 MiB/s",
-				  "usagetype" : "CNW1-EBS:VolumeUsage.gp3",
-				  "operation" : "",
-				  "regionCode" : "cn-northwest-1",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "volumeApiName" : "gp3"
-				}
-			}
-		},
-		"terms" : {
-			"OnDemand" : {
-			  "R83VXG9NAPDASEGN" : {
-				"R83VXG9NAPDASEGN.5Y9WH78GDR" : {
-				  "offerTermCode" : "5Y9WH78GDR",
-				  "sku" : "R83VXG9NAPDASEGN",
-				  "effectiveDate" : "2023-03-01T00:00:00Z",
-				  "priceDimensions" : {
-					"R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6" : {
-					  "rateCode" : "R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6",
-					  "description" : "0.5312 CNY per GB-month of General Purpose (gp3) provisioned storage - China (Ningxia)",
-					  "beginRange" : "0",
-					  "endRange" : "Inf",
-					  "unit" : "GB-Mo",
-					  "pricePerUnit" : {
-						"CNY" : "0.5312000000"
-					  },
-					  "appliesTo" : [ ]
-					}
-				  },
-				  "termAttributes" : { }
-				}
-			  }
-			}
-	    },
-	  "attributesList" : { }
-	}
-	`
 	awsTest = AWS{
 		ValidPricingKeys: map[string]bool{},
 	}
 
+	fixture, err = os.Open("testdata/pricing-cn-northwest-1.json")
+	if err != nil {
+		t.Fatalf("failed to load pricing fixture: %s", err)
+	}
+
 	testResponse = http.Response{
-		Body: io.NopCloser(bytes.NewBufferString(awsCnString)),
+		Body: io.NopCloser(fixture),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",

pkg/cloud/aws/testdata/pricing-cn-northwest-1.json

@@ -0,0 +1,54 @@
+{
+  "formatVersion": "v1.0",
+  "disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://www.amazonaws.cn.",
+  "offerCode": "AmazonEC2",
+  "version": "20230314154740",
+  "publicationDate": "2023-03-14T15:47:40Z",
+  "products": {
+    "R83VXG9NAPDASEGN": {
+      "sku": "R83VXG9NAPDASEGN",
+      "productFamily": "Storage",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "China (Ningxia)",
+        "locationType": "AWS Region",
+        "storageMedia": "SSD-backed",
+        "volumeType": "General Purpose",
+        "maxVolumeSize": "16 TiB",
+        "maxIopsvolume": "16000",
+        "maxThroughputvolume": "1000 MiB/s",
+        "usagetype": "CNW1-EBS:VolumeUsage.gp3",
+        "operation": "",
+        "regionCode": "cn-northwest-1",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "volumeApiName": "gp3"
+      }
+    }
+  },
+  "terms": {
+    "OnDemand": {
+      "R83VXG9NAPDASEGN": {
+        "R83VXG9NAPDASEGN.5Y9WH78GDR": {
+          "offerTermCode": "5Y9WH78GDR",
+          "sku": "R83VXG9NAPDASEGN",
+          "effectiveDate": "2023-03-01T00:00:00Z",
+          "priceDimensions": {
+            "R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6": {
+              "rateCode": "R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6",
+              "description": "0.5312 CNY per GB-month of General Purpose (gp3) provisioned storage - China (Ningxia)",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "GB-Mo",
+              "pricePerUnit": {
+                "CNY": "0.5312000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      }
+    }
+  },
+  "attributesList": {}
+}

pkg/cloud/aws/testdata/pricing-us-east-1.json

@@ -0,0 +1,140 @@
+{
+  "formatVersion": "v1.0",
+  "disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
+  "offerCode": "AmazonEC2",
+  "version": "20240528203522",
+  "publicationDate": "2024-05-28T20:35:22Z",
+  "products": {
+    "H7NGEAC6UEHNTKSJ": {
+      "sku": "H7NGEAC6UEHNTKSJ",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (N. Virginia)",
+        "locationType": "AWS Region",
+        "instanceType": "p4d.24xlarge",
+        "currentGeneration": "Yes",
+        "instanceFamily": "GPU instance",
+        "vcpu": "96",
+        "physicalProcessor": "Intel Xeon Platinum 8275L",
+        "clockSpeed": "3 GHz",
+        "memory": "1152 GiB",
+        "storage": "8 x 1000 SSD",
+        "networkPerformance": "400 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Linux",
+        "licenseModel": "No License required",
+        "usagetype": "BoxUsage:p4d.24xlarge",
+        "operation": "RunInstances",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "19000 Mbps",
+        "ecu": "345",
+        "enhancedNetworkingSupported": "No",
+        "gpu": "8",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "OnDemand",
+        "normalizationSizeFactor": "192",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
+        "regionCode": "us-east-1",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    },
+    "YSXJGN78QTXNVGDQ": {
+      "sku": "YSXJGN78QTXNVGDQ",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (N. Virginia)",
+        "locationType": "AWS Region",
+        "instanceType": "p4d.24xlarge",
+        "currentGeneration": "Yes",
+        "instanceFamily": "GPU instance",
+        "vcpu": "96",
+        "physicalProcessor": "Intel Xeon Platinum 8275L",
+        "clockSpeed": "3 GHz",
+        "memory": "1152 GiB",
+        "storage": "8 x 1000 SSD",
+        "networkPerformance": "400 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Linux",
+        "licenseModel": "No License required",
+        "usagetype": "BoxUsage:p4d.24xlarge",
+        "operation": "RunInstances:CB",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "19000 Mbps",
+        "ecu": "345",
+        "enhancedNetworkingSupported": "No",
+        "gpu": "8",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "CapacityBlock",
+        "normalizationSizeFactor": "192",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
+        "regionCode": "us-east-1",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    }
+  },
+  "terms": {
+    "OnDemand": {
+      "H7NGEAC6UEHNTKSJ": {
+        "H7NGEAC6UEHNTKSJ.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "H7NGEAC6UEHNTKSJ",
+          "effectiveDate": "2024-05-01T00:00:00Z",
+          "priceDimensions": {
+            "H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$32.7726 per On Demand Linux p4d.24xlarge Instance Hour",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "32.7726000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      },
+      "YSXJGN78QTXNVGDQ": {
+        "YSXJGN78QTXNVGDQ.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "YSXJGN78QTXNVGDQ",
+          "effectiveDate": "2024-05-01T00:00:00Z",
+          "priceDimensions": {
+            "YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$0.00 per Capacity Block Linux p4d.24xlarge Instance Hour",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "0.0000000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      }
+    }
+  },
+  "attributesList": {}
+}

pkg/cloud/aws/testdata/pricing-us-east-2.json

@@ -0,0 +1,217 @@
+{
+  "formatVersion": "v1.0",
+  "disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
+  "offerCode": "AmazonEC2",
+  "version": "20230322145651",
+  "publicationDate": "2023-03-22T14:56:51Z",
+  "products": {
+    "8D49XP354UEYTHGM": {
+      "sku": "8D49XP354UEYTHGM",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "instanceType": "m5.large",
+        "currentGeneration": "Yes",
+        "instanceFamily": "General purpose",
+        "vcpu": "2",
+        "physicalProcessor": "Intel Xeon Platinum 8175",
+        "clockSpeed": "3.1 GHz",
+        "memory": "8 GiB",
+        "storage": "EBS only",
+        "networkPerformance": "Up to 10 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Linux",
+        "licenseModel": "No License required",
+        "usagetype": "USE2-BoxUsage:m5.large",
+        "operation": "RunInstances",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "Up to 2120 Mbps",
+        "ecu": "10",
+        "enhancedNetworkingSupported": "Yes",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "OnDemand",
+        "normalizationSizeFactor": "4",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    },
+    "9ZEEN7WWWQKAG292": {
+      "sku": "9ZEEN7WWWQKAG292",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "instanceType": "p3.8xlarge",
+        "currentGeneration": "Yes",
+        "instanceFamily": "GPU instance",
+        "vcpu": "32",
+        "physicalProcessor": "Intel Xeon E5-2686 v4 (Broadwell)",
+        "clockSpeed": "2.3 GHz",
+        "memory": "244 GiB",
+        "storage": "EBS only",
+        "networkPerformance": "10 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Windows",
+        "licenseModel": "Bring your own license",
+        "usagetype": "USE2-BoxUsage:p3.8xlarge",
+        "operation": "RunInstances:0800",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "7000 Mbps",
+        "ecu": "97",
+        "enhancedNetworkingSupported": "Yes",
+        "gpu": "4",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "OnDemand",
+        "normalizationSizeFactor": "64",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel Turbo",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    },
+    "M6UGCCQ3CDJQAA37": {
+      "sku": "M6UGCCQ3CDJQAA37",
+      "productFamily": "Storage",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "storageMedia": "SSD-backed",
+        "volumeType": "General Purpose",
+        "maxVolumeSize": "16 TiB",
+        "maxIopsvolume": "16000",
+        "maxThroughputvolume": "1000 MiB/s",
+        "usagetype": "USE2-EBS:VolumeUsage.gp3",
+        "operation": "",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "volumeApiName": "gp3"
+      }
+    },
+    "Y9RYMSE644KDSV4S": {
+      "sku": "Y9RYMSE644KDSV4S",
+      "productFamily": "Load Balancer-Network",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "group": "ELB:Balancer",
+        "groupDescription": "LoadBalancer hourly usage by Network Load Balancer",
+        "usagetype": "USE2-LoadBalancerUsage",
+        "operation": "LoadBalancing:Network",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud"
+      }
+    }
+  },
+  "terms": {
+    "OnDemand": {
+      "M6UGCCQ3CDJQAA37": {
+        "M6UGCCQ3CDJQAA37.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "M6UGCCQ3CDJQAA37",
+          "effectiveDate": "2023-03-01T00:00:00Z",
+          "priceDimensions": {
+            "M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$0.08 per GB-month of General Purpose (gp3) provisioned storage - US East (Ohio)",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "GB-Mo",
+              "pricePerUnit": {
+                "USD": "0.0800000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      },
+      "9ZEEN7WWWQKAG292": {
+        "9ZEEN7WWWQKAG292.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "9ZEEN7WWWQKAG292",
+          "effectiveDate": "2023-03-01T00:00:00Z",
+          "priceDimensions": {
+            "9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$12.24 per On Demand Windows BYOL p3.8xlarge Instance Hour",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "12.2400000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      },
+      "8D49XP354UEYTHGM": {
+        "8D49XP354UEYTHGM.MZU6U2429S": {
+          "offerTermCode": "MZU6U2429S",
+          "sku": "8D49XP354UEYTHGM",
+          "effectiveDate": "2019-01-01T00:00:00Z",
+          "priceDimensions": {
+            "8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U": {
+              "rateCode": "8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U",
+              "description": "Upfront Fee",
+              "unit": "Quantity",
+              "pricePerUnit": {
+                "USD": "1161"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {
+            "LeaseContractLength": "3yr",
+            "OfferingClass": "convertible",
+            "PurchaseOption": "All Upfront"
+          }
+        }
+      },
+      "Y9RYMSE644KDSV4S": {
+        "Y9RYMSE644KDSV4S.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "Y9RYMSE644KDSV4S",
+          "effectiveDate": "2024-05-01T00:00:00Z",
+          "priceDimensions": {
+            "Y9RYMSE644KDSV4S.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "Y9RYMSE644KDSV4S.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$0.0225 per Network LoadBalancer-hour (or partial hour)",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "0.0225000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      }
+    },
+    "attributesList": {}
+  }
+}