Merge pull request #1810 from ameijer/alex/kcm-1253-node-cost

[KCM-1253] Provide a2 instance support, fix pricing

pkg/cloud/gcpprovider.go

@@ -638,6 +638,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 				if err != nil {
 					return nil, "", err
 				}
+
 				usageType := strings.ToLower(product.Category.UsageType)
 				instanceType := strings.ToLower(product.Category.ResourceGroup)
 
@@ -741,6 +742,10 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 					}
 				}
 
+				if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "A2 INSTANCE") {
+					instanceType = "a2"
+				}
+
 				if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "COMPUTE OPTIMIZED") {
 					instanceType = "c2standard"
 				}
@@ -774,13 +779,17 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 				}
 
 				for _, region := range product.ServiceRegions {
-					if instanceType == "e2" { // this needs to be done to handle a partial cpu mapping
+					switch instanceType {
+					case "e2":
 						candidateKeys = append(candidateKeys, region+","+"e2micro"+","+usageType)
 						candidateKeys = append(candidateKeys, region+","+"e2small"+","+usageType)
 						candidateKeys = append(candidateKeys, region+","+"e2medium"+","+usageType)
 						candidateKeys = append(candidateKeys, region+","+"e2standard"+","+usageType)
 						candidateKeys = append(candidateKeys, region+","+"e2custom"+","+usageType)
-					} else {
+					case "a2":
+						candidateKeys = append(candidateKeys, region+","+"a2highgpu"+","+usageType)
+						candidateKeys = append(candidateKeys, region+","+"a2megagpu"+","+usageType)
+					default:
 						candidateKey := region + "," + instanceType + "," + usageType
 						candidateKeys = append(candidateKeys, candidateKey)
 					}
@@ -795,12 +804,28 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 					if gpuType != "" {
 						lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
 						var nanos float64
+						var unitsBaseCurrency int
 						if lastRateIndex > -1 && len(product.PricingInfo) > 0 {
 							nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
+							unitsBaseCurrency, err = strconv.Atoi(product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Units)
+							if err != nil {
+								return nil, "", fmt.Errorf("error parsing base unit price for gpu: %w", err)
+							}
 						} else {
 							continue
 						}
-						hourlyPrice := nanos * math.Pow10(-9)
+
+						// as per https://cloud.google.com/billing/v1/how-tos/catalog-api
+						// the hourly price is the whole currency price + the fractional currency price
+						hourlyPrice := (nanos * math.Pow10(-9)) + float64(unitsBaseCurrency)
+
+						// GPUs with an hourly price of 0 are reserved versions of GPUs
+						// (E.g., SKU "2013-37B4-22EA")
+						// and are excluded from cost computations
+						if hourlyPrice == 0 {
+							log.Infof("Excluding reserved GPU SKU #%s", product.SKUID)
+							continue
+						}
 
 						for k, key := range inputKeys {
 							if key.GPUType() == gpuType+","+usageType {
@@ -830,12 +855,20 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						if ok || ok2 {
 							lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
 							var nanos float64
+							var unitsBaseCurrency int
 							if lastRateIndex > -1 && len(product.PricingInfo) > 0 {
 								nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
+								unitsBaseCurrency, err = strconv.Atoi(product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Units)
+								if err != nil {
+									return nil, "", fmt.Errorf("error parsing base unit price for instance: %w", err)
+								}
 							} else {
 								continue
 							}
-							hourlyPrice := nanos * math.Pow10(-9)
+
+							// as per https://cloud.google.com/billing/v1/how-tos/catalog-api
+							// the hourly price is the whole currency price + the fractional currency price
+							hourlyPrice := (nanos * math.Pow10(-9)) + float64(unitsBaseCurrency)
 
 							if hourlyPrice == 0 {
 								continue

pkg/cloud/gcpprovider_test.go

@@ -1,6 +1,9 @@
 package cloud
 
 import (
+	"bytes"
+	"io/ioutil"
+	"reflect"
 	"testing"
 )
 
@@ -112,3 +115,253 @@ func TestGetUsageType(t *testing.T) {
 		}
 	}
 }
+
+// tests basic parsing of GCP pricing API responses
+// Load a reader object on a portion of a GCP api response
+// Confirm that the resting *GCP object contains the correctly parsed pricing info
+func TestParsePage(t *testing.T) {
+
+	gcpSkuString := `
+	{
+		"skus": [
+			{
+				"name": "services/6F81-5844-456A/skus/039F-D0DA-4055",
+				"skuId": "039F-D0DA-4055",
+				"description": "Nvidia Tesla A100 GPU running in Americas",
+				"category": {
+				  "serviceDisplayName": "Compute Engine",
+				  "resourceFamily": "Compute",
+				  "resourceGroup": "GPU",
+				  "usageType": "OnDemand"
+				},
+				"serviceRegions": [
+				  "us-central1",
+				  "us-east1",
+				  "us-west1"
+				],
+				"pricingInfo": [
+				  {
+					"summary": "",
+					"pricingExpression": {
+					  "usageUnit": "h",
+					  "displayQuantity": 1,
+					  "tieredRates": [
+						{
+						  "startUsageAmount": 0,
+						  "unitPrice": {
+							"currencyCode": "USD",
+							"units": "2",
+							"nanos": 933908000
+						  }
+						}
+					  ],
+					  "usageUnitDescription": "hour",
+					  "baseUnit": "s",
+					  "baseUnitDescription": "second",
+					  "baseUnitConversionFactor": 3600
+					},
+					"currencyConversionRate": 1,
+					"effectiveTime": "2023-03-24T10:52:50.681Z"
+				  }
+				],
+				"serviceProviderName": "Google",
+				"geoTaxonomy": {
+				  "type": "MULTI_REGIONAL",
+				  "regions": [
+					"us-central1",
+					"us-east1",
+					"us-west1"
+				  ]
+				}
+			},
+			{
+				"name": "services/6F81-5844-456A/skus/2390-DCAF-DA38",
+				"skuId": "2390-DCAF-DA38",
+				"description": "A2 Instance Ram running in Americas",
+				"category": {
+				  "serviceDisplayName": "Compute Engine",
+				  "resourceFamily": "Compute",
+				  "resourceGroup": "RAM",
+				  "usageType": "OnDemand"
+				},
+				"serviceRegions": [
+				  "us-central1",
+				  "us-east1",
+				  "us-west1"
+				],
+				"pricingInfo": [
+				  {
+					"summary": "",
+					"pricingExpression": {
+					  "usageUnit": "GiBy.h",
+					  "displayQuantity": 1,
+					  "tieredRates": [
+						{
+						  "startUsageAmount": 0,
+						  "unitPrice": {
+							"currencyCode": "USD",
+							"units": "0",
+							"nanos": 4237000
+						  }
+						}
+					  ],
+					  "usageUnitDescription": "gibibyte hour",
+					  "baseUnit": "By.s",
+					  "baseUnitDescription": "byte second",
+					  "baseUnitConversionFactor": 3865470566400
+					},
+					"currencyConversionRate": 1,
+					"effectiveTime": "2023-03-24T10:52:50.681Z"
+				  }
+				],
+				"serviceProviderName": "Google",
+				"geoTaxonomy": {
+				  "type": "MULTI_REGIONAL",
+				  "regions": [
+					"us-central1",
+					"us-east1",
+					"us-west1"
+				  ]
+				}
+			},
+			{
+				"name": "services/6F81-5844-456A/skus/2922-40C5-B19F",
+				"skuId": "2922-40C5-B19F",
+				"description": "A2 Instance Core running in Americas",
+				"category": {
+				  "serviceDisplayName": "Compute Engine",
+				  "resourceFamily": "Compute",
+				  "resourceGroup": "CPU",
+				  "usageType": "OnDemand"
+				},
+				"serviceRegions": [
+				  "us-central1",
+				  "us-east1",
+				  "us-west1"
+				],
+				"pricingInfo": [
+				  {
+					"summary": "",
+					"pricingExpression": {
+					  "usageUnit": "h",
+					  "displayQuantity": 1,
+					  "tieredRates": [
+						{
+						  "startUsageAmount": 0,
+						  "unitPrice": {
+							"currencyCode": "USD",
+							"units": "0",
+							"nanos": 31611000
+						  }
+						}
+					  ],
+					  "usageUnitDescription": "hour",
+					  "baseUnit": "s",
+					  "baseUnitDescription": "second",
+					  "baseUnitConversionFactor": 3600
+					},
+					"currencyConversionRate": 1,
+					"effectiveTime": "2023-03-24T10:52:50.681Z"
+				  }
+				],
+				"serviceProviderName": "Google",
+				"geoTaxonomy": {
+				  "type": "MULTI_REGIONAL",
+				  "regions": [
+					"us-central1",
+					"us-east1",
+					"us-west1"
+				  ]
+				}
+			}
+		],
+			"nextPageToken": "APKCS1HVa0YpwgyTFbqbJ1eGwzKZmsPwLqzMZPTSNia5ck1Hc54Tx_Kz3oBxwSnRIdGVxXoSPdf-XlDpyNBf4QuxKcIEgtrQ1LDLWAgZowI0ns7HjrGta2s="
+		}
+	`
+	reader := ioutil.NopCloser(bytes.NewBufferString(gcpSkuString))
+
+	testGcp := &GCP{}
+
+	inputKeys := map[string]Key{
+		"us-central1,a2highgpu,ondemand,gpu": &gcpKey{
+			Labels: map[string]string{
+				"node.kubernetes.io/instance-type": "a2-highgpu-1g",
+				"cloud.google.com/gke-gpu":         "true",
+				"cloud.google.com/gke-accelerator": "nvidia-tesla-a100",
+				"topology.kubernetes.io/region":    "us-central1",
+			},
+		},
+	}
+
+	pvKeys := map[string]PVKey{}
+
+	actualPrices, token, err := testGcp.parsePage(reader, inputKeys, pvKeys)
+	if err != nil {
+		t.Fatalf("got error parsing page: %v", err)
+	}
+
+	const expectedToken = "APKCS1HVa0YpwgyTFbqbJ1eGwzKZmsPwLqzMZPTSNia5ck1Hc54Tx_Kz3oBxwSnRIdGVxXoSPdf-XlDpyNBf4QuxKcIEgtrQ1LDLWAgZowI0ns7HjrGta2s="
+	if token != expectedToken {
+		t.Fatalf("error parsing GCP next page token, parsed %s but expected %s", token, expectedToken)
+	}
+
+	expectedActualPrices := map[string]*GCPPricing{
+		"us-central1,a2highgpu,ondemand,gpu": &GCPPricing{
+			Name:        "services/6F81-5844-456A/skus/039F-D0DA-4055",
+			SKUID:       "039F-D0DA-4055",
+			Description: "Nvidia Tesla A100 GPU running in Americas",
+			Category: &GCPResourceInfo{
+				ServiceDisplayName: "Compute Engine",
+				ResourceFamily:     "Compute",
+				ResourceGroup:      "GPU",
+				UsageType:          "OnDemand",
+			},
+			ServiceRegions: []string{"us-central1", "us-east1", "us-west1"},
+			PricingInfo: []*PricingInfo{
+				&PricingInfo{
+					Summary: "",
+					PricingExpression: &PricingExpression{
+						UsageUnit:                "h",
+						UsageUnitDescription:     "hour",
+						BaseUnit:                 "s",
+						BaseUnitConversionFactor: 0,
+						DisplayQuantity:          1,
+						TieredRates: []*TieredRates{
+							&TieredRates{
+								StartUsageAmount: 0,
+								UnitPrice: &UnitPriceInfo{
+									CurrencyCode: "USD",
+									Units:        "2",
+									Nanos:        933908000,
+								},
+							},
+						},
+					},
+					CurrencyConversionRate: 1,
+					EffectiveTime:          "2023-03-24T10:52:50.681Z",
+				},
+			},
+			ServiceProviderName: "Google",
+			Node: &Node{
+				VCPUCost:         "0.031611",
+				RAMCost:          "0.004237",
+				UsesBaseCPUPrice: false,
+				GPU:              "1",
+				GPUName:          "nvidia-tesla-a100",
+				GPUCost:          "2.933908",
+			},
+		},
+		"us-central1,a2highgpu,ondemand": &GCPPricing{
+			Node: &Node{
+				VCPUCost:         "0.031611",
+				RAMCost:          "0.004237",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
+	}
+
+	if !reflect.DeepEqual(actualPrices, expectedActualPrices) {
+		t.Fatalf("error parsing GCP prices. parsed %v but expected %v", actualPrices, expectedActualPrices)
+	}
+}

pkg/costmodel/costmodel.go

@@ -1216,7 +1216,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 				}
 			} else { // add case to use default pricing model when API data fails.
 				log.Debugf("No node price or CPUprice found, falling back to default")
-				nodePrice = defaultCPU*cpu + defaultRAM*ram
+				nodePrice = defaultCPU*cpu + defaultRAM*ramGB
 			}
 			if math.IsNaN(nodePrice) {
 				log.Warnf("nodePrice parsed as NaN. Setting to 0.")