OTC Pricing api (#3217)

Signed-off-by: Muhammet Arslan <muhammet.arslan@thryve.de>

pkg/cloud/otc/pricingapi.go

@@ -0,0 +1,78 @@
+package otc
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/opencost/opencost/core/pkg/log"
+)
+
+// Fetches and flattens all product entries across multiple services with pagination
+func (otc *OTC) fetchPaginatedProducts(serviceNames []string) ([]Product, error) {
+	const baseURL = "https://calculator.otc-service.com/de/open-telekom-price-api/"
+	var allProducts []Product
+
+	limitFrom := 0
+	query := buildServiceNameQueryParam(serviceNames)
+
+	for {
+		url := fmt.Sprintf("%s?%s&columns%%5B0%%5D=productIdParameter&columns%%5B1%%5D=opiFlavour&columns%%5B2%%5D=osUnit&columns%%5B3%%5D=vCpu&columns%%5B4%%5D=ram&columns%%5B5%%5D=priceAmount&limitFrom=%d", baseURL, query, limitFrom)
+
+		resp, err := http.Get(url)
+		if err != nil {
+			log.Errorf("Error fetching products from OTC API: %v", err)
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		pageData, stats, err := otc.loadPaginatedResponse(resp)
+		if err != nil {
+			log.Errorf("Error loading paginated response: %v", err)
+			return nil, err
+		}
+
+		for _, products := range pageData {
+			allProducts = append(allProducts, products...)
+		}
+
+		if stats.CurrentPage >= stats.MaxPages {
+			log.Infof("Fetched all products for services: %v", serviceNames)
+			break
+		}
+
+		limitFrom += stats.RecordsPerPage
+	}
+
+	return allProducts, nil
+}
+
+// Parses the OTC API response into a map of service → []Product and pagination stats
+func (otc *OTC) loadPaginatedResponse(resp *http.Response) (map[string][]Product, *OTCStats, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Errorf("Error reading OTC API response: %v", err)
+		return nil, nil, err
+	}
+
+	var raw map[string]map[string]json.RawMessage
+	if err := json.Unmarshal(body, &raw); err != nil {
+		log.Errorf("Error unmarshalling OTC API response: %v", err)
+		return nil, nil, err
+	}
+
+	var data map[string][]Product
+	if err := json.Unmarshal(raw["response"]["result"], &data); err != nil {
+		log.Errorf("Error unmarshalling result section: %v", err)
+		return nil, nil, err
+	}
+
+	var stats OTCStats
+	if err := json.Unmarshal(raw["response"]["stats"], &stats); err != nil {
+		log.Errorf("Error unmarshalling stats section: %v", err)
+		return nil, nil, err
+	}
+
+	return data, &stats, nil
+}

pkg/cloud/otc/provider.go

@@ -1,13 +1,10 @@
 package otc
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/opencost/opencost/core/pkg/clustercache"
 	"github.com/opencost/opencost/core/pkg/log"
@@ -17,41 +14,6 @@ import (
 	"github.com/opencost/opencost/pkg/env"
 )
 
-// OTC node pricing attributes
-type OTCNodeAttributes struct {
-	Type  string // like s2.large.1
-	OS    string // like windows
-	Price string // (in EUR) like 0.023
-	RAM   string // (in GB) like 2
-	VCPU  string // like 8
-}
-
-type OTCPVAttributes struct {
-	Type  string // like vss.ssd
-	Price string // (in EUR/GB/h) like 0.01
-}
-
-// OTC pricing is either for a node, a persistent volume (or a database, network, cluster, ...)
-type OTCPricing struct {
-	NodeAttributes *OTCNodeAttributes
-	PVAttributes   *OTCPVAttributes
-}
-
-// the main provider struct
-type OTC struct {
-	Clientset               clustercache.ClusterCache
-	Pricing                 map[string]*OTCPricing
-	Config                  models.ProviderConfig
-	ClusterRegion           string
-	projectID               string
-	clusterManagementPrice  float64
-	BaseCPUPrice            string
-	BaseRAMPrice            string
-	BaseGPUPrice            string
-	ValidPricingKeys        map[string]bool
-	DownloadPricingDataLock sync.RWMutex
-}
-
 // Kubernetes to OTC OS conversion
 /* Note:
 Kubernetes cannot fill the "kubernetes.io/os" label with the variety that OTC provides
@@ -142,47 +104,6 @@ func (otc *OTC) GetPVKey(pv *clustercache.PersistentVolume, parameters map[strin
 	}
 }
 
-// Takes a resopnse from the otc api and the respective service name as an input
-// and extracts the resulting data into a product slice.
-func (otc *OTC) loadStructFromResponse(resp http.Response, serviceName string) ([]Product, error) {
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	// Unmarshal the first bit of the response.
-	wrapper := make(map[string]map[string]interface{})
-	err = json.Unmarshal(body, &wrapper)
-	if err != nil {
-		return nil, err
-	}
-
-	// Unmarshal the second, more specific, bit of the response.
-	data := make(map[string][]Product)
-	tmp, err := json.Marshal(wrapper["response"]["result"])
-	if err != nil {
-		return nil, err
-	}
-	err = json.Unmarshal(tmp, &data)
-	if err != nil {
-		return nil, err
-	}
-
-	return data[serviceName], nil
-}
-
-// The product (price) data that is fetched from OTC
-//
-// If OsUnit, VCpu and Ram aren't given, the product
-// is a persistent volume, else it's a node.
-type Product struct {
-	OpiFlavour  string `json:"opiFlavour"`
-	OsUnit      string `json:"osUnit,omitempty"`
-	PriceAmount string `json:"priceAmount"`
-	VCpu        string `json:"vCpu,omitempty"`
-	Ram         string `json:"ram,omitempty"`
-}
-
 /*
 Download the pricing data from the OTC API
 
@@ -253,41 +174,19 @@ func (otc *OTC) DownloadPricingData() error {
 	otc.Pricing = make(map[string]*OTCPricing)
 	otc.ValidPricingKeys = make(map[string]bool)
 
-	// Get pricing data from API.
-	nodePricingURL := "https://calculator.otc-service.com/de/open-telekom-price-api/?serviceName=ecs" /* + "&limitMax=200"*/ + "&columns%5B1%5D=opiFlavour" + "&columns%5B2%5D=osUnit" + "&columns%5B3%5D=vCpu" + "&columns%5B4%5D=ram" + "&columns%5B5%5D=priceAmount"
-	pvPricingURL := "https://calculator.otc-service.com/de/open-telekom-price-api/?serviceName%5B0%5D=evs&columns%5B1%5D=opiFlavour&columns%5B2%5D=priceAmount&limitFrom=0&region%5B3%5D=eu-de"
-
-	log.Info("Started downloading OTC pricing data...")
-	resp, err := http.Get(nodePricingURL)
-	if err != nil {
-		return err
-	}
-	pvResp, err := http.Get(pvPricingURL)
-	if err != nil {
-		return err
-	}
-	log.Info("Succesfully downloaded OTC pricing data")
-
-	var products []Product
-
-	nodeProducts, err := otc.loadStructFromResponse(*resp, "ecs")
+	products, err := otc.fetchPaginatedProducts([]string{"ecs", "ecsnoc", "memo", "uhio", "evs"})
 	if err != nil {
+		log.Errorf("Failed to fetch OTC pricing data: %v", err)
 		return err
 	}
-	products = append(products, nodeProducts...)
-	pvProducts, err := otc.loadStructFromResponse(*pvResp, "evs")
-	if err != nil {
-		return err
-	}
-	products = append(products, pvProducts...)
 
 	// convert the otc-reponse product-structs to opencost-compatible node structs
 	const ClusterRegion = "eu-de"
 	for _, product := range products {
 		var productPricing *OTCPricing
 		var key string
-		// if os is empty the product must be a persistent volume
-		if product.OsUnit == "" {
+		// if the product is a persistent volume, it has no osUnit and no vCpu
+		if strings.ToLower(strings.TrimSpace(product.ProductIdParameter)) == "evs" {
 			productPricing = &OTCPricing{
 				PVAttributes: &OTCPVAttributes{
 					Type:  product.OpiFlavour,
@@ -318,6 +217,11 @@ func (otc *OTC) DownloadPricingData() error {
 		otc.ValidPricingKeys[key] = true
 	}
 
+	// debug the whole pricing
+	log.Debugf("OTC Pricing Data: %v", otc.Pricing)
+
+	// exit
+
 	return nil
 }
 
@@ -471,6 +375,7 @@ func (otc *OTC) getClusterName(cfg *models.CustomPricing) string {
 // in the provider's pricing list and return it
 func (otc *OTC) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	pricing, ok := otc.Pricing[pvk.Features()]
+	log.Info("looking for persistent volume pricing for features \"" + pvk.Features() + "\"")
 	if !ok {
 		log.Info("Persistent Volume pricing not found for features \"" + pvk.Features() + "\"")
 		log.Info("continuing with pricing for \"eu-de,vss.ssd\"")

pkg/cloud/otc/types.go

@@ -0,0 +1,58 @@
+package otc
+
+import (
+	"sync"
+
+	"github.com/opencost/opencost/core/pkg/clustercache"
+	"github.com/opencost/opencost/pkg/cloud/models"
+)
+
+type OTCStats struct {
+	CurrentPage    int `json:"currentPage"`
+	MaxPages       int `json:"maxPages"`
+	RecordsPerPage int `json:"recordsPerPage"`
+}
+
+type Product struct {
+	ProductIdParameter string `json:"productIdParameter"`
+	OpiFlavour         string `json:"opiFlavour"`
+	OsUnit             string `json:"osUnit,omitempty"`
+	PriceAmount        string `json:"priceAmount"`
+	VCpu               string `json:"vCpu,omitempty"`
+	Ram                string `json:"ram,omitempty"`
+}
+
+// OTC node pricing attributes
+type OTCNodeAttributes struct {
+	Type  string // like s2.large.1
+	OS    string // like windows
+	Price string // (in EUR) like 0.023
+	RAM   string // (in GB) like 2
+	VCPU  string // like 8
+}
+
+type OTCPVAttributes struct {
+	Type  string // like vss.ssd
+	Price string // (in EUR/GB/h) like 0.01
+}
+
+// OTC pricing is either for a node, a persistent volume (or a database, network, cluster, ...)
+type OTCPricing struct {
+	NodeAttributes *OTCNodeAttributes
+	PVAttributes   *OTCPVAttributes
+}
+
+// the main provider struct
+type OTC struct {
+	Clientset               clustercache.ClusterCache
+	Pricing                 map[string]*OTCPricing
+	Config                  models.ProviderConfig
+	ClusterRegion           string
+	projectID               string
+	clusterManagementPrice  float64
+	BaseCPUPrice            string
+	BaseRAMPrice            string
+	BaseGPUPrice            string
+	ValidPricingKeys        map[string]bool
+	DownloadPricingDataLock sync.RWMutex
+}

pkg/cloud/otc/utils.go

@@ -0,0 +1,18 @@
+package otc
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Builds query string for serviceName[0]=ecs&serviceName[1]=memo&...
+func buildServiceNameQueryParam(serviceNames []string) string {
+	var sb strings.Builder
+	for i, name := range serviceNames {
+		sb.WriteString(fmt.Sprintf("serviceName[%d]=%s", i, name))
+		if i < len(serviceNames)-1 {
+			sb.WriteString("&")
+		}
+	}
+	return sb.String()
+}