provider.go 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421
  1. package alibaba
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "os"
  7. "regexp"
  8. "strconv"
  9. "strings"
  10. "sync"
  11. "github.com/aliyun/alibaba-cloud-sdk-go/sdk"
  12. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
  13. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/signers"
  14. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
  15. "github.com/opencost/opencost/core/pkg/clustercache"
  16. coreenv "github.com/opencost/opencost/core/pkg/env"
  17. "github.com/opencost/opencost/core/pkg/log"
  18. "github.com/opencost/opencost/core/pkg/opencost"
  19. "github.com/opencost/opencost/core/pkg/util/fileutil"
  20. "github.com/opencost/opencost/core/pkg/util/json"
  21. "github.com/opencost/opencost/core/pkg/util/stringutil"
  22. "github.com/opencost/opencost/pkg/cloud/models"
  23. "github.com/opencost/opencost/pkg/cloud/utils"
  24. "github.com/opencost/opencost/pkg/env"
  25. "golang.org/x/exp/slices"
  26. )
  27. const (
  28. ALIBABA_ECS_PRODUCT_CODE = "ecs"
  29. ALIBABA_ECS_VERSION = "2014-05-26"
  30. ALIBABA_ECS_DOMAIN = "ecs.aliyuncs.com"
  31. ALIBABA_DESCRIBE_PRICE_API_ACTION = "DescribePrice"
  32. ALIBABA_DESCRIBE_DISK_API_ACTION = "DescribeDisks"
  33. ALIBABA_INSTANCE_RESOURCE_TYPE = "instance"
  34. ALIBABA_DISK_RESOURCE_TYPE = "disk"
  35. ALIBABA_PAY_AS_YOU_GO_BILLING = "Pay-As-You-Go"
  36. ALIBABA_SUBSCRIPTION_BILLING = "Subscription"
  37. ALIBABA_PREEMPTIBLE_BILLING = "Preemptible"
  38. ALIBABA_OPTIMIZE_KEYWORD = "optimize"
  39. ALIBABA_NON_OPTIMIZE_KEYWORD = "nonoptimize"
  40. ALIBABA_HOUR_PRICE_UNIT = "Hour"
  41. ALIBABA_MONTH_PRICE_UNIT = "Month"
  42. ALIBABA_YEAR_PRICE_UNIT = "Year"
  43. ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE = "unknown"
  44. ALIBABA_NOT_SUPPORTED_INSTANCE_FAMILY_TYPE = "unsupported"
  45. ALIBABA_DISK_CLOUD_ESSD_CATEGORY = "cloud_essd"
  46. ALIBABA_DISK_CLOUD_CATEGORY = "cloud"
  47. ALIBABA_DATA_DISK_CATEGORY = "data"
  48. ALIBABA_SYSTEM_DISK_CATEGORY = "system"
  49. ALIBABA_DATA_DISK_PREFIX = "DataDisk"
  50. ALIBABA_PV_CLOUD_DISK_TYPE = "CloudDisk"
  51. ALIBABA_PV_NAS_TYPE = "NAS"
  52. ALIBABA_PV_OSS_TYPE = "OSS"
  53. ALIBABA_DEFAULT_DATADISK_SIZE = "2000"
  54. ALIBABA_DISK_TOPOLOGY_REGION_LABEL = "topology.diskplugin.csi.alibabacloud.com/region"
  55. ALIBABA_DISK_TOPOLOGY_ZONE_LABEL = "topology.diskplugin.csi.alibabacloud.com/zone"
  56. )
  57. var (
  58. // Regular expression to get the numerical value of PV suffix with GiB from *v1.PersistentVolume.
  59. sizeRegEx = regexp.MustCompile("(.*?)Gi")
  60. )
  61. // Variable to keep track of instance families that fail in DescribePrice API due improper defaulting of systemDisk if the information is not available
  62. var alibabaDefaultToCloudEssd = []string{"g6e", "r6e"}
  63. var alibabaRegions = []string{
  64. "cn-qingdao",
  65. "cn-beijing",
  66. "cn-zhangjiakou",
  67. "cn-huhehaote",
  68. "cn-wulanchabu",
  69. "cn-hangzhou",
  70. "cn-shanghai",
  71. "cn-nanjing",
  72. "cn-shenzhen",
  73. "cn-heyuan",
  74. "cn-guangzhou",
  75. "cn-fuzhou",
  76. "cn-wuhan-lr",
  77. "cn-chengdu",
  78. "cn-hongkong",
  79. "ap-northeast-1",
  80. "ap-northeast-2",
  81. "ap-southeast-1",
  82. "ap-southeast-2",
  83. "ap-southeast-3",
  84. "ap-southeast-6",
  85. "ap-southeast-5",
  86. "ap-south-1",
  87. "ap-southeast-7",
  88. "us-east-1",
  89. "us-west-1",
  90. "eu-west-1",
  91. "me-east-1",
  92. "me-central-1",
  93. "eu-central-1",
  94. }
  95. // AlibabaInfo contains configuration for Alibaba's CUR integration
  96. // Deprecated: v1.104 Use BOAConfiguration instead
  97. type AlibabaInfo struct {
  98. AlibabaClusterRegion string `json:"ClusterRegion"`
  99. AlibabaServiceKeyName string `json:"serviceKeyName"`
  100. AlibabaServiceKeySecret string `json:"serviceKeySecret"`
  101. AlibabaAccountID string `json:"accountID"`
  102. }
  103. // IsEmpty returns true if all fields in config are empty, false if not.
  104. func (ai *AlibabaInfo) IsEmpty() bool {
  105. return ai.AlibabaClusterRegion == "" &&
  106. ai.AlibabaServiceKeyName == "" &&
  107. ai.AlibabaServiceKeySecret == "" &&
  108. ai.AlibabaAccountID == ""
  109. }
  110. // AlibabaAccessKey holds Alibaba credentials parsing from the service-key.json file.
  111. // Deprecated: v1.104 Use AccessKey instead
  112. type AlibabaAccessKey struct {
  113. AccessKeyID string `json:"alibaba_access_key_id"`
  114. SecretAccessKey string `json:"alibaba_secret_access_key"`
  115. }
  116. // Slim Version of k8s disk assigned to a node or PV.
  117. type SlimK8sDisk struct {
  118. DiskType string
  119. RegionID string
  120. PriceUnit string
  121. SizeInGiB string
  122. DiskCategory string
  123. PerformanceLevel string
  124. ProviderID string
  125. StorageClass string
  126. }
  127. func NewSlimK8sDisk(diskType, regionID, priceUnit, diskCategory, performanceLevel, providerID, storageClass, sizeInGiB string) *SlimK8sDisk {
  128. return &SlimK8sDisk{
  129. DiskType: diskType,
  130. RegionID: regionID,
  131. PriceUnit: priceUnit,
  132. SizeInGiB: sizeInGiB,
  133. DiskCategory: diskCategory,
  134. PerformanceLevel: performanceLevel,
  135. ProviderID: providerID,
  136. StorageClass: storageClass,
  137. }
  138. }
  139. // Slim version of a k8s v1.node just to pass along the object of this struct instead of constant getting the labels from within v1.Node & unit testing.
  140. type SlimK8sNode struct {
  141. InstanceType string
  142. RegionID string
  143. PriceUnit string
  144. MemorySizeInKiB string // TO-DO : Possible to convert to float?
  145. IsIoOptimized bool
  146. OSType string
  147. ProviderID string
  148. SystemDisk *SlimK8sDisk
  149. InstanceTypeFamily string // Bug in DescribePrice, doesn't default to enhanced type correctly and you get an error in DescribePrice to get around need the family of the InstanceType.
  150. }
  151. func NewSlimK8sNode(instanceType, regionID, priceUnit, memorySizeInKiB, osType, providerID, instanceTypeFamily string, isIOOptimized bool, systemDiskInfo *SlimK8sDisk) *SlimK8sNode {
  152. return &SlimK8sNode{
  153. InstanceType: instanceType,
  154. RegionID: regionID,
  155. PriceUnit: priceUnit,
  156. MemorySizeInKiB: memorySizeInKiB,
  157. IsIoOptimized: isIOOptimized,
  158. OSType: osType,
  159. SystemDisk: systemDiskInfo,
  160. ProviderID: providerID,
  161. InstanceTypeFamily: instanceTypeFamily,
  162. }
  163. }
  164. // AlibabaNodeAttributes represents metadata about the Node in its pricing information.
  165. // Basic Attributes needed atleast to get the key, Some attributes from k8s Node response
  166. // be populated directly into *Node object.
  167. type AlibabaNodeAttributes struct {
  168. // InstanceType represents the type of instance.
  169. InstanceType string `json:"instanceType"`
  170. // MemorySizeInKiB represents the size of memory of instance.
  171. MemorySizeInKiB string `json:"memorySizeInKiB"`
  172. // IsIoOptimized represents the if instance is I/O optimized.
  173. IsIoOptimized bool `json:"isIoOptimized"`
  174. // OSType represents the OS installed in the Instance.
  175. OSType string `json:"osType"`
  176. // SystemDiskCategory represents the exact category of the system disk attached to the node.
  177. SystemDiskCategory string `json:"systemDiskCategory"`
  178. // SystemDiskSizeInGiB represents the size of the system disk attached to the node.
  179. SystemDiskSizeInGiB string `json:"systemDiskSizeInGiB"`
  180. // SystemDiskPerformanceLevel represents the performance level of the system disk attached to the node.
  181. SystemDiskPerformanceLevel string `json:"systemPerformanceLevel"`
  182. }
  183. func NewAlibabaNodeAttributes(node *SlimK8sNode) *AlibabaNodeAttributes {
  184. if node == nil {
  185. return nil
  186. }
  187. var diskCategory, sizeInGiB, performanceLevel string
  188. if node.SystemDisk != nil {
  189. diskCategory = node.SystemDisk.DiskCategory
  190. sizeInGiB = node.SystemDisk.SizeInGiB
  191. performanceLevel = node.SystemDisk.PerformanceLevel
  192. }
  193. return &AlibabaNodeAttributes{
  194. InstanceType: node.InstanceType,
  195. MemorySizeInKiB: node.MemorySizeInKiB,
  196. IsIoOptimized: node.IsIoOptimized,
  197. OSType: node.OSType,
  198. SystemDiskCategory: diskCategory,
  199. SystemDiskSizeInGiB: sizeInGiB,
  200. SystemDiskPerformanceLevel: performanceLevel,
  201. }
  202. }
  203. // AlibabaPVAttributes represents metadata the PV in its pricing information.
  204. // Basic Attributes needed atleast to get the keys. Some attributes from k8s PV response
  205. // be populated directly into *PV object.
  206. type AlibabaPVAttributes struct {
  207. // PVType can be Cloud Disk, NetWork Attached Storage(NAS) or Object Storage Service (OSS).
  208. // Represents the way the PV was attached
  209. PVType string `json:"pvType"`
  210. // PVSubType represent the sub category of PVType. This is Data in case of Cloud Disk.
  211. PVSubType string `json:"pvSubType"`
  212. // Example for PVCategory with cloudDisk PVType are cloud, cloud_efficiency, cloud_ssd,
  213. // ephemeral_ssd and cloud_essd. If not present returns empty.
  214. PVCategory string `json:"pvCategory"`
  215. // Example for PerformanceLevel with cloudDisk PVType are PL0,PL1,PL2 &PL3. If not present returns empty.
  216. PVPerformanceLevel string `json:"performanceLevel"`
  217. // The Size of the PV in terms of GiB
  218. SizeInGiB string `json:"sizeInGiB"`
  219. }
  220. // TO-Do: next iteration of Alibaba provider support NetWork Attached Storage(NAS) and Object Storage Service (OSS type PVs).
  221. // Currently defaulting to cloudDisk with provision to add work in future.
  222. func NewAlibabaPVAttributes(disk *SlimK8sDisk) *AlibabaPVAttributes {
  223. if disk == nil {
  224. return nil
  225. }
  226. return &AlibabaPVAttributes{
  227. PVType: ALIBABA_PV_CLOUD_DISK_TYPE,
  228. PVSubType: disk.DiskType,
  229. PVCategory: disk.DiskCategory,
  230. PVPerformanceLevel: disk.PerformanceLevel,
  231. SizeInGiB: disk.SizeInGiB,
  232. }
  233. }
  234. // Stage 1 support will be Pay-As-You-Go with HourlyPrice equal to TradePrice with PriceUnit as Hour
  235. // TO-DO: Subscription and Premptible support, Information can be gathered from describing instance for subscription type
  236. // and spotprice can be gather from DescribeSpotPriceHistory API.
  237. // TO-DO: how would you calculate hourly price for subscription type, is it PRICE_YEARLY/HOURS_IN_THE_YEAR|MONTH?
  238. type AlibabaPricingDetails struct {
  239. // Represents hourly price for the given Alibaba cloud Product.
  240. HourlyPrice float32 `json:"hourlyPrice"`
  241. // Represents the unit in which Alibaba Product is billed can be Hour, Month or Year based on the billingMethod.
  242. PriceUnit string `json:"priceUnit"`
  243. // Original Price paid to acquire the Alibaba Product.
  244. TradePrice float32 `json:"tradePrice"`
  245. // Represents the currency unit of the price for billing Alibaba Product.
  246. CurrencyCode string `json:"currencyCode"`
  247. }
  248. func NewAlibabaPricingDetails(hourlyPrice float32, priceUnit string, tradePrice float32, currencyCode string) *AlibabaPricingDetails {
  249. return &AlibabaPricingDetails{
  250. HourlyPrice: hourlyPrice,
  251. PriceUnit: priceUnit,
  252. TradePrice: tradePrice,
  253. CurrencyCode: currencyCode,
  254. }
  255. }
  256. // AlibabaPricingTerms can have three types of supported billing method Pay-As-You-Go, Subscription and Premptible
  257. type AlibabaPricingTerms struct {
  258. BillingMethod string `json:"billingMethod"`
  259. PricingDetails *AlibabaPricingDetails `json:"pricingDetails"`
  260. }
  261. func NewAlibabaPricingTerms(billingMethod string, pricingDetails *AlibabaPricingDetails) *AlibabaPricingTerms {
  262. return &AlibabaPricingTerms{
  263. BillingMethod: billingMethod,
  264. PricingDetails: pricingDetails,
  265. }
  266. }
  267. // Alibaba Pricing struct carry the Attributes and pricing information for Node or PV
  268. type AlibabaPricing struct {
  269. NodeAttributes *AlibabaNodeAttributes
  270. PVAttributes *AlibabaPVAttributes
  271. PricingTerms *AlibabaPricingTerms
  272. Node *models.Node
  273. PV *models.PV
  274. }
  275. // Alibaba cloud's Provider struct
  276. type Alibaba struct {
  277. // Data to store Alibaba cloud's pricing struct, key in the map represents exact match to
  278. // node.features() or pv.features for easy lookup
  279. Pricing map[string]*AlibabaPricing
  280. // Lock Needed to provide thread safe
  281. DownloadPricingDataLock sync.RWMutex
  282. Clientset clustercache.ClusterCache
  283. Config models.ProviderConfig
  284. ServiceAccountChecks *models.ServiceAccountChecks
  285. ClusterAccountId string
  286. ClusterRegion string
  287. // The following fields are unexported because of avoiding any leak of secrets of these keys.
  288. // Alibaba Access key used specifically in signer interface used to sign API calls
  289. accessKey *credentials.AccessKeyCredential
  290. // Map of regionID to sdk.client to call API for that region
  291. clients map[string]*sdk.Client
  292. }
  293. // GetAlibabaAccessKey return the Access Key used to interact with the Alibaba cloud, if not set it
  294. // set it first by looking at env variables else load it from secret files.
  295. func (alibaba *Alibaba) GetAlibabaAccessKey() (*credentials.AccessKeyCredential, error) {
  296. if alibaba.accessKeyisLoaded() {
  297. return alibaba.accessKey, nil
  298. }
  299. config, err := alibaba.GetConfig()
  300. if err != nil {
  301. return nil, fmt.Errorf("error getting the default config for Alibaba Cloud provider: %w", err)
  302. }
  303. if config.AlibabaServiceKeyName == "" {
  304. config.AlibabaServiceKeyName = env.GetAlibabaAccessKeyID()
  305. }
  306. if config.AlibabaServiceKeySecret == "" {
  307. config.AlibabaServiceKeySecret = env.GetAlibabaAccessKeySecret()
  308. }
  309. if config.AlibabaServiceKeyName == "" && config.AlibabaServiceKeySecret == "" {
  310. log.Debugf("missing service key values for Alibaba cloud integration attempting to use service account integration")
  311. err := alibaba.loadAlibabaAuthSecretAndSetEnv(true)
  312. if err != nil {
  313. return nil, fmt.Errorf("unable to set the Alibaba Cloud key/secret from config file %w", err)
  314. }
  315. config.AlibabaServiceKeyName = env.GetAlibabaAccessKeyID()
  316. config.AlibabaServiceKeySecret = env.GetAlibabaAccessKeySecret()
  317. }
  318. if config.AlibabaServiceKeyName == "" && config.AlibabaServiceKeySecret == "" {
  319. return nil, fmt.Errorf("failed to get the access key for the current alibaba account")
  320. }
  321. // At this point either user is using the alibaba key and secret from secret passed in helm config if not he will use the secret that is passed in custom pricing
  322. // There's no check at this time for if the custom pricing key and secret is valid and that's on the user else there will be errors recorded.
  323. // Key and secret passed in config will supersede key and secret passed while installing Closed source helm chart.
  324. alibaba.accessKey = &credentials.AccessKeyCredential{AccessKeyId: config.AlibabaServiceKeyName, AccessKeySecret: config.AlibabaServiceKeySecret}
  325. return alibaba.accessKey, nil
  326. }
  327. func (alibaba *Alibaba) GetAlibabaCloudInfo() (*AlibabaInfo, error) {
  328. config, err := alibaba.GetConfig()
  329. if err != nil {
  330. return nil, fmt.Errorf("could not retrieve AlibabaCloudInfo %s", err)
  331. }
  332. aak, err := alibaba.GetAlibabaAccessKey()
  333. if err != nil {
  334. return nil, err
  335. }
  336. return &AlibabaInfo{
  337. AlibabaClusterRegion: config.AlibabaClusterRegion,
  338. AlibabaServiceKeyName: aak.AccessKeyId,
  339. AlibabaServiceKeySecret: aak.AccessKeySecret,
  340. AlibabaAccountID: config.ProjectID,
  341. }, nil
  342. }
  343. // DownloadPricingData satisfies the provider interface and downloads the prices for Node instances and PVs.
  344. func (alibaba *Alibaba) DownloadPricingData() error {
  345. alibaba.DownloadPricingDataLock.Lock()
  346. defer alibaba.DownloadPricingDataLock.Unlock()
  347. var aak *credentials.AccessKeyCredential
  348. var err error
  349. if !alibaba.accessKeyisLoaded() {
  350. aak, err = alibaba.GetAlibabaAccessKey()
  351. if err != nil {
  352. return fmt.Errorf("unable to get the access key information: %w", err)
  353. }
  354. } else {
  355. aak = alibaba.accessKey
  356. }
  357. c, err := alibaba.Config.GetCustomPricingData()
  358. if err != nil {
  359. return fmt.Errorf("error downloading default pricing data: %w", err)
  360. }
  361. // Get all the nodes from Alibaba cluster.
  362. nodeList := alibaba.Clientset.GetAllNodes()
  363. var client *sdk.Client
  364. var signer *signers.AccessKeySigner
  365. var ok bool
  366. var lookupKey string
  367. alibaba.clients = make(map[string]*sdk.Client)
  368. alibaba.Pricing = make(map[string]*AlibabaPricing)
  369. for _, node := range nodeList {
  370. pricingObj := &AlibabaPricing{}
  371. slimK8sNode := generateSlimK8sNodeFromV1Node(node)
  372. if client, ok = alibaba.clients[slimK8sNode.RegionID]; !ok {
  373. client, err = sdk.NewClientWithAccessKey(slimK8sNode.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  374. if err != nil {
  375. return fmt.Errorf("unable to initiate alibaba cloud sdk client for region %s : %w", slimK8sNode.RegionID, err)
  376. }
  377. alibaba.clients[slimK8sNode.RegionID] = client
  378. }
  379. signer = signers.NewAccessKeySigner(aak)
  380. // Adjust the system Disk information of a Node by retrieving the details of associated disk. If unable to retrieve set it to empty
  381. // system disk to pass through and use defaults with Alibaba pricing API.
  382. instanceID := getInstanceIDFromProviderID(slimK8sNode.ProviderID)
  383. slimK8sNode.SystemDisk = getSystemDiskInfoOfANode(instanceID, slimK8sNode.RegionID, client, signer)
  384. lookupKey, err = determineKeyForPricing(slimK8sNode)
  385. if err != nil {
  386. return fmt.Errorf("unable to determine key for pricing: %w", err)
  387. }
  388. if _, ok := alibaba.Pricing[lookupKey]; ok {
  389. log.Debugf("Pricing information for node with same features %s already exists hence skipping", lookupKey)
  390. continue
  391. }
  392. pricingObj, err = processDescribePriceAndCreateAlibabaPricing(client, slimK8sNode, signer, c)
  393. if err != nil {
  394. return fmt.Errorf("failed to create pricing information for node with type %s with error: %w", slimK8sNode.InstanceType, err)
  395. }
  396. alibaba.Pricing[lookupKey] = pricingObj
  397. }
  398. // set the first occurrence of region from the node
  399. if alibaba.ClusterRegion == "" {
  400. for _, node := range nodeList {
  401. if regionID, ok := node.Labels["topology.kubernetes.io/region"]; ok {
  402. alibaba.ClusterRegion = regionID
  403. break
  404. }
  405. }
  406. }
  407. // PV pricing for only Cloud Disk for now.
  408. // TO-DO: Support both NAS(Network Attached storage) and OSS(Object Storage Service) type PVs
  409. pvList := alibaba.Clientset.GetAllPersistentVolumes()
  410. for _, pv := range pvList {
  411. pvRegion := determinePVRegion(pv)
  412. if pvRegion == "" {
  413. pvRegion = alibaba.ClusterRegion
  414. }
  415. pricingObj := &AlibabaPricing{}
  416. slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, pvRegion)
  417. lookupKey, err = determineKeyForPricing(slimK8sDisk)
  418. if err != nil {
  419. return fmt.Errorf("unable to determine key for pricing: %w", err)
  420. }
  421. if _, ok := alibaba.Pricing[lookupKey]; ok {
  422. log.Debugf("Pricing information for pv with same features %s already exists hence skipping", lookupKey)
  423. continue
  424. }
  425. if client, ok = alibaba.clients[slimK8sDisk.RegionID]; !ok {
  426. client, err = sdk.NewClientWithAccessKey(slimK8sDisk.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  427. if err != nil {
  428. return fmt.Errorf("unable to initiate alibaba cloud sdk client for region %s : %w", slimK8sDisk.RegionID, err)
  429. }
  430. alibaba.clients[slimK8sDisk.RegionID] = client
  431. }
  432. signer = signers.NewAccessKeySigner(aak)
  433. pricingObj, err = processDescribePriceAndCreateAlibabaPricing(client, slimK8sDisk, signer, c)
  434. if err != nil {
  435. return fmt.Errorf("failed to create pricing information for pv with category %s with error: %w", slimK8sDisk.DiskCategory, err)
  436. }
  437. alibaba.Pricing[lookupKey] = pricingObj
  438. }
  439. return nil
  440. }
  441. // AllNodePricing returns all the pricing data for all nodes and pvs
  442. func (alibaba *Alibaba) AllNodePricing() (interface{}, error) {
  443. alibaba.DownloadPricingDataLock.RLock()
  444. defer alibaba.DownloadPricingDataLock.RUnlock()
  445. return alibaba.Pricing, nil
  446. }
  447. // NodePricing gives pricing information of a specific node given by the key
  448. func (alibaba *Alibaba) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
  449. alibaba.DownloadPricingDataLock.RLock()
  450. defer alibaba.DownloadPricingDataLock.RUnlock()
  451. // Get node features for the key
  452. keyFeature := key.Features()
  453. meta := models.PricingMetadata{}
  454. pricing, ok := alibaba.Pricing[keyFeature]
  455. if !ok {
  456. keys := make([]string, 0, len(alibaba.Pricing))
  457. for k := range alibaba.Pricing {
  458. keys = append(keys, k)
  459. }
  460. kf := key.(*AlibabaNodeKey)
  461. // Try to look up pricing with no disk attached
  462. pricing, ok = alibaba.Pricing[kf.FeaturesWithOtherDisk("")]
  463. if !ok {
  464. log.Errorf("Node pricing information not found for node with feature: %s . Existing keys are: %+v", keyFeature, keys)
  465. return nil, meta, fmt.Errorf("Node pricing information not found for node with feature: %s letting it use default values", keyFeature)
  466. }
  467. }
  468. log.Debugf("returning the node price for the node with feature: %s", keyFeature)
  469. returnNode := pricing.Node
  470. return returnNode, meta, nil
  471. }
  472. func (alibaba *Alibaba) GpuPricing(nodeLabels map[string]string) (string, error) {
  473. return "", nil
  474. }
  475. // PVPricing gives a pricing information of a specific PV given by PVkey
  476. func (alibaba *Alibaba) PVPricing(pvk models.PVKey) (*models.PV, error) {
  477. alibaba.DownloadPricingDataLock.RLock()
  478. defer alibaba.DownloadPricingDataLock.RUnlock()
  479. keyFeature := pvk.Features()
  480. pricing, ok := alibaba.Pricing[keyFeature]
  481. if !ok {
  482. log.Debugf("Persistent Volume pricing not found for PV with feature: %s", keyFeature)
  483. return nil, fmt.Errorf("Persistent Volume pricing not found for PV with feature: %s letting it use default values", keyFeature)
  484. }
  485. log.Debugf("returning the PV price for the node with feature: %s", keyFeature)
  486. return pricing.PV, nil
  487. }
  488. // Inter zone and Inter region network cost are defaulted based on https://www.alibabacloud.com/help/en/cloud-data-transmission/latest/cross-region-data-transfers
  489. // Internet cost is default based on https://www.alibabacloud.com/help/en/elastic-compute-service/latest/public-bandwidth to $0.123
  490. func (alibaba *Alibaba) NetworkPricing() (*models.Network, error) {
  491. cpricing, err := alibaba.Config.GetCustomPricingData()
  492. if err != nil {
  493. return nil, err
  494. }
  495. znec, err := strconv.ParseFloat(cpricing.ZoneNetworkEgress, 64)
  496. if err != nil {
  497. return nil, err
  498. }
  499. rnec, err := strconv.ParseFloat(cpricing.RegionNetworkEgress, 64)
  500. if err != nil {
  501. return nil, err
  502. }
  503. inec, err := strconv.ParseFloat(cpricing.InternetNetworkEgress, 64)
  504. if err != nil {
  505. return nil, err
  506. }
  507. return &models.Network{
  508. ZoneNetworkEgressCost: znec,
  509. RegionNetworkEgressCost: rnec,
  510. InternetNetworkEgressCost: inec,
  511. }, nil
  512. }
  513. // Alibaba loadbalancer has three different types https://www.alibabacloud.com/product/server-load-balancer,
  514. // defaulted price to classic load balancer https://www.alibabacloud.com/help/en/server-load-balancer/latest/pay-as-you-go.
  515. func (alibaba *Alibaba) LoadBalancerPricing() (*models.LoadBalancer, error) {
  516. cpricing, err := alibaba.Config.GetCustomPricingData()
  517. if err != nil {
  518. return nil, err
  519. }
  520. lbPricing, err := strconv.ParseFloat(cpricing.DefaultLBPrice, 64)
  521. if err != nil {
  522. return nil, err
  523. }
  524. return &models.LoadBalancer{
  525. Cost: lbPricing,
  526. }, nil
  527. }
  528. func (alibaba *Alibaba) GetConfig() (*models.CustomPricing, error) {
  529. c, err := alibaba.Config.GetCustomPricingData()
  530. if err != nil {
  531. return nil, err
  532. }
  533. if c.Discount == "" {
  534. c.Discount = "0%"
  535. }
  536. if c.NegotiatedDiscount == "" {
  537. c.NegotiatedDiscount = "0%"
  538. }
  539. return c, nil
  540. }
  541. // Load once and cache the result (even on failure). This is an install time secret, so
  542. // we don't expect the secret to change. If it does, however, we can force reload using
  543. // the input parameter.
  544. func (alibaba *Alibaba) loadAlibabaAuthSecretAndSetEnv(force bool) error {
  545. if !force && alibaba.accessKeyisLoaded() {
  546. return nil
  547. }
  548. exists, err := fileutil.FileExists(models.AuthSecretPath)
  549. if !exists || err != nil {
  550. return fmt.Errorf("failed to locate service account file: %s with err: %w", models.AuthSecretPath, err)
  551. }
  552. result, err := os.ReadFile(models.AuthSecretPath)
  553. if err != nil {
  554. return fmt.Errorf("failed to read service account file: %s with err: %w", models.AuthSecretPath, err)
  555. }
  556. var ak *AlibabaAccessKey
  557. err = json.Unmarshal(result, &ak)
  558. if err != nil {
  559. return fmt.Errorf("failed to unmarshall access key id and access key secret with err: %w", err)
  560. }
  561. err = coreenv.Set(env.AlibabaAccessKeyIDEnvVar, ak.AccessKeyID)
  562. if err != nil {
  563. return fmt.Errorf("failed to set environment variable: %s with err: %w", env.AlibabaAccessKeyIDEnvVar, err)
  564. }
  565. err = coreenv.Set(env.AlibabaAccessKeySecretEnvVar, ak.SecretAccessKey)
  566. if err != nil {
  567. return fmt.Errorf("failed to set environment variable: %s with err: %w", env.AlibabaAccessKeySecretEnvVar, err)
  568. }
  569. alibaba.accessKey = &credentials.AccessKeyCredential{
  570. AccessKeyId: ak.AccessKeyID,
  571. AccessKeySecret: ak.SecretAccessKey,
  572. }
  573. return nil
  574. }
  575. // Regions returns a current supported list of Alibaba regions
  576. func (alibaba *Alibaba) Regions() []string {
  577. regionOverrides := env.GetRegionOverrideList()
  578. if len(regionOverrides) > 0 {
  579. log.Debugf("Overriding Alibaba regions with configured region list: %+v", regionOverrides)
  580. return regionOverrides
  581. }
  582. return alibabaRegions
  583. }
  584. // ClusterInfo returns information about Alibaba Cloud cluster, as provided by metadata.
  585. func (alibaba *Alibaba) ClusterInfo() (map[string]string, error) {
  586. c, err := alibaba.GetConfig()
  587. if err != nil {
  588. return nil, fmt.Errorf("failed to getConfig with err: %w", err)
  589. }
  590. var clusterName string
  591. if c.ClusterName != "" {
  592. clusterName = c.ClusterName
  593. }
  594. // Set it to environment clusterID if not set at this point
  595. if clusterName == "" {
  596. clusterName = coreenv.GetClusterID()
  597. }
  598. m := make(map[string]string)
  599. m["name"] = clusterName
  600. m["provider"] = opencost.AlibabaProvider
  601. m["project"] = alibaba.ClusterAccountId
  602. m["region"] = alibaba.ClusterRegion
  603. m["id"] = coreenv.GetClusterID()
  604. return m, nil
  605. }
  606. // Will look at this in Next PR if needed
  607. func (alibaba *Alibaba) GetAddresses() ([]byte, error) {
  608. return nil, nil
  609. }
  610. // Will look at this in Next PR if needed
  611. func (alibaba *Alibaba) GetDisks() ([]byte, error) {
  612. return nil, nil
  613. }
  614. func (alibaba *Alibaba) GetOrphanedResources() ([]models.OrphanedResource, error) {
  615. return nil, errors.New("not implemented")
  616. }
  617. func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
  618. return alibaba.Config.Update(func(c *models.CustomPricing) error {
  619. if updateType != "" {
  620. return fmt.Errorf("UpdateConfig for Alibaba Provider doesn't support updateType %s at this time", updateType)
  621. } else {
  622. a := make(map[string]interface{})
  623. err := json.NewDecoder(r).Decode(&a)
  624. if err != nil {
  625. return err
  626. }
  627. for k, v := range a {
  628. kUpper := utils.ToTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
  629. vstr, ok := v.(string)
  630. if ok {
  631. err := models.SetCustomPricingField(c, kUpper, vstr)
  632. if err != nil {
  633. return fmt.Errorf("error setting custom pricing field: %w", err)
  634. }
  635. } else {
  636. return fmt.Errorf("type error while updating config for %s", kUpper)
  637. }
  638. }
  639. }
  640. if env.IsRemoteEnabled() {
  641. err := utils.UpdateClusterMeta(coreenv.GetClusterID(), c.ClusterName)
  642. if err != nil {
  643. return err
  644. }
  645. }
  646. return nil
  647. })
  648. }
  649. func (alibaba *Alibaba) UpdateConfigFromConfigMap(cm map[string]string) (*models.CustomPricing, error) {
  650. return alibaba.Config.UpdateFromMap(cm)
  651. }
  652. // Will look at this in Next PR if needed
  653. func (alibaba *Alibaba) GetManagementPlatform() (string, error) {
  654. return "", nil
  655. }
  656. // Will look at this in Next PR if needed
  657. func (alibaba *Alibaba) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
  658. }
  659. // Will look at this in Next PR if needed
  660. func (alibaba *Alibaba) ServiceAccountStatus() *models.ServiceAccountStatus {
  661. return &models.ServiceAccountStatus{}
  662. }
  663. // Will look at this in Next PR if needed
  664. func (alibaba *Alibaba) PricingSourceStatus() map[string]*models.PricingSource {
  665. return map[string]*models.PricingSource{}
  666. }
  667. // Will look at this in Next PR if needed
  668. func (alibaba *Alibaba) ClusterManagementPricing() (string, float64, error) {
  669. return "", 0.0, nil
  670. }
  671. // Will look at this in Next PR if needed
  672. func (alibaba *Alibaba) CombinedDiscountForNode(string, bool, float64, float64) float64 {
  673. return 0.0
  674. }
  675. func (alibaba *Alibaba) accessKeyisLoaded() bool {
  676. if alibaba.accessKey == nil {
  677. return false
  678. }
  679. if alibaba.accessKey.AccessKeyId == "" {
  680. return false
  681. }
  682. if alibaba.accessKey.AccessKeySecret == "" {
  683. return false
  684. }
  685. return true
  686. }
  687. type AlibabaNodeKey struct {
  688. ProviderID string
  689. RegionID string
  690. InstanceType string
  691. OSType string
  692. OptimizedKeyword string //If IsIoOptimized is true use the word optimize in the Node key and if its not optimized use the word nonoptimize
  693. SystemDiskCategory string
  694. SystemDiskSizeInGiB string
  695. SystemDiskPerformanceLevel string
  696. }
  697. func NewAlibabaNodeKey(node *SlimK8sNode, optimizedKeyword, systemDiskCategory, systemDiskSizeInGiB, systemDiskPerfromanceLevel string) *AlibabaNodeKey {
  698. var providerID, regionID, instanceType, osType string
  699. if node != nil {
  700. providerID = node.ProviderID
  701. regionID = node.RegionID
  702. instanceType = node.InstanceType
  703. osType = node.OSType
  704. }
  705. return &AlibabaNodeKey{
  706. ProviderID: providerID,
  707. RegionID: regionID,
  708. InstanceType: instanceType,
  709. OSType: osType,
  710. OptimizedKeyword: optimizedKeyword,
  711. SystemDiskCategory: systemDiskCategory,
  712. SystemDiskSizeInGiB: systemDiskSizeInGiB,
  713. SystemDiskPerformanceLevel: systemDiskPerfromanceLevel,
  714. }
  715. }
  716. func (alibabaNodeKey *AlibabaNodeKey) ID() string {
  717. return alibabaNodeKey.ProviderID
  718. }
  719. func (alibabaNodeKey *AlibabaNodeKey) Features() string {
  720. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaNodeKey.RegionID, alibabaNodeKey.InstanceType, alibabaNodeKey.OSType,
  721. alibabaNodeKey.OptimizedKeyword, alibabaNodeKey.SystemDiskCategory, alibabaNodeKey.SystemDiskSizeInGiB, alibabaNodeKey.SystemDiskPerformanceLevel})
  722. return strings.Join(keyLookup, "::")
  723. }
  724. func (alibabaNodeKey *AlibabaNodeKey) FeaturesWithOtherDisk(overrideDiskCategory string) string {
  725. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaNodeKey.RegionID, alibabaNodeKey.InstanceType, alibabaNodeKey.OSType,
  726. alibabaNodeKey.OptimizedKeyword, overrideDiskCategory, alibabaNodeKey.SystemDiskSizeInGiB, alibabaNodeKey.SystemDiskPerformanceLevel})
  727. return strings.Join(keyLookup, "::")
  728. }
  729. func (alibabaNodeKey *AlibabaNodeKey) GPUType() string {
  730. return ""
  731. }
  732. func (alibabaNodeKey *AlibabaNodeKey) GPUCount() int {
  733. return 0
  734. }
  735. // Get's the key for the k8s node input
  736. func (alibaba *Alibaba) GetKey(mapValue map[string]string, node *clustercache.Node) models.Key {
  737. slimK8sNode := generateSlimK8sNodeFromV1Node(node)
  738. var aak *credentials.AccessKeyCredential
  739. var err error
  740. var ok bool
  741. var client *sdk.Client
  742. var signer *signers.AccessKeySigner
  743. optimizedKeyword := ""
  744. if slimK8sNode.IsIoOptimized {
  745. optimizedKeyword = ALIBABA_OPTIMIZE_KEYWORD
  746. } else {
  747. optimizedKeyword = ALIBABA_NON_OPTIMIZE_KEYWORD
  748. }
  749. var diskCategory, diskSizeInGiB, diskPerformanceLevel string
  750. if !alibaba.accessKeyisLoaded() {
  751. aak, err = alibaba.GetAlibabaAccessKey()
  752. if err != nil {
  753. log.Warnf("unable to set the signer for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: %v", slimK8sNode.ProviderID, err)
  754. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  755. }
  756. } else {
  757. aak = alibaba.accessKey
  758. }
  759. signer = signers.NewAccessKeySigner(aak)
  760. if aak == nil {
  761. log.Warnf("unable to retrieve the Alibaba API keys for node with providerID %s hence skipping SystemDisk Retrieval", slimK8sNode.ProviderID)
  762. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  763. }
  764. if client, ok = alibaba.clients[slimK8sNode.RegionID]; !ok {
  765. client, err = sdk.NewClientWithAccessKey(slimK8sNode.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  766. if err != nil {
  767. log.Warnf("unable to set the client for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: %v", slimK8sNode.ProviderID, err)
  768. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  769. }
  770. alibaba.clients[slimK8sNode.RegionID] = client
  771. }
  772. instanceID := getInstanceIDFromProviderID(slimK8sNode.ProviderID)
  773. slimK8sNode.SystemDisk = getSystemDiskInfoOfANode(instanceID, slimK8sNode.RegionID, client, signer)
  774. if slimK8sNode.SystemDisk != nil {
  775. diskCategory = slimK8sNode.SystemDisk.DiskCategory
  776. diskSizeInGiB = slimK8sNode.SystemDisk.SizeInGiB
  777. diskPerformanceLevel = slimK8sNode.SystemDisk.PerformanceLevel
  778. }
  779. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  780. }
  781. type AlibabaPVKey struct {
  782. ProviderID string
  783. RegionID string
  784. PVType string
  785. PVSubType string
  786. PVCategory string
  787. PVPerformaceLevel string
  788. StorageClassName string
  789. SizeInGiB string
  790. }
  791. func (alibaba *Alibaba) GetPVKey(pv *clustercache.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
  792. regionID := defaultRegion
  793. // If default Region is not passed default it to cluster region ID.
  794. if defaultRegion == "" {
  795. regionID = alibaba.ClusterRegion
  796. }
  797. slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, defaultRegion)
  798. return &AlibabaPVKey{
  799. ProviderID: slimK8sDisk.ProviderID,
  800. RegionID: regionID,
  801. PVType: ALIBABA_PV_CLOUD_DISK_TYPE,
  802. PVSubType: slimK8sDisk.DiskType,
  803. PVCategory: slimK8sDisk.DiskCategory,
  804. PVPerformaceLevel: slimK8sDisk.PerformanceLevel,
  805. StorageClassName: pv.Spec.StorageClassName,
  806. SizeInGiB: slimK8sDisk.SizeInGiB,
  807. }
  808. }
  809. func (alibabaPVKey *AlibabaPVKey) Features() string {
  810. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaPVKey.RegionID, alibabaPVKey.PVSubType, alibabaPVKey.PVCategory, alibabaPVKey.PVPerformaceLevel, alibabaPVKey.SizeInGiB})
  811. return strings.Join(keyLookup, "::")
  812. }
  813. func (alibabaPVKey *AlibabaPVKey) ID() string {
  814. return alibabaPVKey.ProviderID
  815. }
  816. // Get storage class information for PV.
  817. func (alibabaPVKey *AlibabaPVKey) GetStorageClass() string {
  818. return alibabaPVKey.StorageClassName
  819. }
  820. // Helper functions for alibabaprovider.go
  821. // createDescribePriceACSRequest creates the HTTP GET request for the required resources' Price information,
  822. // When supporting subscription and Premptible resources this HTTP call needs to be modified with PriceUnit information
  823. // When supporting different new type of instances like Compute Optimized, Memory Optimized etc make sure you add the instance type
  824. // in unit test and check if it works or not to create the ack request and processDescribePriceAndCreateAlibabaPricing function
  825. // else more parameters need to be pulled from kubernetes node response or gather information from elsewhere and function modified.
  826. func createDescribePriceACSRequest(i interface{}) (*requests.CommonRequest, error) {
  827. request := requests.NewCommonRequest()
  828. request.Method = requests.GET
  829. request.Product = ALIBABA_ECS_PRODUCT_CODE
  830. request.Domain = ALIBABA_ECS_DOMAIN
  831. request.Version = ALIBABA_ECS_VERSION
  832. request.Scheme = requests.HTTPS
  833. request.ApiName = ALIBABA_DESCRIBE_PRICE_API_ACTION
  834. switch i.(type) {
  835. case *SlimK8sNode:
  836. node := i.(*SlimK8sNode)
  837. request.QueryParams["RegionId"] = node.RegionID
  838. request.QueryParams["ResourceType"] = ALIBABA_INSTANCE_RESOURCE_TYPE
  839. request.QueryParams["InstanceType"] = node.InstanceType
  840. request.QueryParams["PriceUnit"] = node.PriceUnit
  841. if node.SystemDisk != nil {
  842. // Only if the required information is present it should be overridden else default it via the API
  843. if node.SystemDisk.DiskCategory != "" {
  844. request.QueryParams["SystemDisk.Category"] = node.SystemDisk.DiskCategory
  845. }
  846. if node.SystemDisk.SizeInGiB != "" {
  847. request.QueryParams["SystemDisk.Size"] = node.SystemDisk.SizeInGiB
  848. }
  849. if node.SystemDisk.PerformanceLevel != "" {
  850. request.QueryParams["SystemDisk.PerformanceLevel"] = node.SystemDisk.PerformanceLevel
  851. }
  852. } else {
  853. // When the system disk information is not available, and the instance family is g6e or r6e,
  854. // or the instance generation is 6 or above, the default disk category in DescribePrice should be cloud_essd.
  855. if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) || getInstanceFamilyGenerationFromType(node.InstanceType) > 6 {
  856. request.QueryParams["SystemDisk.Category"] = ALIBABA_DISK_CLOUD_ESSD_CATEGORY
  857. }
  858. }
  859. request.TransToAcsRequest()
  860. return request, nil
  861. case *SlimK8sDisk:
  862. disk := i.(*SlimK8sDisk)
  863. request.QueryParams["RegionId"] = disk.RegionID
  864. request.QueryParams["PriceUnit"] = disk.PriceUnit
  865. request.QueryParams["ResourceType"] = ALIBABA_DISK_RESOURCE_TYPE
  866. request.QueryParams[fmt.Sprintf("%s.%d.Size", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.SizeInGiB
  867. request.QueryParams[fmt.Sprintf("%s.%d.Category", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.DiskCategory
  868. // Performance level defaults to PL1 if not present in volume attribute.
  869. if disk.PerformanceLevel != "" {
  870. request.QueryParams[fmt.Sprintf("%s.%d.PerformanceLevel", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.PerformanceLevel
  871. }
  872. request.TransToAcsRequest()
  873. return request, nil
  874. default:
  875. return nil, fmt.Errorf("unsupported ECS type (%T) for DescribePrice at this time", i)
  876. }
  877. }
  878. // createDescribeDisksCSRequest creates the HTTP GET Request to map the system disk to the InstanceID
  879. func createDescribeDisksACSRequest(instanceID, regionID, diskType string) (*requests.CommonRequest, error) {
  880. request := requests.NewCommonRequest()
  881. request.Method = requests.GET
  882. request.Product = ALIBABA_ECS_PRODUCT_CODE
  883. request.Domain = ALIBABA_ECS_DOMAIN
  884. request.Version = ALIBABA_ECS_VERSION
  885. request.Scheme = requests.HTTPS
  886. request.ApiName = ALIBABA_DESCRIBE_DISK_API_ACTION
  887. request.QueryParams["RegionId"] = regionID
  888. request.QueryParams["InstanceId"] = instanceID
  889. request.QueryParams["DiskType"] = diskType
  890. request.TransToAcsRequest()
  891. return request, nil
  892. }
  893. // determineKeyForPricing generate a unique key from SlimK8sNode object that is constructed from v1.Node object and
  894. // SlimK8sDisk that is constructed from v1.PersistentVolume.
  895. func determineKeyForPricing(i interface{}) (string, error) {
  896. if i == nil {
  897. return "", fmt.Errorf("nil component passed to determine key")
  898. }
  899. switch i.(type) {
  900. case *SlimK8sNode:
  901. node := i.(*SlimK8sNode)
  902. var diskCategory, diskSizeInGiB, diskPerformanceLevel string
  903. if node.SystemDisk != nil {
  904. diskCategory = node.SystemDisk.DiskCategory
  905. diskSizeInGiB = node.SystemDisk.SizeInGiB
  906. diskPerformanceLevel = node.SystemDisk.PerformanceLevel
  907. }
  908. if node.IsIoOptimized {
  909. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{node.RegionID, node.InstanceType, node.OSType, ALIBABA_OPTIMIZE_KEYWORD, diskCategory, diskSizeInGiB, diskPerformanceLevel})
  910. return strings.Join(keyLookup, "::"), nil
  911. } else {
  912. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{node.RegionID, node.InstanceType, node.OSType, ALIBABA_NON_OPTIMIZE_KEYWORD, diskCategory, diskSizeInGiB, diskPerformanceLevel})
  913. return strings.Join(keyLookup, "::"), nil
  914. }
  915. case *SlimK8sDisk:
  916. disk := i.(*SlimK8sDisk)
  917. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{disk.RegionID, disk.DiskType, disk.DiskCategory, disk.PerformanceLevel, disk.SizeInGiB})
  918. return strings.Join(keyLookup, "::"), nil
  919. default:
  920. return "", fmt.Errorf("unsupported ECS type (%T) at this time", i)
  921. }
  922. }
  923. // Below structs are used to unmarshal json response of Alibaba cloud's API DescribePrice
  924. type Price struct {
  925. OriginalPrice float32 `json:"OriginalPrice"`
  926. ReservedInstanceHourPrice float32 `json:"ReservedInstanceHourPrice"`
  927. DiscountPrice float32 `json:"DiscountPrice"`
  928. Currency string `json:"Currency"`
  929. TradePrice float32 `json:"TradePrice"`
  930. }
  931. type PriceInfo struct {
  932. Price Price `json:"Price"`
  933. }
  934. type DescribePriceResponse struct {
  935. RequestId string `json:"RequestId"`
  936. PriceInfo PriceInfo `json:"PriceInfo"`
  937. }
  938. // processDescribePriceAndCreateAlibabaPricing processes the DescribePrice API and generates the pricing information for alibaba node resource and alibaba pv resource that's backed by cloud disk.
  939. func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface{}, signer *signers.AccessKeySigner, custom *models.CustomPricing) (pricing *AlibabaPricing, err error) {
  940. pricing = &AlibabaPricing{}
  941. var response DescribePriceResponse
  942. if client == nil {
  943. return nil, fmt.Errorf("nil client passed to process the pricing information")
  944. }
  945. if i == nil {
  946. return nil, fmt.Errorf("nil component passed to process the pricing information")
  947. }
  948. switch i.(type) {
  949. case *SlimK8sNode:
  950. node := i.(*SlimK8sNode)
  951. req, err := createDescribePriceACSRequest(node)
  952. if err != nil {
  953. return nil, err
  954. }
  955. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  956. pricing.NodeAttributes = NewAlibabaNodeAttributes(node)
  957. if err != nil || resp.GetHttpStatus() != 200 {
  958. // Try again but default the disk to something else
  959. return nil, fmt.Errorf("unable to fetch information for node with InstanceType: %v", node.InstanceType)
  960. } else {
  961. // This is where population of Pricing happens
  962. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  963. if err != nil {
  964. return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
  965. }
  966. // TO-DO : Ask in PR How to get the defaults is it equal to AWS/GCP defaults? And what needs to be returned
  967. pricing.Node = &models.Node{
  968. Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
  969. BaseCPUPrice: custom.CPU,
  970. BaseRAMPrice: custom.RAM,
  971. BaseGPUPrice: custom.GPU,
  972. }
  973. // TO-DO : Currently with Pay-As-You-go Offering TradePrice = HourlyPrice , When support happens to other type HourlyPrice Need to be determined.
  974. pricing.PricingTerms = NewAlibabaPricingTerms(ALIBABA_PAY_AS_YOU_GO_BILLING, NewAlibabaPricingDetails(response.PriceInfo.Price.TradePrice, ALIBABA_HOUR_PRICE_UNIT, response.PriceInfo.Price.TradePrice, response.PriceInfo.Price.Currency))
  975. }
  976. case *SlimK8sDisk:
  977. disk := i.(*SlimK8sDisk)
  978. req, err := createDescribePriceACSRequest(disk)
  979. if err != nil {
  980. return nil, err
  981. }
  982. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  983. if err != nil || resp.GetHttpStatus() != 200 {
  984. return nil, fmt.Errorf("unable to fetch information for disk with DiskType: %v with err: %w", disk.DiskCategory, err)
  985. } else {
  986. // This is where population of Pricing happens
  987. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  988. if err != nil {
  989. return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
  990. }
  991. pricing.PVAttributes = NewAlibabaPVAttributes(disk)
  992. pricing.PV = &models.PV{
  993. Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
  994. }
  995. // TO-DO : Disk has support for Hour and Month but pricing API is failing for month for disk(Research why?) and same challenge as node pricing no prepaid/postpaid distinction in v1.PersistentVolume object have to look at APIs for th information.
  996. pricing.PricingTerms = NewAlibabaPricingTerms(ALIBABA_PAY_AS_YOU_GO_BILLING, NewAlibabaPricingDetails(response.PriceInfo.Price.TradePrice, ALIBABA_HOUR_PRICE_UNIT, response.PriceInfo.Price.TradePrice, response.PriceInfo.Price.Currency))
  997. }
  998. default:
  999. return nil, fmt.Errorf("unsupported ECS Pricing component of type (%T) at this time", i)
  1000. }
  1001. return pricing, nil
  1002. }
  1003. // This function is to get the InstanceFamily from the InstanceType , convention followed in
  1004. // instance type is ecs.[FamilyName].[DifferentSize], it gets the familyName , if it is unable to get it
  1005. // it lists the instance family name as Unknown.
  1006. func getInstanceFamilyFromType(instanceType string) string {
  1007. splitinstanceType := strings.Split(instanceType, ".")
  1008. if len(splitinstanceType) != 3 {
  1009. log.Warnf("unable to find the family of the instance type %s, returning its family type unknown", instanceType)
  1010. return ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE
  1011. }
  1012. return splitinstanceType[1]
  1013. }
  1014. // This function is used to obtain the generation of the instance family from the InstanceType,
  1015. // because when the generation is higher than or equal to 7, the instance disk type will not support cloud_efficiency.
  1016. // In such cases, when calling the DescribePrice interface, the system disk type will default to cloud_essd.
  1017. func getInstanceFamilyGenerationFromType(instanceType string) int {
  1018. // FamilyName format: g7ne or g7 or r7 or r6e,
  1019. familyName := getInstanceFamilyFromType(instanceType)
  1020. re := regexp.MustCompile(`(\d+)`)
  1021. match := re.FindString(familyName)
  1022. if match != "" {
  1023. generation, err := strconv.Atoi(match)
  1024. if err != nil {
  1025. log.Errorf("unable to convert the generation of the instance type %s to integer", instanceType)
  1026. } else {
  1027. return generation
  1028. }
  1029. }
  1030. log.Warnf("unable to find the generation of the instance type %s,", instanceType)
  1031. return -1
  1032. }
  1033. // getInstanceIDFromProviderID returns the instance ID associated with the Node. A *v1.Node providerID in Alibaba cloud
  1034. // is of <REGION-ID>.<INSTANCE-ID>. This function returns the Instance ID for the given ProviderID. if its unable to interpret
  1035. // it defaults to empty string.
  1036. func getInstanceIDFromProviderID(providerID string) string {
  1037. if providerID == "" {
  1038. return ""
  1039. }
  1040. splitStrings := strings.Split(providerID, ".")
  1041. if len(splitStrings) < 2 {
  1042. return ""
  1043. }
  1044. return splitStrings[1]
  1045. }
  1046. type Disk struct {
  1047. Category string `json:"Category"`
  1048. Size int `json:"Size"`
  1049. PerformanceLevel string `json:"PerformanceLevel"`
  1050. Type string `json:"Type"`
  1051. RegionId string `json:"RegionId"`
  1052. DiskId string `json:"DiskId"`
  1053. DiskChargeType string `json:"DiskChargeType"`
  1054. }
  1055. type Disks struct {
  1056. Disk []*Disk `json:"Disk"`
  1057. }
  1058. type DescribeDiskResponse struct {
  1059. TotalCount int `json:"TotalCount"`
  1060. Disks *Disks `json:"Disks"`
  1061. }
  1062. // getSystemDiskInfoOfANode gets the relevant System disk information associated with the Node given by the instanceID
  1063. // in form of a SlimK8sDisk with only relevant information that can adjust the node pricing. If any error occurs return
  1064. // an empty disk to not impact any default set at the price retrieval of the node.
  1065. func getSystemDiskInfoOfANode(instanceID, regionID string, client *sdk.Client, signer *signers.AccessKeySigner) (systemDisk *SlimK8sDisk) {
  1066. systemDisk = &SlimK8sDisk{}
  1067. var response DescribeDiskResponse
  1068. // if instanceID is empty string return an empty k8s
  1069. if instanceID == "" {
  1070. return
  1071. }
  1072. // if client is nil return an empty disk to not impact default pricing
  1073. if client == nil {
  1074. log.Warnf("unable to set the signer for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: nil client", instanceID)
  1075. return
  1076. }
  1077. req, err := createDescribeDisksACSRequest(instanceID, regionID, ALIBABA_SYSTEM_DISK_CATEGORY)
  1078. // if any error occurs return an empty disk to not impact default pricing.
  1079. if err != nil {
  1080. log.Warnf("Unable to create Describe Disk Request with err: %v for node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, instanceID)
  1081. return
  1082. }
  1083. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  1084. if err != nil || resp.GetHttpStatus() != 200 {
  1085. log.Warnf("Unable to process Describe Disk request with err: %v and errcode: %d for the node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, resp.GetHttpStatus(), instanceID)
  1086. return
  1087. } else {
  1088. // This is where population of Pricing happens
  1089. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  1090. if err != nil {
  1091. log.Warnf("Unable to unmarshall Describe Disk response with err: %v for the node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, instanceID)
  1092. return
  1093. }
  1094. // Every instance should only have one system disk per Alibaba Cloud documentation https://www.alibabacloud.com/help/en/elastic-compute-service/latest/block-storage-overview-disks,
  1095. // if TotalCount is not 1 just return empty and let it not impact default pricing.
  1096. if response.TotalCount != 1 {
  1097. log.Warnf("Total count of system disk for node with InstanceID: %s is not 1, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  1098. return
  1099. }
  1100. if response.Disks == nil {
  1101. log.Warnf("Disks information missing for node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  1102. return
  1103. }
  1104. if len(response.Disks.Disk) < 1 {
  1105. log.Warnf("Total number of system disk for node with InstanceID: %s is less than 1, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  1106. return
  1107. }
  1108. // TO-DO: When supporting Subscription type disk, you can leverge the disk.DiskChargeType here to map it to subscription type.
  1109. systemDisk := response.Disks.Disk[0]
  1110. return NewSlimK8sDisk(systemDisk.Type, systemDisk.RegionId, ALIBABA_HOUR_PRICE_UNIT, systemDisk.Category, systemDisk.PerformanceLevel, systemDisk.DiskId, "", fmt.Sprintf("%d", systemDisk.Size))
  1111. }
  1112. }
  1113. // generateSlimK8sNodeFromV1Node generates SlimK8sNode struct from v1.Node to fetch pricing information and call alibaba API.
  1114. func generateSlimK8sNodeFromV1Node(node *clustercache.Node) *SlimK8sNode {
  1115. var regionID, osType, instanceType, providerID, priceUnit, instanceFamily string
  1116. var memorySizeInKiB string // TO-DO: try to convert it into float
  1117. var ok, IsIoOptimized bool
  1118. if regionID, ok = node.Labels["topology.kubernetes.io/region"]; !ok {
  1119. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1120. log.Debugf("No RegionID label for the node: %s", node.Name)
  1121. }
  1122. if osType, ok = node.Labels["beta.kubernetes.io/os"]; !ok {
  1123. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1124. log.Debugf("OS type undetected for the node: %s", node.Name)
  1125. }
  1126. if instanceType, ok = node.Labels["node.kubernetes.io/instance-type"]; !ok {
  1127. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1128. log.Debugf("Instance Type undetected for the node: %s", node.Name)
  1129. }
  1130. instanceFamily = getInstanceFamilyFromType(instanceType)
  1131. memorySizeInKiB = fmt.Sprintf("%s", node.Status.Capacity.Memory())
  1132. providerID = node.SpecProviderID // Alibaba Cloud provider doesnt follow convention of prefix with cloud provider name
  1133. // Looking at current Instance offering , all of the Instances seem to be I/O optimized - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/instance-family
  1134. // Basic price Json has it as part of the key so defaulting to true.
  1135. IsIoOptimized = true
  1136. priceUnit = ALIBABA_HOUR_PRICE_UNIT
  1137. systemDisk := &SlimK8sDisk{}
  1138. return NewSlimK8sNode(instanceType, regionID, priceUnit, memorySizeInKiB, osType, providerID, instanceFamily, IsIoOptimized, systemDisk)
  1139. }
  1140. // getNumericalValueFromResourceQuantity returns the numericalValue of the resourceQuantity
  1141. // An example is: 20Gi returns to 20. If any error occurs it returns the default value used in describePrice API which is 2000.
  1142. func getNumericalValueFromResourceQuantity(quantity string) (value string) {
  1143. // defaulting when any panic or empty string occurs.
  1144. defer func() {
  1145. log.Debugf("unable to determine the size of the PV so defaulting the size to %s", ALIBABA_DEFAULT_DATADISK_SIZE)
  1146. if err := recover(); err != nil {
  1147. value = ALIBABA_DEFAULT_DATADISK_SIZE
  1148. }
  1149. if value == "" {
  1150. value = ALIBABA_DEFAULT_DATADISK_SIZE
  1151. }
  1152. }()
  1153. res := sizeRegEx.FindAllStringSubmatch(quantity, 1)
  1154. value = res[0][1]
  1155. return
  1156. }
  1157. // generateSlimK8sDiskFromV1PV function generates SlimK8sDisk from v1.PersistentVolume
  1158. // to generate slim disk type that can be used to fetch pricing information for Data disk type.
  1159. func generateSlimK8sDiskFromV1PV(pv *clustercache.PersistentVolume, regionID string) *SlimK8sDisk {
  1160. // All PVs are data disks while local disk are categorized as system disk
  1161. diskType := ALIBABA_DATA_DISK_CATEGORY
  1162. //TO-DO: Disk supports month and hour prices , defaulting to hour
  1163. priceUnit := ALIBABA_HOUR_PRICE_UNIT
  1164. sizeQuantity := fmt.Sprintf("%s", pv.Spec.Capacity.Storage())
  1165. // res := sizeRegEx.FindAllStringSubmatch(sizeQuantity, 1)
  1166. sizeInGiB := getNumericalValueFromResourceQuantity(sizeQuantity)
  1167. providerID := ""
  1168. if pv.Spec.CSI != nil {
  1169. providerID = pv.Spec.CSI.VolumeHandle
  1170. } else {
  1171. providerID = pv.Name // Looks like pv name is same as providerID in Alibaba k8s cluster
  1172. }
  1173. // Performance level being empty string gets defaulted in describePrice to PL1.
  1174. performanceLevel := ""
  1175. diskCategory := ""
  1176. if pv.Spec.CSI != nil {
  1177. if val, ok := pv.Spec.CSI.VolumeAttributes["performanceLevel"]; ok {
  1178. performanceLevel = val
  1179. }
  1180. if val, ok := pv.Spec.CSI.VolumeAttributes["type"]; ok {
  1181. diskCategory = val
  1182. }
  1183. }
  1184. // Highly unlikely that label pv.Spec.CSI.VolumeAttributes["type"] doesn't exist but if occurred default to cloud (most basic disk type)
  1185. if diskCategory == "" {
  1186. diskCategory = ALIBABA_DISK_CLOUD_CATEGORY
  1187. }
  1188. return NewSlimK8sDisk(diskType, regionID, priceUnit, diskCategory, performanceLevel, providerID, pv.Spec.StorageClassName, sizeInGiB)
  1189. }
  1190. // determinePVRegion determines associated region for a particular PV based on the following priority, which can be changed and any other path to determine region can be added!
  1191. // if topology.diskplugin.csi.alibabacloud.com/region label/annotation is passed during PV creation return that as the PV region.
  1192. // if topology.diskplugin.csi.alibabacloud.com/zone label/annotation is passed during PV creation determine the region based on this pv label.
  1193. // if neither of the above label/annotation is present check node affinity for the zone affinity and determine the region based on this zone.
  1194. // if nether of the above yields a region , return empty string to default it to cluster region.
  1195. func determinePVRegion(pv *clustercache.PersistentVolume) string {
  1196. // if "topology.diskplugin.csi.alibabacloud.com/region" is present as a label or annotation return that as the PV region
  1197. if val, ok := pv.Labels[ALIBABA_DISK_TOPOLOGY_REGION_LABEL]; ok {
  1198. log.Debugf("determinePVRegion returned a region value of: %s through label: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_REGION_LABEL, pv.Name)
  1199. return val
  1200. }
  1201. if val, ok := pv.Annotations[ALIBABA_DISK_TOPOLOGY_REGION_LABEL]; ok {
  1202. log.Debugf("determinePVRegion returned a region value of: %s through annotation: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_REGION_LABEL, pv.Name)
  1203. return val
  1204. }
  1205. // if "topology.diskplugin.csi.alibabacloud.com/zone" is present as a label or annotation set it as the PV zone before looking at node affinity to determine the region PV belongs too
  1206. var pvZone string
  1207. if val, ok := pv.Labels[ALIBABA_DISK_TOPOLOGY_ZONE_LABEL]; ok {
  1208. log.Debugf("determinePVRegion will set zone value to: %s through label: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1209. pvZone = val
  1210. }
  1211. if pvZone == "" {
  1212. if val, ok := pv.Annotations[ALIBABA_DISK_TOPOLOGY_ZONE_LABEL]; ok {
  1213. log.Debugf("determinePVRegion will set zone value to: %s through annotation: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1214. pvZone = val
  1215. }
  1216. }
  1217. if pvZone == "" {
  1218. // zone and regionID labels are optional in Alibaba PV creation, while PV through UI creation put's a zone PV is associated with and the region
  1219. // can be determined from this information. If pv is provision via yaml and the block is missing that's the only time it gets defaulted to ClusterRegion.
  1220. if pv.Spec.NodeAffinity != nil {
  1221. nodeAffinity := pv.Spec.NodeAffinity
  1222. if nodeAffinity.Required != nil && nodeAffinity.Required.NodeSelectorTerms != nil {
  1223. for _, nodeSelectorTerm := range nodeAffinity.Required.NodeSelectorTerms {
  1224. matchExpression := nodeSelectorTerm.MatchExpressions
  1225. for _, nodeSelectorRequirement := range matchExpression {
  1226. if nodeSelectorRequirement.Key == ALIBABA_DISK_TOPOLOGY_ZONE_LABEL {
  1227. log.Debugf("determinePVRegion will set zone value to: %s through node affinity label: %s for PV name: %s", nodeSelectorRequirement.Values[0], ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1228. pvZone = nodeSelectorRequirement.Values[0]
  1229. }
  1230. }
  1231. }
  1232. }
  1233. }
  1234. }
  1235. regionOverrides := env.GetRegionOverrideList()
  1236. regions := alibabaRegions
  1237. if len(regionOverrides) > 0 {
  1238. regions = regionOverrides
  1239. }
  1240. for _, region := range regions {
  1241. if strings.Contains(pvZone, region) {
  1242. log.Debugf("determinePVRegion determined region of %s through zone affiliation of the PV %s\n", region, pvZone)
  1243. return region
  1244. }
  1245. }
  1246. return ""
  1247. }
  1248. // PricingSourceSummary returns the pricing source summary for the provider.
  1249. // The summary represents what was _parsed_ from the pricing source, not
  1250. // everything that was _available_ in the pricing source.
  1251. func (a *Alibaba) PricingSourceSummary() interface{} {
  1252. return a.Pricing
  1253. }