aliyunprovider.go 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281
  1. package cloud
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "regexp"
  8. "strings"
  9. "sync"
  10. "time"
  11. "github.com/aliyun/alibaba-cloud-sdk-go/sdk"
  12. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
  13. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/signers"
  14. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
  15. "github.com/opencost/opencost/pkg/clustercache"
  16. "github.com/opencost/opencost/pkg/env"
  17. "github.com/opencost/opencost/pkg/kubecost"
  18. "github.com/opencost/opencost/pkg/log"
  19. "github.com/opencost/opencost/pkg/util/fileutil"
  20. "github.com/opencost/opencost/pkg/util/json"
  21. "github.com/opencost/opencost/pkg/util/stringutil"
  22. "golang.org/x/exp/slices"
  23. v1 "k8s.io/api/core/v1"
  24. )
  25. const (
  26. ALIBABA_ECS_PRODUCT_CODE = "ecs"
  27. ALIBABA_ECS_VERSION = "2014-05-26"
  28. ALIBABA_ECS_DOMAIN = "ecs.aliyuncs.com"
  29. ALIBABA_DESCRIBE_PRICE_API_ACTION = "DescribePrice"
  30. ALIBABA_DESCRIBE_DISK_API_ACTION = "DescribeDisks"
  31. ALIBABA_INSTANCE_RESOURCE_TYPE = "instance"
  32. ALIBABA_DISK_RESOURCE_TYPE = "disk"
  33. ALIBABA_PAY_AS_YOU_GO_BILLING = "Pay-As-You-Go"
  34. ALIBABA_SUBSCRIPTION_BILLING = "Subscription"
  35. ALIBABA_PREEMPTIBLE_BILLING = "Preemptible"
  36. ALIBABA_OPTIMIZE_KEYWORD = "optimize"
  37. ALIBABA_NON_OPTIMIZE_KEYWORD = "nonoptimize"
  38. ALIBABA_HOUR_PRICE_UNIT = "Hour"
  39. ALIBABA_MONTH_PRICE_UNIT = "Month"
  40. ALIBABA_YEAR_PRICE_UNIT = "Year"
  41. ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE = "unknown"
  42. ALIBABA_NOT_SUPPORTED_INSTANCE_FAMILY_TYPE = "unsupported"
  43. ALIBABA_ENHANCED_GENERAL_PURPOSE_TYPE = "g6e"
  44. ALIBABA_DISK_CLOUD_ESSD_CATEGORY = "cloud_essd"
  45. ALIBABA_DISK_CLOUD_CATEGORY = "cloud"
  46. ALIBABA_DATA_DISK_CATEGORY = "data"
  47. ALIBABA_SYSTEM_DISK_CATEGORY = "system"
  48. ALIBABA_DATA_DISK_PREFIX = "DataDisk"
  49. ALIBABA_PV_CLOUD_DISK_TYPE = "CloudDisk"
  50. ALIBABA_PV_NAS_TYPE = "NAS"
  51. ALIBABA_PV_OSS_TYPE = "OSS"
  52. ALIBABA_DEFAULT_DATADISK_SIZE = "2000"
  53. ALIBABA_DISK_TOPOLOGY_REGION_LABEL = "topology.diskplugin.csi.alibabacloud.com/region"
  54. ALIBABA_DISK_TOPOLOGY_ZONE_LABEL = "topology.diskplugin.csi.alibabacloud.com/zone"
  55. )
  56. var (
  57. // Regular expression to get the numerical value of PV suffix with GiB from *v1.PersistentVolume.
  58. sizeRegEx = regexp.MustCompile("(.*?)Gi")
  59. )
  60. // Why predefined and dependency on code? Can be converted to API call - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/regions-describeregions
  61. var alibabaRegions = []string{
  62. "cn-qingdao",
  63. "cn-beijing",
  64. "cn-zhangjiakou",
  65. "cn-huhehaote",
  66. "cn-wulanchabu",
  67. "cn-hangzhou",
  68. "cn-shanghai",
  69. "cn-nanjing",
  70. "cn-fuzhou",
  71. "cn-shenzhen",
  72. "cn-guangzhou",
  73. "cn-chengdu",
  74. "cn-hongkong",
  75. "ap-southeast-1",
  76. "ap-southeast-2",
  77. "ap-southeast-3",
  78. "ap-southeast-5",
  79. "ap-southeast-6",
  80. "ap-southeast-7",
  81. "ap-south-1",
  82. "ap-northeast-1",
  83. "ap-northeast-2",
  84. "us-west-1",
  85. "us-east-1",
  86. "eu-central-1",
  87. "me-east-1",
  88. }
  89. // To-Do: Convert to API call - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/describeinstancetypefamilies
  90. // Also first pass only completely tested pricing API for General pupose instances families.
  91. var alibabaInstanceFamilies = []string{
  92. "g6e",
  93. "g6",
  94. "g5",
  95. "sn2",
  96. "sn2ne",
  97. }
  98. // AlibabaAccessKey holds Alibaba credentials parsing from the service-key.json file.
  99. type AlibabaAccessKey struct {
  100. AccessKeyID string `json:"alibaba_access_key_id"`
  101. SecretAccessKey string `json:"alibaba_secret_access_key"`
  102. }
  103. // Slim Version of k8s disk assigned to a node or PV.
  104. type SlimK8sDisk struct {
  105. DiskType string
  106. RegionID string
  107. PriceUnit string
  108. SizeInGiB string
  109. DiskCategory string
  110. PerformanceLevel string
  111. ProviderID string
  112. StorageClass string
  113. }
  114. func NewSlimK8sDisk(diskType, regionID, priceUnit, diskCategory, performanceLevel, providerID, storageClass, sizeInGiB string) *SlimK8sDisk {
  115. return &SlimK8sDisk{
  116. DiskType: diskType,
  117. RegionID: regionID,
  118. PriceUnit: priceUnit,
  119. SizeInGiB: sizeInGiB,
  120. DiskCategory: diskCategory,
  121. PerformanceLevel: performanceLevel,
  122. ProviderID: providerID,
  123. StorageClass: storageClass,
  124. }
  125. }
  126. // Slim version of a k8s v1.node just to pass along the object of this struct instead of constant getting the labels from within v1.Node & unit testing.
  127. type SlimK8sNode struct {
  128. InstanceType string
  129. RegionID string
  130. PriceUnit string
  131. MemorySizeInKiB string // TO-DO : Possible to convert to float?
  132. IsIoOptimized bool
  133. OSType string
  134. ProviderID string
  135. SystemDisk *SlimK8sDisk
  136. InstanceTypeFamily string // Bug in DescribePrice, doesn't default to enhanced type correctly and you get an error in DescribePrice to get around need the family of the InstanceType.
  137. }
  138. func NewSlimK8sNode(instanceType, regionID, priceUnit, memorySizeInKiB, osType, providerID, instanceTypeFamily string, isIOOptimized bool, systemDiskInfo *SlimK8sDisk) *SlimK8sNode {
  139. return &SlimK8sNode{
  140. InstanceType: instanceType,
  141. RegionID: regionID,
  142. PriceUnit: priceUnit,
  143. MemorySizeInKiB: memorySizeInKiB,
  144. IsIoOptimized: isIOOptimized,
  145. OSType: osType,
  146. SystemDisk: systemDiskInfo,
  147. ProviderID: providerID,
  148. InstanceTypeFamily: instanceTypeFamily,
  149. }
  150. }
  151. // AlibabaNodeAttributes represents metadata about the Node in its pricing information.
  152. // Basic Attributes needed atleast to get the key, Some attributes from k8s Node response
  153. // be populated directly into *Node object.
  154. type AlibabaNodeAttributes struct {
  155. // InstanceType represents the type of instance.
  156. InstanceType string `json:"instanceType"`
  157. // MemorySizeInKiB represents the size of memory of instance.
  158. MemorySizeInKiB string `json:"memorySizeInKiB"`
  159. // IsIoOptimized represents the if instance is I/O optimized.
  160. IsIoOptimized bool `json:"isIoOptimized"`
  161. // OSType represents the OS installed in the Instance.
  162. OSType string `json:"osType"`
  163. // SystemDiskCategory represents the exact category of the system disk attached to the node.
  164. SystemDiskCategory string `json:"systemDiskCategory"`
  165. // SystemDiskSizeInGiB represents the size of the system disk attached to the node.
  166. SystemDiskSizeInGiB string `json:"systemDiskSizeInGiB"`
  167. // SystemDiskPerformanceLevel represents the performance level of the system disk attached to the node.
  168. SystemDiskPerformanceLevel string `json:"systemPerformanceLevel"`
  169. }
  170. func NewAlibabaNodeAttributes(node *SlimK8sNode) *AlibabaNodeAttributes {
  171. if node == nil {
  172. return nil
  173. }
  174. var diskCategory, sizeInGiB, performanceLevel string
  175. if node.SystemDisk != nil {
  176. diskCategory = node.SystemDisk.DiskCategory
  177. sizeInGiB = node.SystemDisk.SizeInGiB
  178. performanceLevel = node.SystemDisk.PerformanceLevel
  179. }
  180. return &AlibabaNodeAttributes{
  181. InstanceType: node.InstanceType,
  182. MemorySizeInKiB: node.MemorySizeInKiB,
  183. IsIoOptimized: node.IsIoOptimized,
  184. OSType: node.OSType,
  185. SystemDiskCategory: diskCategory,
  186. SystemDiskSizeInGiB: sizeInGiB,
  187. SystemDiskPerformanceLevel: performanceLevel,
  188. }
  189. }
  190. // AlibabaPVAttributes represents metadata the PV in its pricing information.
  191. // Basic Attributes needed atleast to get the keys. Some attributes from k8s PV response
  192. // be populated directly into *PV object.
  193. type AlibabaPVAttributes struct {
  194. // PVType can be Cloud Disk, NetWork Attached Storage(NAS) or Object Storage Service (OSS).
  195. // Represents the way the PV was attached
  196. PVType string `json:"pvType"`
  197. // PVSubType represent the sub category of PVType. This is Data in case of Cloud Disk.
  198. PVSubType string `json:"pvSubType"`
  199. // Example for PVCategory with cloudDisk PVType are cloud, cloud_efficiency, cloud_ssd,
  200. // ephemeral_ssd and cloud_essd. If not present returns empty.
  201. PVCategory string `json:"pvCategory"`
  202. // Example for PerformanceLevel with cloudDisk PVType are PL0,PL1,PL2 &PL3. If not present returns empty.
  203. PVPerformanceLevel string `json:"performanceLevel"`
  204. // The Size of the PV in terms of GiB
  205. SizeInGiB string `json:"sizeInGiB"`
  206. }
  207. // TO-Do: next iteration of Alibaba provider support NetWork Attached Storage(NAS) and Object Storage Service (OSS type PVs).
  208. // Currently defaulting to cloudDisk with provision to add work in future.
  209. func NewAlibabaPVAttributes(disk *SlimK8sDisk) *AlibabaPVAttributes {
  210. if disk == nil {
  211. return nil
  212. }
  213. return &AlibabaPVAttributes{
  214. PVType: ALIBABA_PV_CLOUD_DISK_TYPE,
  215. PVSubType: disk.DiskType,
  216. PVCategory: disk.DiskCategory,
  217. PVPerformanceLevel: disk.PerformanceLevel,
  218. SizeInGiB: disk.SizeInGiB,
  219. }
  220. }
  221. // Stage 1 support will be Pay-As-You-Go with HourlyPrice equal to TradePrice with PriceUnit as Hour
  222. // TO-DO: Subscription and Premptible support, Information can be gathered from describing instance for subscription type
  223. // and spotprice can be gather from DescribeSpotPriceHistory API.
  224. // TO-DO: how would you calculate hourly price for subscription type, is it PRICE_YEARLY/HOURS_IN_THE_YEAR|MONTH?
  225. type AlibabaPricingDetails struct {
  226. // Represents hourly price for the given Alibaba cloud Product.
  227. HourlyPrice float32 `json:"hourlyPrice"`
  228. // Represents the unit in which Alibaba Product is billed can be Hour, Month or Year based on the billingMethod.
  229. PriceUnit string `json:"priceUnit"`
  230. // Original Price paid to acquire the Alibaba Product.
  231. TradePrice float32 `json:"tradePrice"`
  232. // Represents the currency unit of the price for billing Alibaba Product.
  233. CurrencyCode string `json:"currencyCode"`
  234. }
  235. func NewAlibabaPricingDetails(hourlyPrice float32, priceUnit string, tradePrice float32, currencyCode string) *AlibabaPricingDetails {
  236. return &AlibabaPricingDetails{
  237. HourlyPrice: hourlyPrice,
  238. PriceUnit: priceUnit,
  239. TradePrice: tradePrice,
  240. CurrencyCode: currencyCode,
  241. }
  242. }
  243. // AlibabaPricingTerms can have three types of supported billing method Pay-As-You-Go, Subscription and Premptible
  244. type AlibabaPricingTerms struct {
  245. BillingMethod string `json:"billingMethod"`
  246. PricingDetails *AlibabaPricingDetails `json:"pricingDetails"`
  247. }
  248. func NewAlibabaPricingTerms(billingMethod string, pricingDetails *AlibabaPricingDetails) *AlibabaPricingTerms {
  249. return &AlibabaPricingTerms{
  250. BillingMethod: billingMethod,
  251. PricingDetails: pricingDetails,
  252. }
  253. }
  254. // Alibaba Pricing struct carry the Attributes and pricing information for Node or PV
  255. type AlibabaPricing struct {
  256. NodeAttributes *AlibabaNodeAttributes
  257. PVAttributes *AlibabaPVAttributes
  258. PricingTerms *AlibabaPricingTerms
  259. Node *Node
  260. PV *PV
  261. }
  262. // Alibaba cloud's Provider struct
  263. type Alibaba struct {
  264. // Data to store Alibaba cloud's pricing struct, key in the map represents exact match to
  265. // node.features() or pv.features for easy lookup
  266. Pricing map[string]*AlibabaPricing
  267. // Lock Needed to provide thread safe
  268. DownloadPricingDataLock sync.RWMutex
  269. Clientset clustercache.ClusterCache
  270. Config *ProviderConfig
  271. *CustomProvider
  272. // The following fields are unexported because of avoiding any leak of secrets of these keys.
  273. // Alibaba Access key used specifically in signer interface used to sign API calls
  274. serviceAccountChecks *ServiceAccountChecks
  275. clusterAccountId string
  276. clusterRegion string
  277. accessKey *credentials.AccessKeyCredential
  278. // Map of regionID to sdk.client to call API for that region
  279. clients map[string]*sdk.Client
  280. }
  281. // GetAlibabaAccessKey return the Access Key used to interact with the Alibaba cloud, if not set it
  282. // set it first by looking at env variables else load it from secret files.
  283. func (alibaba *Alibaba) GetAlibabaAccessKey() (*credentials.AccessKeyCredential, error) {
  284. if alibaba.accessKeyisLoaded() {
  285. return alibaba.accessKey, nil
  286. }
  287. config, err := alibaba.GetConfig()
  288. if err != nil {
  289. return nil, fmt.Errorf("error getting the default config for Alibaba Cloud provider: %w", err)
  290. }
  291. if config.AlibabaServiceKeyName == "" {
  292. config.AlibabaServiceKeyName = env.GetAlibabaAccessKeyID()
  293. }
  294. if config.AlibabaServiceKeySecret == "" {
  295. config.AlibabaServiceKeySecret = env.GetAlibabaAccessKeySecret()
  296. }
  297. if config.AlibabaServiceKeyName == "" && config.AlibabaServiceKeySecret == "" {
  298. log.Debugf("missing service key values for Alibaba cloud integration attempting to use service account integration")
  299. err := alibaba.loadAlibabaAuthSecretAndSetEnv(true)
  300. if err != nil {
  301. return nil, fmt.Errorf("unable to set the Alibaba Cloud key/secret from config file %w", err)
  302. }
  303. config.AlibabaServiceKeyName = env.GetAlibabaAccessKeyID()
  304. config.AlibabaServiceKeySecret = env.GetAlibabaAccessKeySecret()
  305. }
  306. if config.AlibabaServiceKeyName == "" && config.AlibabaServiceKeySecret == "" {
  307. return nil, fmt.Errorf("failed to get the access key for the current alibaba account")
  308. }
  309. alibaba.accessKey = &credentials.AccessKeyCredential{AccessKeyId: env.GetAlibabaAccessKeyID(), AccessKeySecret: env.GetAlibabaAccessKeySecret()}
  310. return alibaba.accessKey, nil
  311. }
  312. // DownloadPricingData satisfies the provider interface and downloads the prices for Node instances and PVs.
  313. func (alibaba *Alibaba) DownloadPricingData() error {
  314. alibaba.DownloadPricingDataLock.Lock()
  315. defer alibaba.DownloadPricingDataLock.Unlock()
  316. var aak *credentials.AccessKeyCredential
  317. var err error
  318. if !alibaba.accessKeyisLoaded() {
  319. aak, err = alibaba.GetAlibabaAccessKey()
  320. if err != nil {
  321. return fmt.Errorf("unable to get the access key information: %w", err)
  322. }
  323. } else {
  324. aak = alibaba.accessKey
  325. }
  326. c, err := alibaba.Config.GetCustomPricingData()
  327. if err != nil {
  328. return fmt.Errorf("error downloading default pricing data: %w", err)
  329. }
  330. // Get all the nodes from Alibaba cluster.
  331. nodeList := alibaba.Clientset.GetAllNodes()
  332. var client *sdk.Client
  333. var signer *signers.AccessKeySigner
  334. var ok bool
  335. var lookupKey string
  336. alibaba.clients = make(map[string]*sdk.Client)
  337. alibaba.Pricing = make(map[string]*AlibabaPricing)
  338. for _, node := range nodeList {
  339. pricingObj := &AlibabaPricing{}
  340. slimK8sNode := generateSlimK8sNodeFromV1Node(node)
  341. if client, ok = alibaba.clients[slimK8sNode.RegionID]; !ok {
  342. client, err = sdk.NewClientWithAccessKey(slimK8sNode.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  343. if err != nil {
  344. return fmt.Errorf("unable to initiate alibaba cloud sdk client for region %s : %w", slimK8sNode.RegionID, err)
  345. }
  346. alibaba.clients[slimK8sNode.RegionID] = client
  347. }
  348. signer = signers.NewAccessKeySigner(aak)
  349. // Adjust the system Disk information of a Node by retrieving the details of associated disk. If unable to retrieve set it to empty
  350. // system disk to pass through and use defaults with Alibaba pricing API.
  351. instanceID := getInstanceIDFromProviderID(slimK8sNode.ProviderID)
  352. slimK8sNode.SystemDisk = getSystemDiskInfoOfANode(instanceID, slimK8sNode.RegionID, client, signer)
  353. lookupKey, err = determineKeyForPricing(slimK8sNode)
  354. if _, ok := alibaba.Pricing[lookupKey]; ok {
  355. log.Debugf("Pricing information for node with same features %s already exists hence skipping", lookupKey)
  356. continue
  357. }
  358. pricingObj, err = processDescribePriceAndCreateAlibabaPricing(client, slimK8sNode, signer, c)
  359. if err != nil {
  360. return fmt.Errorf("failed to create pricing information for node with type %s with error: %w", slimK8sNode.InstanceType, err)
  361. }
  362. alibaba.Pricing[lookupKey] = pricingObj
  363. }
  364. // set the first occurance of region from the node
  365. if alibaba.clusterRegion == "" {
  366. for _, node := range nodeList {
  367. if regionID, ok := node.Labels["topology.kubernetes.io/region"]; ok {
  368. alibaba.clusterRegion = regionID
  369. break
  370. }
  371. }
  372. }
  373. // PV pricing for only Cloud Disk for now.
  374. // TO-DO: Support both NAS(Network Attached storage) and OSS(Object Storage Service) type PVs
  375. pvList := alibaba.Clientset.GetAllPersistentVolumes()
  376. for _, pv := range pvList {
  377. pvRegion := determinePVRegion(pv)
  378. if pvRegion == "" {
  379. pvRegion = alibaba.clusterRegion
  380. }
  381. pricingObj := &AlibabaPricing{}
  382. slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, pvRegion)
  383. lookupKey, err = determineKeyForPricing(slimK8sDisk)
  384. if _, ok := alibaba.Pricing[lookupKey]; ok {
  385. log.Debugf("Pricing information for pv with same features %s already exists hence skipping", lookupKey)
  386. continue
  387. }
  388. if client, ok = alibaba.clients[slimK8sDisk.RegionID]; !ok {
  389. client, err = sdk.NewClientWithAccessKey(slimK8sDisk.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  390. if err != nil {
  391. return fmt.Errorf("unable to initiate alibaba cloud sdk client for region %s : %w", slimK8sDisk.RegionID, err)
  392. }
  393. alibaba.clients[slimK8sDisk.RegionID] = client
  394. }
  395. signer = signers.NewAccessKeySigner(aak)
  396. pricingObj, err = processDescribePriceAndCreateAlibabaPricing(client, slimK8sDisk, signer, c)
  397. if err != nil {
  398. return fmt.Errorf("failed to create pricing information for pv with category %s with error: %w", slimK8sDisk.DiskCategory, err)
  399. }
  400. alibaba.Pricing[lookupKey] = pricingObj
  401. }
  402. return nil
  403. }
  404. // AllNodePricing returns all the pricing data for all nodes and pvs
  405. func (alibaba *Alibaba) AllNodePricing() (interface{}, error) {
  406. alibaba.DownloadPricingDataLock.RLock()
  407. defer alibaba.DownloadPricingDataLock.RUnlock()
  408. return alibaba.Pricing, nil
  409. }
  410. // NodePricing gives pricing information of a specific node given by the key
  411. func (alibaba *Alibaba) NodePricing(key Key) (*Node, error) {
  412. alibaba.DownloadPricingDataLock.RLock()
  413. defer alibaba.DownloadPricingDataLock.RUnlock()
  414. // Get node features for the key
  415. keyFeature := key.Features()
  416. pricing, ok := alibaba.Pricing[keyFeature]
  417. if !ok {
  418. log.Warnf("Node pricing information not found for node with feature: %s", keyFeature)
  419. return &Node{}, nil
  420. }
  421. log.Debugf("returning the node price for the node with feature: %s", keyFeature)
  422. returnNode := pricing.Node
  423. return returnNode, nil
  424. }
  425. // PVPricing gives a pricing information of a specific PV given by PVkey
  426. func (alibaba *Alibaba) PVPricing(pvk PVKey) (*PV, error) {
  427. alibaba.DownloadPricingDataLock.RLock()
  428. defer alibaba.DownloadPricingDataLock.RUnlock()
  429. keyFeature := pvk.Features()
  430. pricing, ok := alibaba.Pricing[keyFeature]
  431. if !ok {
  432. log.Warnf("Persistent Volume pricing not found for PV with feature: %s", keyFeature)
  433. return &PV{}, nil
  434. }
  435. log.Debugf("returning the PV price for the node with feature: %s", keyFeature)
  436. return pricing.PV, nil
  437. }
  438. // Stubbed NetworkPricing for Alibaba Cloud. Will look at this in Next PR
  439. func (alibaba *Alibaba) NetworkPricing() (*Network, error) {
  440. return &Network{
  441. ZoneNetworkEgressCost: 0.0,
  442. RegionNetworkEgressCost: 0.0,
  443. InternetNetworkEgressCost: 0.0,
  444. }, nil
  445. }
  446. // Stubbed LoadBalancerPricing for Alibaba Cloud. Will look at this in Next PR
  447. func (alibaba *Alibaba) LoadBalancerPricing() (*LoadBalancer, error) {
  448. return &LoadBalancer{
  449. Cost: 0.0,
  450. }, nil
  451. }
  452. func (alibaba *Alibaba) GetConfig() (*CustomPricing, error) {
  453. c, err := alibaba.Config.GetCustomPricingData()
  454. if err != nil {
  455. return nil, err
  456. }
  457. if c.Discount == "" {
  458. c.Discount = "0%"
  459. }
  460. if c.NegotiatedDiscount == "" {
  461. c.NegotiatedDiscount = "0%"
  462. }
  463. if c.ShareTenancyCosts == "" {
  464. c.ShareTenancyCosts = defaultShareTenancyCost
  465. }
  466. return c, nil
  467. }
  468. // Load once and cache the result (even on failure). This is an install time secret, so
  469. // we don't expect the secret to change. If it does, however, we can force reload using
  470. // the input parameter.
  471. func (alibaba *Alibaba) loadAlibabaAuthSecretAndSetEnv(force bool) error {
  472. if !force && alibaba.accessKeyisLoaded() {
  473. return nil
  474. }
  475. exists, err := fileutil.FileExists(authSecretPath)
  476. if !exists || err != nil {
  477. return fmt.Errorf("failed to locate service account file: %s with err: %w", authSecretPath, err)
  478. }
  479. result, err := ioutil.ReadFile(authSecretPath)
  480. if err != nil {
  481. return fmt.Errorf("failed to read service account file: %s with err: %w", authSecretPath, err)
  482. }
  483. var ak *AlibabaAccessKey
  484. err = json.Unmarshal(result, &ak)
  485. if err != nil {
  486. return fmt.Errorf("failed to unmarshall access key id and access key secret with err: %w", err)
  487. }
  488. err = env.Set(env.AlibabaAccessKeyIDEnvVar, ak.AccessKeyID)
  489. if err != nil {
  490. return fmt.Errorf("failed to set environment variable: %s with err: %w", env.AlibabaAccessKeyIDEnvVar, err)
  491. }
  492. err = env.Set(env.AlibabaAccessKeySecretEnvVar, ak.SecretAccessKey)
  493. if err != nil {
  494. return fmt.Errorf("failed to set environment variable: %s with err: %w", env.AlibabaAccessKeySecretEnvVar, err)
  495. }
  496. alibaba.accessKey = &credentials.AccessKeyCredential{
  497. AccessKeyId: ak.AccessKeyID,
  498. AccessKeySecret: ak.SecretAccessKey,
  499. }
  500. return nil
  501. }
  502. // Regions returns a current supported list of Alibaba regions
  503. func (alibaba *Alibaba) Regions() []string {
  504. return alibabaRegions
  505. }
  506. // ClusterInfo returns information about Alibaba Cloud cluster, as provided by metadata.
  507. func (alibaba *Alibaba) ClusterInfo() (map[string]string, error) {
  508. c, err := alibaba.GetConfig()
  509. if err != nil {
  510. return nil, fmt.Errorf("failed to getConfig with err: %w", err)
  511. }
  512. var clusterName string
  513. if c.ClusterName != "" {
  514. clusterName = c.ClusterName
  515. }
  516. // Set it to environment clusterID if not set at this point
  517. if clusterName == "" {
  518. clusterName = env.GetClusterID()
  519. }
  520. m := make(map[string]string)
  521. m["name"] = clusterName
  522. m["provider"] = kubecost.AlibabaProvider
  523. m["project"] = alibaba.clusterAccountId
  524. m["region"] = alibaba.clusterRegion
  525. m["id"] = env.GetClusterID()
  526. return m, nil
  527. }
  528. // Will look at this in Next PR if needed
  529. func (alibaba *Alibaba) GetAddresses() ([]byte, error) {
  530. return nil, nil
  531. }
  532. // Will look at this in Next PR if needed
  533. func (alibaba *Alibaba) GetDisks() ([]byte, error) {
  534. return nil, nil
  535. }
  536. func (alibaba *Alibaba) GetOrphanedResources() ([]OrphanedResource, error) {
  537. return nil, errors.New("not implemented")
  538. }
  539. func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
  540. return alibaba.Config.Update(func(c *CustomPricing) error {
  541. if updateType != "" {
  542. return fmt.Errorf("UpdateConfig for Alibaba Provider doesn't support updateType %s at this time", updateType)
  543. } else {
  544. a := make(map[string]interface{})
  545. err := json.NewDecoder(r).Decode(&a)
  546. if err != nil {
  547. return err
  548. }
  549. for k, v := range a {
  550. kUpper := strings.Title(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
  551. vstr, ok := v.(string)
  552. if ok {
  553. err := SetCustomPricingField(c, kUpper, vstr)
  554. if err != nil {
  555. return err
  556. }
  557. } else {
  558. return fmt.Errorf("type error while updating config for %s", kUpper)
  559. }
  560. }
  561. }
  562. if env.IsRemoteEnabled() {
  563. err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
  564. if err != nil {
  565. return err
  566. }
  567. }
  568. return nil
  569. })
  570. }
  571. func (alibaba *Alibaba) UpdateConfigFromConfigMap(cm map[string]string) (*CustomPricing, error) {
  572. return alibaba.Config.UpdateFromMap(cm)
  573. }
  574. // Will look at this in Next PR if needed
  575. func (alibaba *Alibaba) GetManagementPlatform() (string, error) {
  576. return "", nil
  577. }
  578. // Will look at this in Next PR if needed
  579. func (alibaba *Alibaba) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string {
  580. return ""
  581. }
  582. // Will look at this in Next PR if needed
  583. func (alibaba *Alibaba) ApplyReservedInstancePricing(nodes map[string]*Node) {
  584. }
  585. // Will look at this in Next PR if needed
  586. func (alibaba *Alibaba) ServiceAccountStatus() *ServiceAccountStatus {
  587. return &ServiceAccountStatus{}
  588. }
  589. // Will look at this in Next PR if needed
  590. func (alibaba *Alibaba) PricingSourceStatus() map[string]*PricingSource {
  591. return map[string]*PricingSource{}
  592. }
  593. // Will look at this in Next PR if needed
  594. func (alibaba *Alibaba) ClusterManagementPricing() (string, float64, error) {
  595. return "", 0.0, nil
  596. }
  597. // Will look at this in Next PR if needed
  598. func (alibaba *Alibaba) CombinedDiscountForNode(string, bool, float64, float64) float64 {
  599. return 0.0
  600. }
  601. func (alibaba *Alibaba) accessKeyisLoaded() bool {
  602. return alibaba.accessKey != nil
  603. }
  604. type AlibabaNodeKey struct {
  605. ProviderID string
  606. RegionID string
  607. InstanceType string
  608. OSType string
  609. OptimizedKeyword string //If IsIoOptimized is true use the word optimize in the Node key and if its not optimized use the word nonoptimize
  610. SystemDiskCategory string
  611. SystemDiskSizeInGiB string
  612. SystemDiskPerformanceLevel string
  613. }
  614. func NewAlibabaNodeKey(node *SlimK8sNode, optimizedKeyword, systemDiskCategory, systemDiskSizeInGiB, systemDiskPerfromanceLevel string) *AlibabaNodeKey {
  615. var providerID, regionID, instanceType, osType string
  616. if node != nil {
  617. providerID = node.ProviderID
  618. regionID = node.RegionID
  619. instanceType = node.InstanceType
  620. osType = node.OSType
  621. }
  622. return &AlibabaNodeKey{
  623. ProviderID: providerID,
  624. RegionID: regionID,
  625. InstanceType: instanceType,
  626. OSType: osType,
  627. OptimizedKeyword: optimizedKeyword,
  628. SystemDiskCategory: systemDiskCategory,
  629. SystemDiskSizeInGiB: systemDiskSizeInGiB,
  630. SystemDiskPerformanceLevel: systemDiskPerfromanceLevel,
  631. }
  632. }
  633. func (alibabaNodeKey *AlibabaNodeKey) ID() string {
  634. return alibabaNodeKey.ProviderID
  635. }
  636. func (alibabaNodeKey *AlibabaNodeKey) Features() string {
  637. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaNodeKey.RegionID, alibabaNodeKey.InstanceType, alibabaNodeKey.OSType,
  638. alibabaNodeKey.OptimizedKeyword, alibabaNodeKey.SystemDiskCategory, alibabaNodeKey.SystemDiskSizeInGiB, alibabaNodeKey.SystemDiskPerformanceLevel})
  639. return strings.Join(keyLookup, "::")
  640. }
  641. func (alibabaNodeKey *AlibabaNodeKey) GPUType() string {
  642. return ""
  643. }
  644. func (alibabaNodeKey *AlibabaNodeKey) GPUCount() int {
  645. return 0
  646. }
  647. // Get's the key for the k8s node input
  648. func (alibaba *Alibaba) GetKey(mapValue map[string]string, node *v1.Node) Key {
  649. slimK8sNode := generateSlimK8sNodeFromV1Node(node)
  650. var aak *credentials.AccessKeyCredential
  651. var err error
  652. var skipSystemDiskRetrieval, ok bool
  653. var client *sdk.Client
  654. var signer *signers.AccessKeySigner
  655. if !alibaba.accessKeyisLoaded() {
  656. aak, err = alibaba.GetAlibabaAccessKey()
  657. if err != nil {
  658. log.Warnf("unable to set the signer for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: %v", slimK8sNode.ProviderID, err)
  659. skipSystemDiskRetrieval = true
  660. }
  661. } else {
  662. aak = alibaba.accessKey
  663. }
  664. signer = signers.NewAccessKeySigner(aak)
  665. if client, ok = alibaba.clients[slimK8sNode.RegionID]; !ok {
  666. client, err = sdk.NewClientWithAccessKey(slimK8sNode.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  667. if err != nil {
  668. log.Warnf("unable to set the client for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: %v", slimK8sNode.ProviderID, err)
  669. skipSystemDiskRetrieval = true
  670. }
  671. alibaba.clients[slimK8sNode.RegionID] = client
  672. }
  673. optimizedKeyword := ""
  674. if slimK8sNode.IsIoOptimized {
  675. optimizedKeyword = ALIBABA_OPTIMIZE_KEYWORD
  676. } else {
  677. optimizedKeyword = ALIBABA_NON_OPTIMIZE_KEYWORD
  678. }
  679. var diskCategory, diskSizeInGiB, diskPerformanceLevel string
  680. if skipSystemDiskRetrieval {
  681. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  682. }
  683. instanceID := getInstanceIDFromProviderID(slimK8sNode.ProviderID)
  684. slimK8sNode.SystemDisk = getSystemDiskInfoOfANode(instanceID, slimK8sNode.RegionID, client, signer)
  685. if slimK8sNode.SystemDisk != nil {
  686. diskCategory = slimK8sNode.SystemDisk.DiskCategory
  687. diskSizeInGiB = slimK8sNode.SystemDisk.SizeInGiB
  688. diskPerformanceLevel = slimK8sNode.SystemDisk.PerformanceLevel
  689. }
  690. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  691. }
  692. type AlibabaPVKey struct {
  693. ProviderID string
  694. RegionID string
  695. PVType string
  696. PVSubType string
  697. PVCategory string
  698. PVPerformaceLevel string
  699. StorageClassName string
  700. SizeInGiB string
  701. }
  702. func (alibaba *Alibaba) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
  703. regionID := defaultRegion
  704. // If default Region is not passed default it to cluster region ID.
  705. if defaultRegion == "" {
  706. regionID = alibaba.clusterRegion
  707. }
  708. slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, defaultRegion)
  709. return &AlibabaPVKey{
  710. ProviderID: slimK8sDisk.ProviderID,
  711. RegionID: regionID,
  712. PVType: ALIBABA_PV_CLOUD_DISK_TYPE,
  713. PVSubType: slimK8sDisk.DiskType,
  714. PVCategory: slimK8sDisk.DiskCategory,
  715. PVPerformaceLevel: slimK8sDisk.PerformanceLevel,
  716. StorageClassName: pv.Spec.StorageClassName,
  717. SizeInGiB: slimK8sDisk.SizeInGiB,
  718. }
  719. }
  720. func (alibabaPVKey *AlibabaPVKey) Features() string {
  721. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaPVKey.RegionID, alibabaPVKey.PVSubType, alibabaPVKey.PVCategory, alibabaPVKey.PVPerformaceLevel, alibabaPVKey.SizeInGiB})
  722. return strings.Join(keyLookup, "::")
  723. }
  724. func (alibabaPVKey *AlibabaPVKey) ID() string {
  725. return alibabaPVKey.ProviderID
  726. }
  727. // Get storage class information for PV.
  728. func (alibabaPVKey *AlibabaPVKey) GetStorageClass() string {
  729. return alibabaPVKey.StorageClassName
  730. }
  731. // Helper functions for alibabaprovider.go
  732. // createDescribePriceACSRequest creates the HTTP GET request for the required resources' Price information,
  733. // When supporting subscription and Premptible resources this HTTP call needs to be modified with PriceUnit information
  734. // When supporting different new type of instances like Compute Optimized, Memory Optimized etc make sure you add the instance type
  735. // in unit test and check if it works or not to create the ack request and processDescribePriceAndCreateAlibabaPricing function
  736. // else more paramters need to be pulled from kubernetes node response or gather infromation from elsewhere and function modified.
  737. func createDescribePriceACSRequest(i interface{}) (*requests.CommonRequest, error) {
  738. request := requests.NewCommonRequest()
  739. request.Method = requests.GET
  740. request.Product = ALIBABA_ECS_PRODUCT_CODE
  741. request.Domain = ALIBABA_ECS_DOMAIN
  742. request.Version = ALIBABA_ECS_VERSION
  743. request.Scheme = requests.HTTPS
  744. request.ApiName = ALIBABA_DESCRIBE_PRICE_API_ACTION
  745. switch i.(type) {
  746. case *SlimK8sNode:
  747. node := i.(*SlimK8sNode)
  748. request.QueryParams["RegionId"] = node.RegionID
  749. request.QueryParams["ResourceType"] = ALIBABA_INSTANCE_RESOURCE_TYPE
  750. request.QueryParams["InstanceType"] = node.InstanceType
  751. request.QueryParams["PriceUnit"] = node.PriceUnit
  752. if node.SystemDisk != nil {
  753. // Only if the required information is present it should be overridden else default it via the API
  754. if node.SystemDisk.DiskCategory != "" {
  755. request.QueryParams["SystemDisk.Category"] = node.SystemDisk.DiskCategory
  756. }
  757. if node.SystemDisk.SizeInGiB != "" {
  758. request.QueryParams["SystemDisk.Size"] = node.SystemDisk.SizeInGiB
  759. }
  760. if node.SystemDisk.PerformanceLevel != "" {
  761. request.QueryParams["SystemDisk.PerformanceLevel"] = node.SystemDisk.PerformanceLevel
  762. }
  763. } else {
  764. // For Enhanced General Purpose Type g6e SystemDisk.Category param doesn't default right,
  765. // need it to be specifically assigned to "cloud_ssd" otherwise there's errors
  766. if node.InstanceTypeFamily == ALIBABA_ENHANCED_GENERAL_PURPOSE_TYPE {
  767. request.QueryParams["SystemDisk.Category"] = ALIBABA_DISK_CLOUD_ESSD_CATEGORY
  768. }
  769. }
  770. request.TransToAcsRequest()
  771. return request, nil
  772. case *SlimK8sDisk:
  773. disk := i.(*SlimK8sDisk)
  774. request.QueryParams["RegionId"] = disk.RegionID
  775. request.QueryParams["PriceUnit"] = disk.PriceUnit
  776. request.QueryParams["ResourceType"] = ALIBABA_DISK_RESOURCE_TYPE
  777. request.QueryParams[fmt.Sprintf("%s.%d.Size", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.SizeInGiB
  778. request.QueryParams[fmt.Sprintf("%s.%d.Category", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.DiskCategory
  779. // Performance level defaults to PL1 if not present in volume attribute.
  780. if disk.PerformanceLevel != "" {
  781. request.QueryParams[fmt.Sprintf("%s.%d.PerformanceLevel", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.PerformanceLevel
  782. }
  783. request.TransToAcsRequest()
  784. return request, nil
  785. default:
  786. return nil, fmt.Errorf("unsupported ECS type (%T) for DescribePrice at this time", i)
  787. }
  788. }
  789. // createDescribeDisksCSRequest creates the HTTP GET Request to map the system disk to the InstanceID
  790. func createDescribeDisksACSRequest(instanceID, regionID, diskType string) (*requests.CommonRequest, error) {
  791. request := requests.NewCommonRequest()
  792. request.Method = requests.GET
  793. request.Product = ALIBABA_ECS_PRODUCT_CODE
  794. request.Domain = ALIBABA_ECS_DOMAIN
  795. request.Version = ALIBABA_ECS_VERSION
  796. request.Scheme = requests.HTTPS
  797. request.ApiName = ALIBABA_DESCRIBE_DISK_API_ACTION
  798. request.QueryParams["RegionId"] = regionID
  799. request.QueryParams["InstanceId"] = instanceID
  800. request.QueryParams["DiskType"] = diskType
  801. request.TransToAcsRequest()
  802. return request, nil
  803. }
  804. // determineKeyForPricing generate a unique key from SlimK8sNode object that is constructed from v1.Node object and
  805. // SlimK8sDisk that is constructed from v1.PersistentVolume.
  806. func determineKeyForPricing(i interface{}) (string, error) {
  807. if i == nil {
  808. return "", fmt.Errorf("nil component passed to determine key")
  809. }
  810. switch i.(type) {
  811. case *SlimK8sNode:
  812. node := i.(*SlimK8sNode)
  813. var diskCategory, diskSizeInGiB, diskPerformanceLevel string
  814. if node.SystemDisk != nil {
  815. diskCategory = node.SystemDisk.DiskCategory
  816. diskSizeInGiB = node.SystemDisk.SizeInGiB
  817. diskPerformanceLevel = node.SystemDisk.PerformanceLevel
  818. }
  819. if node.IsIoOptimized {
  820. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{node.RegionID, node.InstanceType, node.OSType, ALIBABA_OPTIMIZE_KEYWORD, diskCategory, diskSizeInGiB, diskPerformanceLevel})
  821. return strings.Join(keyLookup, "::"), nil
  822. } else {
  823. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{node.RegionID, node.InstanceType, node.OSType, ALIBABA_NON_OPTIMIZE_KEYWORD, diskCategory, diskSizeInGiB, diskPerformanceLevel})
  824. return strings.Join(keyLookup, "::"), nil
  825. }
  826. case *SlimK8sDisk:
  827. disk := i.(*SlimK8sDisk)
  828. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{disk.RegionID, disk.DiskType, disk.DiskCategory, disk.PerformanceLevel, disk.SizeInGiB})
  829. return strings.Join(keyLookup, "::"), nil
  830. default:
  831. return "", fmt.Errorf("unsupported ECS type (%T) at this time", i)
  832. }
  833. }
  834. // Below structs are used to unmarshal json response of Alibaba cloud's API DescribePrice
  835. type Price struct {
  836. OriginalPrice float32 `json:"OriginalPrice"`
  837. ReservedInstanceHourPrice float32 `json:"ReservedInstanceHourPrice"`
  838. DiscountPrice float32 `json:"DiscountPrice"`
  839. Currency string `json:"Currency"`
  840. TradePrice float32 `json:"TradePrice"`
  841. }
  842. type PriceInfo struct {
  843. Price Price `json:"Price"`
  844. }
  845. type DescribePriceResponse struct {
  846. RequestId string `json:"RequestId"`
  847. PriceInfo PriceInfo `json:"PriceInfo"`
  848. }
  849. // processDescribePriceAndCreateAlibabaPricing processes the DescribePrice API and generates the pricing information for alibaba node resource and alibaba pv resource that's backed by cloud disk.
  850. func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface{}, signer *signers.AccessKeySigner, custom *CustomPricing) (pricing *AlibabaPricing, err error) {
  851. pricing = &AlibabaPricing{}
  852. var response DescribePriceResponse
  853. if i == nil {
  854. return nil, fmt.Errorf("nil component passed to process the pricing information")
  855. }
  856. switch i.(type) {
  857. case *SlimK8sNode:
  858. node := i.(*SlimK8sNode)
  859. req, err := createDescribePriceACSRequest(node)
  860. if err != nil {
  861. return nil, err
  862. }
  863. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  864. pricing.NodeAttributes = NewAlibabaNodeAttributes(node)
  865. if err != nil || resp.GetHttpStatus() != 200 {
  866. // Can be defaulted to some value here?
  867. return nil, fmt.Errorf("unable to fetch information for node with InstanceType: %v", node.InstanceType)
  868. } else {
  869. // This is where population of Pricing happens
  870. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  871. if err != nil {
  872. return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
  873. }
  874. // TO-DO : Ask in PR How to get the defaults is it equal to AWS/GCP defaults? And what needs to be returned
  875. pricing.Node = &Node{
  876. Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
  877. BaseCPUPrice: custom.CPU,
  878. BaseRAMPrice: custom.RAM,
  879. BaseGPUPrice: custom.GPU,
  880. }
  881. // TO-DO : Currently with Pay-As-You-go Offering TradePrice = HourlyPrice , When support happens to other type HourlyPrice Need to be determined.
  882. pricing.PricingTerms = NewAlibabaPricingTerms(ALIBABA_PAY_AS_YOU_GO_BILLING, NewAlibabaPricingDetails(response.PriceInfo.Price.TradePrice, ALIBABA_HOUR_PRICE_UNIT, response.PriceInfo.Price.TradePrice, response.PriceInfo.Price.Currency))
  883. }
  884. case *SlimK8sDisk:
  885. disk := i.(*SlimK8sDisk)
  886. req, err := createDescribePriceACSRequest(disk)
  887. if err != nil {
  888. return nil, err
  889. }
  890. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  891. if err != nil || resp.GetHttpStatus() != 200 {
  892. return nil, fmt.Errorf("unable to fetch information for disk with DiskType: %v with err: %w", disk.DiskCategory, err)
  893. } else {
  894. // This is where population of Pricing happens
  895. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  896. if err != nil {
  897. return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
  898. }
  899. pricing.PVAttributes = NewAlibabaPVAttributes(disk)
  900. pricing.PV = &PV{
  901. Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
  902. }
  903. // TO-DO : Disk has support for Hour and Month but pricing API is failing for month for disk(Research why?) and same challenge as node pricing no prepaid/postpaid distinction in v1.PersistentVolume object have to look at APIs for th information.
  904. pricing.PricingTerms = NewAlibabaPricingTerms(ALIBABA_PAY_AS_YOU_GO_BILLING, NewAlibabaPricingDetails(response.PriceInfo.Price.TradePrice, ALIBABA_HOUR_PRICE_UNIT, response.PriceInfo.Price.TradePrice, response.PriceInfo.Price.Currency))
  905. }
  906. default:
  907. return nil, fmt.Errorf("unsupported ECS Pricing component of type (%T) at this time", i)
  908. }
  909. return pricing, nil
  910. }
  911. // This function is to get the InstanceFamily from the InstanceType , convention followed in
  912. // instance type is ecs.[FamilyName].[DifferentSize], it gets the familyName , if it is unable to get it
  913. // it lists the instance family name as Unknown.
  914. func getInstanceFamilyFromType(instanceType string) string {
  915. splitinstanceType := strings.Split(instanceType, ".")
  916. if len(splitinstanceType) != 3 {
  917. log.Warnf("unable to find the family of the instance type %s, returning its family type unknown", instanceType)
  918. return ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE
  919. }
  920. if !slices.Contains(alibabaInstanceFamilies, splitinstanceType[1]) {
  921. log.Warnf("currently the instance family type %s is not valid or not tested completely for pricing API", instanceType)
  922. return ALIBABA_NOT_SUPPORTED_INSTANCE_FAMILY_TYPE
  923. }
  924. return splitinstanceType[1]
  925. }
  926. // getInstanceIDFromProviderID returns the instance ID associated with the Node. A *v1.Node providerID in Alibaba cloud
  927. // is of <REGION-ID>.<INSTANCE-ID>. This function returns the Instance ID for the given ProviderID. if its unable to interpret
  928. // it defaults to empty string.
  929. func getInstanceIDFromProviderID(providerID string) string {
  930. if providerID == "" {
  931. return ""
  932. }
  933. splitStrings := strings.Split(providerID, ".")
  934. if len(splitStrings) < 2 {
  935. return ""
  936. }
  937. return splitStrings[1]
  938. }
  939. type Disk struct {
  940. Category string `json:"Category"`
  941. Size int `json:"Size"`
  942. PerformanceLevel string `json:"PerformanceLevel"`
  943. Type string `json:"Type"`
  944. RegionId string `json:"RegionId"`
  945. DiskId string `json:"DiskId"`
  946. DiskChargeType string `json:"DiskChargeType"`
  947. }
  948. type Disks struct {
  949. Disk []*Disk `json:"Disk"`
  950. }
  951. type DescribeDiskResponse struct {
  952. TotalCount int `json:"TotalCount"`
  953. Disks *Disks `json:"Disks"`
  954. }
  955. // getSystemDiskInfoOfANode gets the relevant System disk information associated with the Node given by the instanceID
  956. // in form of a SlimK8sDisk with only relevant information that can adjust the node pricing. If any error occurs return
  957. // an empty disk to not impact any default set at the price retrieval of the node.
  958. func getSystemDiskInfoOfANode(instanceID, regionID string, client *sdk.Client, signer *signers.AccessKeySigner) (systemDisk *SlimK8sDisk) {
  959. systemDisk = &SlimK8sDisk{}
  960. var response DescribeDiskResponse
  961. // if instanceID is empty string return an empty k8s
  962. if instanceID == "" {
  963. return
  964. }
  965. req, err := createDescribeDisksACSRequest(instanceID, regionID, ALIBABA_SYSTEM_DISK_CATEGORY)
  966. // if any error occurs return an empty disk to not impact default pricing.
  967. if err != nil {
  968. log.Warnf("Unable to create Describe Disk Request with err: %v for node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, instanceID)
  969. return
  970. }
  971. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  972. if err != nil || resp.GetHttpStatus() != 200 {
  973. log.Warnf("Unable to process Describe Disk request with err: %v and errcode: %d for the node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, resp.GetHttpStatus(), instanceID)
  974. return
  975. } else {
  976. // This is where population of Pricing happens
  977. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  978. if err != nil {
  979. log.Warnf("Unable to unmarshall Describe Disk response with err: %v for the node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, instanceID)
  980. return
  981. }
  982. // Every instance should only have one system disk per Alibaba Cloud documentation https://www.alibabacloud.com/help/en/elastic-compute-service/latest/block-storage-overview-disks,
  983. // if TotalCount is not 1 just return empty and let it not impact default pricing.
  984. if response.TotalCount != 1 {
  985. log.Warnf("Total count of system disk for node with InstanceID: %s is not 1, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  986. return
  987. }
  988. if response.Disks == nil {
  989. log.Warnf("Disks information missing for node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  990. return
  991. }
  992. if len(response.Disks.Disk) < 1 {
  993. log.Warnf("Total number of system disk for node with InstanceID: %s is less than 1, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  994. return
  995. }
  996. // TO-DO: When supporting Subscription type disk, you can leverge the disk.DiskChargeType here to map it to subscription type.
  997. systemDisk := response.Disks.Disk[0]
  998. return NewSlimK8sDisk(systemDisk.Type, systemDisk.RegionId, ALIBABA_HOUR_PRICE_UNIT, systemDisk.Category, systemDisk.PerformanceLevel, systemDisk.DiskId, "", fmt.Sprintf("%d", systemDisk.Size))
  999. }
  1000. }
  1001. // generateSlimK8sNodeFromV1Node generates SlimK8sNode struct from v1.Node to fetch pricing information and call alibaba API.
  1002. func generateSlimK8sNodeFromV1Node(node *v1.Node) *SlimK8sNode {
  1003. var regionID, osType, instanceType, providerID, priceUnit, instanceFamily string
  1004. var memorySizeInKiB string // TO-DO: try to convert it into float
  1005. var ok, IsIoOptimized bool
  1006. if regionID, ok = node.Labels["topology.kubernetes.io/region"]; !ok {
  1007. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1008. log.Debugf("No RegionID label for the node: %s", node.Name)
  1009. }
  1010. if osType, ok = node.Labels["beta.kubernetes.io/os"]; !ok {
  1011. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1012. log.Debugf("OS type undetected for the node: %s", node.Name)
  1013. }
  1014. if instanceType, ok = node.Labels["node.kubernetes.io/instance-type"]; !ok {
  1015. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1016. log.Debugf("Instance Type undetected for the node: %s", node.Name)
  1017. }
  1018. instanceFamily = getInstanceFamilyFromType(instanceType)
  1019. memorySizeInKiB = fmt.Sprintf("%s", node.Status.Capacity.Memory())
  1020. providerID = node.Spec.ProviderID // Alibaba Cloud provider doesnt follow convention of prefix with cloud provider name
  1021. // Looking at current Instance offering , all of the Instances seem to be I/O optimized - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/instance-family
  1022. // Basic price Json has it as part of the key so defaulting to true.
  1023. IsIoOptimized = true
  1024. priceUnit = ALIBABA_HOUR_PRICE_UNIT
  1025. systemDisk := &SlimK8sDisk{}
  1026. return NewSlimK8sNode(instanceType, regionID, priceUnit, memorySizeInKiB, osType, providerID, instanceFamily, IsIoOptimized, systemDisk)
  1027. }
  1028. // getNumericalValueFromResourceQuantity returns the numericalValue of the resourceQuantity
  1029. // An example is: 20Gi returns to 20. If any error occurs it returns the default value used in describePrice API which is 2000.
  1030. func getNumericalValueFromResourceQuantity(quantity string) (value string) {
  1031. // defaulting when any panic or empty string occurs.
  1032. defer func() {
  1033. log.Debugf("unable to determine the size of the PV so defaulting the size to %s", ALIBABA_DEFAULT_DATADISK_SIZE)
  1034. if err := recover(); err != nil {
  1035. value = ALIBABA_DEFAULT_DATADISK_SIZE
  1036. }
  1037. if value == "" {
  1038. value = ALIBABA_DEFAULT_DATADISK_SIZE
  1039. }
  1040. }()
  1041. res := sizeRegEx.FindAllStringSubmatch(quantity, 1)
  1042. value = res[0][1]
  1043. return
  1044. }
  1045. // generateSlimK8sDiskFromV1PV function generates SlimK8sDisk from v1.PersistentVolume
  1046. // to generate slim disk type that can be used to fetch pricing information for Data disk type.
  1047. func generateSlimK8sDiskFromV1PV(pv *v1.PersistentVolume, regionID string) *SlimK8sDisk {
  1048. // All PVs are data disks while local disk are categorized as system disk
  1049. diskType := ALIBABA_DATA_DISK_CATEGORY
  1050. //TO-DO: Disk supports month and hour prices , defaulting to hour
  1051. priceUnit := ALIBABA_HOUR_PRICE_UNIT
  1052. sizeQuantity := fmt.Sprintf("%s", pv.Spec.Capacity.Storage())
  1053. // res := sizeRegEx.FindAllStringSubmatch(sizeQuantity, 1)
  1054. sizeInGiB := getNumericalValueFromResourceQuantity(sizeQuantity)
  1055. providerID := ""
  1056. if pv.Spec.CSI != nil {
  1057. providerID = pv.Spec.CSI.VolumeHandle
  1058. } else {
  1059. providerID = pv.Name // Looks like pv name is same as providerID in Alibaba k8s cluster
  1060. }
  1061. // Performance level being empty string gets defaulted in describePrice to PL1.
  1062. performanceLevel := ""
  1063. diskCategory := ""
  1064. if pv.Spec.CSI != nil {
  1065. if val, ok := pv.Spec.CSI.VolumeAttributes["performanceLevel"]; ok {
  1066. performanceLevel = val
  1067. }
  1068. if val, ok := pv.Spec.CSI.VolumeAttributes["type"]; ok {
  1069. diskCategory = val
  1070. }
  1071. }
  1072. // Highly unlikely that label pv.Spec.CSI.VolumeAttributes["type"] doesn't exist but if occured default to cloud (most basic disk type)
  1073. if diskCategory == "" {
  1074. diskCategory = ALIBABA_DISK_CLOUD_CATEGORY
  1075. }
  1076. return NewSlimK8sDisk(diskType, regionID, priceUnit, diskCategory, performanceLevel, providerID, pv.Spec.StorageClassName, sizeInGiB)
  1077. }
  1078. // determinePVRegion determines associated region for a particular PV based on the following priority, which can be changed and any other path to determine region can be added!
  1079. // if topology.diskplugin.csi.alibabacloud.com/region label/annotation is passed during PV creation return that as the PV region.
  1080. // if topology.diskplugin.csi.alibabacloud.com/zone label/annotation is passed during PV creation determine the region based on this pv label.
  1081. // if neither of the above label/annotation is present check node affinity for the zone affinity and determine the region based on this zone.
  1082. // if nether of the above yields a region , return empty string to default it to cluster region.
  1083. func determinePVRegion(pv *v1.PersistentVolume) string {
  1084. // if "topology.diskplugin.csi.alibabacloud.com/region" is present as a label or annotation return that as the PV region
  1085. if val, ok := pv.Labels[ALIBABA_DISK_TOPOLOGY_REGION_LABEL]; ok {
  1086. log.Debugf("determinePVRegion returned a region value of: %s through label: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_REGION_LABEL, pv.Name)
  1087. return val
  1088. }
  1089. if val, ok := pv.Annotations[ALIBABA_DISK_TOPOLOGY_REGION_LABEL]; ok {
  1090. log.Debugf("determinePVRegion returned a region value of: %s through annotation: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_REGION_LABEL, pv.Name)
  1091. return val
  1092. }
  1093. // if "topology.diskplugin.csi.alibabacloud.com/zone" is present as a label or annotation set it as the PV zone before looking at node affinity to determine the region PV belongs too
  1094. var pvZone string
  1095. if val, ok := pv.Labels[ALIBABA_DISK_TOPOLOGY_ZONE_LABEL]; ok {
  1096. log.Debugf("determinePVRegion will set zone value to: %s through label: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1097. pvZone = val
  1098. }
  1099. if pvZone == "" {
  1100. if val, ok := pv.Annotations[ALIBABA_DISK_TOPOLOGY_ZONE_LABEL]; ok {
  1101. log.Debugf("determinePVRegion will set zone value to: %s through annotation: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1102. pvZone = val
  1103. }
  1104. }
  1105. if pvZone == "" {
  1106. // zone and regionID labels are optional in Alibaba PV creation, while PV through UI creation put's a zone PV is associated with and the region
  1107. // can be determined from this information. If pv is provision via yaml and the block is missing that's the only time it gets defaulted to clusterRegion.
  1108. if pv.Spec.NodeAffinity != nil {
  1109. nodeAffinity := pv.Spec.NodeAffinity
  1110. if nodeAffinity.Required != nil && nodeAffinity.Required.NodeSelectorTerms != nil {
  1111. for _, nodeSelectorTerm := range nodeAffinity.Required.NodeSelectorTerms {
  1112. matchExpression := nodeSelectorTerm.MatchExpressions
  1113. for _, nodeSelectorRequirement := range matchExpression {
  1114. if nodeSelectorRequirement.Key == ALIBABA_DISK_TOPOLOGY_ZONE_LABEL {
  1115. log.Debugf("determinePVRegion will set zone value to: %s through node affinity label: %s for PV name: %s", nodeSelectorRequirement.Values[0], ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1116. pvZone = nodeSelectorRequirement.Values[0]
  1117. }
  1118. }
  1119. }
  1120. }
  1121. }
  1122. }
  1123. for _, region := range alibabaRegions {
  1124. if strings.Contains(pvZone, region) {
  1125. log.Debugf("determinePVRegion determined region of %s through zone affiliation of the PV %s\n", region, pvZone)
  1126. return region
  1127. }
  1128. }
  1129. return ""
  1130. }