aliyunprovider.go 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297
  1. package cloud
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "regexp"
  8. "strings"
  9. "sync"
  10. "time"
  11. "github.com/aliyun/alibaba-cloud-sdk-go/sdk"
  12. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
  13. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/signers"
  14. "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
  15. "github.com/opencost/opencost/pkg/clustercache"
  16. "github.com/opencost/opencost/pkg/env"
  17. "github.com/opencost/opencost/pkg/kubecost"
  18. "github.com/opencost/opencost/pkg/log"
  19. "github.com/opencost/opencost/pkg/util/fileutil"
  20. "github.com/opencost/opencost/pkg/util/json"
  21. "github.com/opencost/opencost/pkg/util/stringutil"
  22. "golang.org/x/exp/slices"
  23. v1 "k8s.io/api/core/v1"
  24. )
  25. const (
  26. ALIBABA_ECS_PRODUCT_CODE = "ecs"
  27. ALIBABA_ECS_VERSION = "2014-05-26"
  28. ALIBABA_ECS_DOMAIN = "ecs.aliyuncs.com"
  29. ALIBABA_DESCRIBE_PRICE_API_ACTION = "DescribePrice"
  30. ALIBABA_DESCRIBE_DISK_API_ACTION = "DescribeDisks"
  31. ALIBABA_INSTANCE_RESOURCE_TYPE = "instance"
  32. ALIBABA_DISK_RESOURCE_TYPE = "disk"
  33. ALIBABA_PAY_AS_YOU_GO_BILLING = "Pay-As-You-Go"
  34. ALIBABA_SUBSCRIPTION_BILLING = "Subscription"
  35. ALIBABA_PREEMPTIBLE_BILLING = "Preemptible"
  36. ALIBABA_OPTIMIZE_KEYWORD = "optimize"
  37. ALIBABA_NON_OPTIMIZE_KEYWORD = "nonoptimize"
  38. ALIBABA_HOUR_PRICE_UNIT = "Hour"
  39. ALIBABA_MONTH_PRICE_UNIT = "Month"
  40. ALIBABA_YEAR_PRICE_UNIT = "Year"
  41. ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE = "unknown"
  42. ALIBABA_NOT_SUPPORTED_INSTANCE_FAMILY_TYPE = "unsupported"
  43. ALIBABA_DISK_CLOUD_ESSD_CATEGORY = "cloud_essd"
  44. ALIBABA_DISK_CLOUD_CATEGORY = "cloud"
  45. ALIBABA_DATA_DISK_CATEGORY = "data"
  46. ALIBABA_SYSTEM_DISK_CATEGORY = "system"
  47. ALIBABA_DATA_DISK_PREFIX = "DataDisk"
  48. ALIBABA_PV_CLOUD_DISK_TYPE = "CloudDisk"
  49. ALIBABA_PV_NAS_TYPE = "NAS"
  50. ALIBABA_PV_OSS_TYPE = "OSS"
  51. ALIBABA_DEFAULT_DATADISK_SIZE = "2000"
  52. ALIBABA_DISK_TOPOLOGY_REGION_LABEL = "topology.diskplugin.csi.alibabacloud.com/region"
  53. ALIBABA_DISK_TOPOLOGY_ZONE_LABEL = "topology.diskplugin.csi.alibabacloud.com/zone"
  54. )
  55. var (
  56. // Regular expression to get the numerical value of PV suffix with GiB from *v1.PersistentVolume.
  57. sizeRegEx = regexp.MustCompile("(.*?)Gi")
  58. )
  59. // Variable to keep track of instance families that fail in DescribePrice API due improper defaulting of systemDisk if the information is not available
  60. var alibabaDefaultToCloudEssd = []string{"g6e", "r6e", "r7", "g7", "g7a", "r7a"}
  61. // Why predefined and dependency on code? Can be converted to API call - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/regions-describeregions
  62. var alibabaRegions = []string{
  63. "cn-qingdao",
  64. "cn-beijing",
  65. "cn-zhangjiakou",
  66. "cn-huhehaote",
  67. "cn-wulanchabu",
  68. "cn-hangzhou",
  69. "cn-shanghai",
  70. "cn-nanjing",
  71. "cn-fuzhou",
  72. "cn-shenzhen",
  73. "cn-guangzhou",
  74. "cn-chengdu",
  75. "cn-hongkong",
  76. "ap-southeast-1",
  77. "ap-southeast-2",
  78. "ap-southeast-3",
  79. "ap-southeast-5",
  80. "ap-southeast-6",
  81. "ap-southeast-7",
  82. "ap-south-1",
  83. "ap-northeast-1",
  84. "ap-northeast-2",
  85. "us-west-1",
  86. "us-east-1",
  87. "eu-central-1",
  88. "me-east-1",
  89. }
  90. // To-Do: Convert to API call - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/describeinstancetypefamilies
  91. // Also first pass only completely tested pricing API for General pupose instances families & memory optimized instance families
  92. var alibabaInstanceFamilies = []string{
  93. "g7",
  94. "g7a",
  95. "g6e",
  96. "g6",
  97. "g5",
  98. "sn2",
  99. "sn2ne",
  100. "r7",
  101. "r7a",
  102. "r6e",
  103. "r6a",
  104. "r6",
  105. "r5",
  106. "se1",
  107. "se1ne",
  108. "re6",
  109. "re6p",
  110. "re4",
  111. "se1",
  112. }
  113. // AlibabaAccessKey holds Alibaba credentials parsing from the service-key.json file.
  114. type AlibabaAccessKey struct {
  115. AccessKeyID string `json:"alibaba_access_key_id"`
  116. SecretAccessKey string `json:"alibaba_secret_access_key"`
  117. }
  118. // Slim Version of k8s disk assigned to a node or PV.
  119. type SlimK8sDisk struct {
  120. DiskType string
  121. RegionID string
  122. PriceUnit string
  123. SizeInGiB string
  124. DiskCategory string
  125. PerformanceLevel string
  126. ProviderID string
  127. StorageClass string
  128. }
  129. func NewSlimK8sDisk(diskType, regionID, priceUnit, diskCategory, performanceLevel, providerID, storageClass, sizeInGiB string) *SlimK8sDisk {
  130. return &SlimK8sDisk{
  131. DiskType: diskType,
  132. RegionID: regionID,
  133. PriceUnit: priceUnit,
  134. SizeInGiB: sizeInGiB,
  135. DiskCategory: diskCategory,
  136. PerformanceLevel: performanceLevel,
  137. ProviderID: providerID,
  138. StorageClass: storageClass,
  139. }
  140. }
  141. // Slim version of a k8s v1.node just to pass along the object of this struct instead of constant getting the labels from within v1.Node & unit testing.
  142. type SlimK8sNode struct {
  143. InstanceType string
  144. RegionID string
  145. PriceUnit string
  146. MemorySizeInKiB string // TO-DO : Possible to convert to float?
  147. IsIoOptimized bool
  148. OSType string
  149. ProviderID string
  150. SystemDisk *SlimK8sDisk
  151. InstanceTypeFamily string // Bug in DescribePrice, doesn't default to enhanced type correctly and you get an error in DescribePrice to get around need the family of the InstanceType.
  152. }
  153. func NewSlimK8sNode(instanceType, regionID, priceUnit, memorySizeInKiB, osType, providerID, instanceTypeFamily string, isIOOptimized bool, systemDiskInfo *SlimK8sDisk) *SlimK8sNode {
  154. return &SlimK8sNode{
  155. InstanceType: instanceType,
  156. RegionID: regionID,
  157. PriceUnit: priceUnit,
  158. MemorySizeInKiB: memorySizeInKiB,
  159. IsIoOptimized: isIOOptimized,
  160. OSType: osType,
  161. SystemDisk: systemDiskInfo,
  162. ProviderID: providerID,
  163. InstanceTypeFamily: instanceTypeFamily,
  164. }
  165. }
  166. // AlibabaNodeAttributes represents metadata about the Node in its pricing information.
  167. // Basic Attributes needed atleast to get the key, Some attributes from k8s Node response
  168. // be populated directly into *Node object.
  169. type AlibabaNodeAttributes struct {
  170. // InstanceType represents the type of instance.
  171. InstanceType string `json:"instanceType"`
  172. // MemorySizeInKiB represents the size of memory of instance.
  173. MemorySizeInKiB string `json:"memorySizeInKiB"`
  174. // IsIoOptimized represents the if instance is I/O optimized.
  175. IsIoOptimized bool `json:"isIoOptimized"`
  176. // OSType represents the OS installed in the Instance.
  177. OSType string `json:"osType"`
  178. // SystemDiskCategory represents the exact category of the system disk attached to the node.
  179. SystemDiskCategory string `json:"systemDiskCategory"`
  180. // SystemDiskSizeInGiB represents the size of the system disk attached to the node.
  181. SystemDiskSizeInGiB string `json:"systemDiskSizeInGiB"`
  182. // SystemDiskPerformanceLevel represents the performance level of the system disk attached to the node.
  183. SystemDiskPerformanceLevel string `json:"systemPerformanceLevel"`
  184. }
  185. func NewAlibabaNodeAttributes(node *SlimK8sNode) *AlibabaNodeAttributes {
  186. if node == nil {
  187. return nil
  188. }
  189. var diskCategory, sizeInGiB, performanceLevel string
  190. if node.SystemDisk != nil {
  191. diskCategory = node.SystemDisk.DiskCategory
  192. sizeInGiB = node.SystemDisk.SizeInGiB
  193. performanceLevel = node.SystemDisk.PerformanceLevel
  194. }
  195. return &AlibabaNodeAttributes{
  196. InstanceType: node.InstanceType,
  197. MemorySizeInKiB: node.MemorySizeInKiB,
  198. IsIoOptimized: node.IsIoOptimized,
  199. OSType: node.OSType,
  200. SystemDiskCategory: diskCategory,
  201. SystemDiskSizeInGiB: sizeInGiB,
  202. SystemDiskPerformanceLevel: performanceLevel,
  203. }
  204. }
  205. // AlibabaPVAttributes represents metadata the PV in its pricing information.
  206. // Basic Attributes needed atleast to get the keys. Some attributes from k8s PV response
  207. // be populated directly into *PV object.
  208. type AlibabaPVAttributes struct {
  209. // PVType can be Cloud Disk, NetWork Attached Storage(NAS) or Object Storage Service (OSS).
  210. // Represents the way the PV was attached
  211. PVType string `json:"pvType"`
  212. // PVSubType represent the sub category of PVType. This is Data in case of Cloud Disk.
  213. PVSubType string `json:"pvSubType"`
  214. // Example for PVCategory with cloudDisk PVType are cloud, cloud_efficiency, cloud_ssd,
  215. // ephemeral_ssd and cloud_essd. If not present returns empty.
  216. PVCategory string `json:"pvCategory"`
  217. // Example for PerformanceLevel with cloudDisk PVType are PL0,PL1,PL2 &PL3. If not present returns empty.
  218. PVPerformanceLevel string `json:"performanceLevel"`
  219. // The Size of the PV in terms of GiB
  220. SizeInGiB string `json:"sizeInGiB"`
  221. }
  222. // TO-Do: next iteration of Alibaba provider support NetWork Attached Storage(NAS) and Object Storage Service (OSS type PVs).
  223. // Currently defaulting to cloudDisk with provision to add work in future.
  224. func NewAlibabaPVAttributes(disk *SlimK8sDisk) *AlibabaPVAttributes {
  225. if disk == nil {
  226. return nil
  227. }
  228. return &AlibabaPVAttributes{
  229. PVType: ALIBABA_PV_CLOUD_DISK_TYPE,
  230. PVSubType: disk.DiskType,
  231. PVCategory: disk.DiskCategory,
  232. PVPerformanceLevel: disk.PerformanceLevel,
  233. SizeInGiB: disk.SizeInGiB,
  234. }
  235. }
  236. // Stage 1 support will be Pay-As-You-Go with HourlyPrice equal to TradePrice with PriceUnit as Hour
  237. // TO-DO: Subscription and Premptible support, Information can be gathered from describing instance for subscription type
  238. // and spotprice can be gather from DescribeSpotPriceHistory API.
  239. // TO-DO: how would you calculate hourly price for subscription type, is it PRICE_YEARLY/HOURS_IN_THE_YEAR|MONTH?
  240. type AlibabaPricingDetails struct {
  241. // Represents hourly price for the given Alibaba cloud Product.
  242. HourlyPrice float32 `json:"hourlyPrice"`
  243. // Represents the unit in which Alibaba Product is billed can be Hour, Month or Year based on the billingMethod.
  244. PriceUnit string `json:"priceUnit"`
  245. // Original Price paid to acquire the Alibaba Product.
  246. TradePrice float32 `json:"tradePrice"`
  247. // Represents the currency unit of the price for billing Alibaba Product.
  248. CurrencyCode string `json:"currencyCode"`
  249. }
  250. func NewAlibabaPricingDetails(hourlyPrice float32, priceUnit string, tradePrice float32, currencyCode string) *AlibabaPricingDetails {
  251. return &AlibabaPricingDetails{
  252. HourlyPrice: hourlyPrice,
  253. PriceUnit: priceUnit,
  254. TradePrice: tradePrice,
  255. CurrencyCode: currencyCode,
  256. }
  257. }
  258. // AlibabaPricingTerms can have three types of supported billing method Pay-As-You-Go, Subscription and Premptible
  259. type AlibabaPricingTerms struct {
  260. BillingMethod string `json:"billingMethod"`
  261. PricingDetails *AlibabaPricingDetails `json:"pricingDetails"`
  262. }
  263. func NewAlibabaPricingTerms(billingMethod string, pricingDetails *AlibabaPricingDetails) *AlibabaPricingTerms {
  264. return &AlibabaPricingTerms{
  265. BillingMethod: billingMethod,
  266. PricingDetails: pricingDetails,
  267. }
  268. }
  269. // Alibaba Pricing struct carry the Attributes and pricing information for Node or PV
  270. type AlibabaPricing struct {
  271. NodeAttributes *AlibabaNodeAttributes
  272. PVAttributes *AlibabaPVAttributes
  273. PricingTerms *AlibabaPricingTerms
  274. Node *Node
  275. PV *PV
  276. }
  277. // Alibaba cloud's Provider struct
  278. type Alibaba struct {
  279. // Data to store Alibaba cloud's pricing struct, key in the map represents exact match to
  280. // node.features() or pv.features for easy lookup
  281. Pricing map[string]*AlibabaPricing
  282. // Lock Needed to provide thread safe
  283. DownloadPricingDataLock sync.RWMutex
  284. Clientset clustercache.ClusterCache
  285. Config *ProviderConfig
  286. *CustomProvider
  287. // The following fields are unexported because of avoiding any leak of secrets of these keys.
  288. // Alibaba Access key used specifically in signer interface used to sign API calls
  289. serviceAccountChecks *ServiceAccountChecks
  290. clusterAccountId string
  291. clusterRegion string
  292. accessKey *credentials.AccessKeyCredential
  293. // Map of regionID to sdk.client to call API for that region
  294. clients map[string]*sdk.Client
  295. }
  296. // GetAlibabaAccessKey return the Access Key used to interact with the Alibaba cloud, if not set it
  297. // set it first by looking at env variables else load it from secret files.
  298. func (alibaba *Alibaba) GetAlibabaAccessKey() (*credentials.AccessKeyCredential, error) {
  299. if alibaba.accessKeyisLoaded() {
  300. return alibaba.accessKey, nil
  301. }
  302. config, err := alibaba.GetConfig()
  303. if err != nil {
  304. return nil, fmt.Errorf("error getting the default config for Alibaba Cloud provider: %w", err)
  305. }
  306. if config.AlibabaServiceKeyName == "" {
  307. config.AlibabaServiceKeyName = env.GetAlibabaAccessKeyID()
  308. }
  309. if config.AlibabaServiceKeySecret == "" {
  310. config.AlibabaServiceKeySecret = env.GetAlibabaAccessKeySecret()
  311. }
  312. if config.AlibabaServiceKeyName == "" && config.AlibabaServiceKeySecret == "" {
  313. log.Debugf("missing service key values for Alibaba cloud integration attempting to use service account integration")
  314. err := alibaba.loadAlibabaAuthSecretAndSetEnv(true)
  315. if err != nil {
  316. return nil, fmt.Errorf("unable to set the Alibaba Cloud key/secret from config file %w", err)
  317. }
  318. config.AlibabaServiceKeyName = env.GetAlibabaAccessKeyID()
  319. config.AlibabaServiceKeySecret = env.GetAlibabaAccessKeySecret()
  320. }
  321. if config.AlibabaServiceKeyName == "" && config.AlibabaServiceKeySecret == "" {
  322. return nil, fmt.Errorf("failed to get the access key for the current alibaba account")
  323. }
  324. alibaba.accessKey = &credentials.AccessKeyCredential{AccessKeyId: env.GetAlibabaAccessKeyID(), AccessKeySecret: env.GetAlibabaAccessKeySecret()}
  325. return alibaba.accessKey, nil
  326. }
  327. // DownloadPricingData satisfies the provider interface and downloads the prices for Node instances and PVs.
  328. func (alibaba *Alibaba) DownloadPricingData() error {
  329. alibaba.DownloadPricingDataLock.Lock()
  330. defer alibaba.DownloadPricingDataLock.Unlock()
  331. var aak *credentials.AccessKeyCredential
  332. var err error
  333. if !alibaba.accessKeyisLoaded() {
  334. aak, err = alibaba.GetAlibabaAccessKey()
  335. if err != nil {
  336. return fmt.Errorf("unable to get the access key information: %w", err)
  337. }
  338. } else {
  339. aak = alibaba.accessKey
  340. }
  341. c, err := alibaba.Config.GetCustomPricingData()
  342. if err != nil {
  343. return fmt.Errorf("error downloading default pricing data: %w", err)
  344. }
  345. // Get all the nodes from Alibaba cluster.
  346. nodeList := alibaba.Clientset.GetAllNodes()
  347. var client *sdk.Client
  348. var signer *signers.AccessKeySigner
  349. var ok bool
  350. var lookupKey string
  351. alibaba.clients = make(map[string]*sdk.Client)
  352. alibaba.Pricing = make(map[string]*AlibabaPricing)
  353. for _, node := range nodeList {
  354. pricingObj := &AlibabaPricing{}
  355. slimK8sNode := generateSlimK8sNodeFromV1Node(node)
  356. if client, ok = alibaba.clients[slimK8sNode.RegionID]; !ok {
  357. client, err = sdk.NewClientWithAccessKey(slimK8sNode.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  358. if err != nil {
  359. return fmt.Errorf("unable to initiate alibaba cloud sdk client for region %s : %w", slimK8sNode.RegionID, err)
  360. }
  361. alibaba.clients[slimK8sNode.RegionID] = client
  362. }
  363. signer = signers.NewAccessKeySigner(aak)
  364. // Adjust the system Disk information of a Node by retrieving the details of associated disk. If unable to retrieve set it to empty
  365. // system disk to pass through and use defaults with Alibaba pricing API.
  366. instanceID := getInstanceIDFromProviderID(slimK8sNode.ProviderID)
  367. slimK8sNode.SystemDisk = getSystemDiskInfoOfANode(instanceID, slimK8sNode.RegionID, client, signer)
  368. lookupKey, err = determineKeyForPricing(slimK8sNode)
  369. if _, ok := alibaba.Pricing[lookupKey]; ok {
  370. log.Debugf("Pricing information for node with same features %s already exists hence skipping", lookupKey)
  371. continue
  372. }
  373. pricingObj, err = processDescribePriceAndCreateAlibabaPricing(client, slimK8sNode, signer, c)
  374. if err != nil {
  375. return fmt.Errorf("failed to create pricing information for node with type %s with error: %w", slimK8sNode.InstanceType, err)
  376. }
  377. alibaba.Pricing[lookupKey] = pricingObj
  378. }
  379. // set the first occurance of region from the node
  380. if alibaba.clusterRegion == "" {
  381. for _, node := range nodeList {
  382. if regionID, ok := node.Labels["topology.kubernetes.io/region"]; ok {
  383. alibaba.clusterRegion = regionID
  384. break
  385. }
  386. }
  387. }
  388. // PV pricing for only Cloud Disk for now.
  389. // TO-DO: Support both NAS(Network Attached storage) and OSS(Object Storage Service) type PVs
  390. pvList := alibaba.Clientset.GetAllPersistentVolumes()
  391. for _, pv := range pvList {
  392. pvRegion := determinePVRegion(pv)
  393. if pvRegion == "" {
  394. pvRegion = alibaba.clusterRegion
  395. }
  396. pricingObj := &AlibabaPricing{}
  397. slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, pvRegion)
  398. lookupKey, err = determineKeyForPricing(slimK8sDisk)
  399. if _, ok := alibaba.Pricing[lookupKey]; ok {
  400. log.Debugf("Pricing information for pv with same features %s already exists hence skipping", lookupKey)
  401. continue
  402. }
  403. if client, ok = alibaba.clients[slimK8sDisk.RegionID]; !ok {
  404. client, err = sdk.NewClientWithAccessKey(slimK8sDisk.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  405. if err != nil {
  406. return fmt.Errorf("unable to initiate alibaba cloud sdk client for region %s : %w", slimK8sDisk.RegionID, err)
  407. }
  408. alibaba.clients[slimK8sDisk.RegionID] = client
  409. }
  410. signer = signers.NewAccessKeySigner(aak)
  411. pricingObj, err = processDescribePriceAndCreateAlibabaPricing(client, slimK8sDisk, signer, c)
  412. if err != nil {
  413. return fmt.Errorf("failed to create pricing information for pv with category %s with error: %w", slimK8sDisk.DiskCategory, err)
  414. }
  415. alibaba.Pricing[lookupKey] = pricingObj
  416. }
  417. return nil
  418. }
  419. // AllNodePricing returns all the pricing data for all nodes and pvs
  420. func (alibaba *Alibaba) AllNodePricing() (interface{}, error) {
  421. alibaba.DownloadPricingDataLock.RLock()
  422. defer alibaba.DownloadPricingDataLock.RUnlock()
  423. return alibaba.Pricing, nil
  424. }
  425. // NodePricing gives pricing information of a specific node given by the key
  426. func (alibaba *Alibaba) NodePricing(key Key) (*Node, error) {
  427. alibaba.DownloadPricingDataLock.RLock()
  428. defer alibaba.DownloadPricingDataLock.RUnlock()
  429. // Get node features for the key
  430. keyFeature := key.Features()
  431. pricing, ok := alibaba.Pricing[keyFeature]
  432. if !ok {
  433. log.Warnf("Node pricing information not found for node with feature: %s", keyFeature)
  434. return &Node{}, nil
  435. }
  436. log.Debugf("returning the node price for the node with feature: %s", keyFeature)
  437. returnNode := pricing.Node
  438. return returnNode, nil
  439. }
  440. // PVPricing gives a pricing information of a specific PV given by PVkey
  441. func (alibaba *Alibaba) PVPricing(pvk PVKey) (*PV, error) {
  442. alibaba.DownloadPricingDataLock.RLock()
  443. defer alibaba.DownloadPricingDataLock.RUnlock()
  444. keyFeature := pvk.Features()
  445. pricing, ok := alibaba.Pricing[keyFeature]
  446. if !ok {
  447. log.Warnf("Persistent Volume pricing not found for PV with feature: %s", keyFeature)
  448. return &PV{}, nil
  449. }
  450. log.Debugf("returning the PV price for the node with feature: %s", keyFeature)
  451. return pricing.PV, nil
  452. }
  453. // Stubbed NetworkPricing for Alibaba Cloud. Will look at this in Next PR
  454. func (alibaba *Alibaba) NetworkPricing() (*Network, error) {
  455. return &Network{
  456. ZoneNetworkEgressCost: 0.0,
  457. RegionNetworkEgressCost: 0.0,
  458. InternetNetworkEgressCost: 0.0,
  459. }, nil
  460. }
  461. // Stubbed LoadBalancerPricing for Alibaba Cloud. Will look at this in Next PR
  462. func (alibaba *Alibaba) LoadBalancerPricing() (*LoadBalancer, error) {
  463. return &LoadBalancer{
  464. Cost: 0.0,
  465. }, nil
  466. }
  467. func (alibaba *Alibaba) GetConfig() (*CustomPricing, error) {
  468. c, err := alibaba.Config.GetCustomPricingData()
  469. if err != nil {
  470. return nil, err
  471. }
  472. if c.Discount == "" {
  473. c.Discount = "0%"
  474. }
  475. if c.NegotiatedDiscount == "" {
  476. c.NegotiatedDiscount = "0%"
  477. }
  478. if c.ShareTenancyCosts == "" {
  479. c.ShareTenancyCosts = defaultShareTenancyCost
  480. }
  481. return c, nil
  482. }
  483. // Load once and cache the result (even on failure). This is an install time secret, so
  484. // we don't expect the secret to change. If it does, however, we can force reload using
  485. // the input parameter.
  486. func (alibaba *Alibaba) loadAlibabaAuthSecretAndSetEnv(force bool) error {
  487. if !force && alibaba.accessKeyisLoaded() {
  488. return nil
  489. }
  490. exists, err := fileutil.FileExists(authSecretPath)
  491. if !exists || err != nil {
  492. return fmt.Errorf("failed to locate service account file: %s with err: %w", authSecretPath, err)
  493. }
  494. result, err := ioutil.ReadFile(authSecretPath)
  495. if err != nil {
  496. return fmt.Errorf("failed to read service account file: %s with err: %w", authSecretPath, err)
  497. }
  498. var ak *AlibabaAccessKey
  499. err = json.Unmarshal(result, &ak)
  500. if err != nil {
  501. return fmt.Errorf("failed to unmarshall access key id and access key secret with err: %w", err)
  502. }
  503. err = env.Set(env.AlibabaAccessKeyIDEnvVar, ak.AccessKeyID)
  504. if err != nil {
  505. return fmt.Errorf("failed to set environment variable: %s with err: %w", env.AlibabaAccessKeyIDEnvVar, err)
  506. }
  507. err = env.Set(env.AlibabaAccessKeySecretEnvVar, ak.SecretAccessKey)
  508. if err != nil {
  509. return fmt.Errorf("failed to set environment variable: %s with err: %w", env.AlibabaAccessKeySecretEnvVar, err)
  510. }
  511. alibaba.accessKey = &credentials.AccessKeyCredential{
  512. AccessKeyId: ak.AccessKeyID,
  513. AccessKeySecret: ak.SecretAccessKey,
  514. }
  515. return nil
  516. }
  517. // Regions returns a current supported list of Alibaba regions
  518. func (alibaba *Alibaba) Regions() []string {
  519. return alibabaRegions
  520. }
  521. // ClusterInfo returns information about Alibaba Cloud cluster, as provided by metadata.
  522. func (alibaba *Alibaba) ClusterInfo() (map[string]string, error) {
  523. c, err := alibaba.GetConfig()
  524. if err != nil {
  525. return nil, fmt.Errorf("failed to getConfig with err: %w", err)
  526. }
  527. var clusterName string
  528. if c.ClusterName != "" {
  529. clusterName = c.ClusterName
  530. }
  531. // Set it to environment clusterID if not set at this point
  532. if clusterName == "" {
  533. clusterName = env.GetClusterID()
  534. }
  535. m := make(map[string]string)
  536. m["name"] = clusterName
  537. m["provider"] = kubecost.AlibabaProvider
  538. m["project"] = alibaba.clusterAccountId
  539. m["region"] = alibaba.clusterRegion
  540. m["id"] = env.GetClusterID()
  541. return m, nil
  542. }
  543. // Will look at this in Next PR if needed
  544. func (alibaba *Alibaba) GetAddresses() ([]byte, error) {
  545. return nil, nil
  546. }
  547. // Will look at this in Next PR if needed
  548. func (alibaba *Alibaba) GetDisks() ([]byte, error) {
  549. return nil, nil
  550. }
  551. func (alibaba *Alibaba) GetOrphanedResources() ([]OrphanedResource, error) {
  552. return nil, errors.New("not implemented")
  553. }
  554. func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
  555. return alibaba.Config.Update(func(c *CustomPricing) error {
  556. if updateType != "" {
  557. return fmt.Errorf("UpdateConfig for Alibaba Provider doesn't support updateType %s at this time", updateType)
  558. } else {
  559. a := make(map[string]interface{})
  560. err := json.NewDecoder(r).Decode(&a)
  561. if err != nil {
  562. return err
  563. }
  564. for k, v := range a {
  565. kUpper := strings.Title(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
  566. vstr, ok := v.(string)
  567. if ok {
  568. err := SetCustomPricingField(c, kUpper, vstr)
  569. if err != nil {
  570. return err
  571. }
  572. } else {
  573. return fmt.Errorf("type error while updating config for %s", kUpper)
  574. }
  575. }
  576. }
  577. if env.IsRemoteEnabled() {
  578. err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
  579. if err != nil {
  580. return err
  581. }
  582. }
  583. return nil
  584. })
  585. }
  586. func (alibaba *Alibaba) UpdateConfigFromConfigMap(cm map[string]string) (*CustomPricing, error) {
  587. return alibaba.Config.UpdateFromMap(cm)
  588. }
  589. // Will look at this in Next PR if needed
  590. func (alibaba *Alibaba) GetManagementPlatform() (string, error) {
  591. return "", nil
  592. }
  593. // Will look at this in Next PR if needed
  594. func (alibaba *Alibaba) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string {
  595. return ""
  596. }
  597. // Will look at this in Next PR if needed
  598. func (alibaba *Alibaba) ApplyReservedInstancePricing(nodes map[string]*Node) {
  599. }
  600. // Will look at this in Next PR if needed
  601. func (alibaba *Alibaba) ServiceAccountStatus() *ServiceAccountStatus {
  602. return &ServiceAccountStatus{}
  603. }
  604. // Will look at this in Next PR if needed
  605. func (alibaba *Alibaba) PricingSourceStatus() map[string]*PricingSource {
  606. return map[string]*PricingSource{}
  607. }
  608. // Will look at this in Next PR if needed
  609. func (alibaba *Alibaba) ClusterManagementPricing() (string, float64, error) {
  610. return "", 0.0, nil
  611. }
  612. // Will look at this in Next PR if needed
  613. func (alibaba *Alibaba) CombinedDiscountForNode(string, bool, float64, float64) float64 {
  614. return 0.0
  615. }
  616. func (alibaba *Alibaba) accessKeyisLoaded() bool {
  617. return alibaba.accessKey != nil
  618. }
  619. type AlibabaNodeKey struct {
  620. ProviderID string
  621. RegionID string
  622. InstanceType string
  623. OSType string
  624. OptimizedKeyword string //If IsIoOptimized is true use the word optimize in the Node key and if its not optimized use the word nonoptimize
  625. SystemDiskCategory string
  626. SystemDiskSizeInGiB string
  627. SystemDiskPerformanceLevel string
  628. }
  629. func NewAlibabaNodeKey(node *SlimK8sNode, optimizedKeyword, systemDiskCategory, systemDiskSizeInGiB, systemDiskPerfromanceLevel string) *AlibabaNodeKey {
  630. var providerID, regionID, instanceType, osType string
  631. if node != nil {
  632. providerID = node.ProviderID
  633. regionID = node.RegionID
  634. instanceType = node.InstanceType
  635. osType = node.OSType
  636. }
  637. return &AlibabaNodeKey{
  638. ProviderID: providerID,
  639. RegionID: regionID,
  640. InstanceType: instanceType,
  641. OSType: osType,
  642. OptimizedKeyword: optimizedKeyword,
  643. SystemDiskCategory: systemDiskCategory,
  644. SystemDiskSizeInGiB: systemDiskSizeInGiB,
  645. SystemDiskPerformanceLevel: systemDiskPerfromanceLevel,
  646. }
  647. }
  648. func (alibabaNodeKey *AlibabaNodeKey) ID() string {
  649. return alibabaNodeKey.ProviderID
  650. }
  651. func (alibabaNodeKey *AlibabaNodeKey) Features() string {
  652. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaNodeKey.RegionID, alibabaNodeKey.InstanceType, alibabaNodeKey.OSType,
  653. alibabaNodeKey.OptimizedKeyword, alibabaNodeKey.SystemDiskCategory, alibabaNodeKey.SystemDiskSizeInGiB, alibabaNodeKey.SystemDiskPerformanceLevel})
  654. return strings.Join(keyLookup, "::")
  655. }
  656. func (alibabaNodeKey *AlibabaNodeKey) GPUType() string {
  657. return ""
  658. }
  659. func (alibabaNodeKey *AlibabaNodeKey) GPUCount() int {
  660. return 0
  661. }
  662. // Get's the key for the k8s node input
  663. func (alibaba *Alibaba) GetKey(mapValue map[string]string, node *v1.Node) Key {
  664. slimK8sNode := generateSlimK8sNodeFromV1Node(node)
  665. var aak *credentials.AccessKeyCredential
  666. var err error
  667. var skipSystemDiskRetrieval, ok bool
  668. var client *sdk.Client
  669. var signer *signers.AccessKeySigner
  670. if !alibaba.accessKeyisLoaded() {
  671. aak, err = alibaba.GetAlibabaAccessKey()
  672. if err != nil {
  673. log.Warnf("unable to set the signer for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: %v", slimK8sNode.ProviderID, err)
  674. skipSystemDiskRetrieval = true
  675. }
  676. } else {
  677. aak = alibaba.accessKey
  678. }
  679. signer = signers.NewAccessKeySigner(aak)
  680. if client, ok = alibaba.clients[slimK8sNode.RegionID]; !ok {
  681. client, err = sdk.NewClientWithAccessKey(slimK8sNode.RegionID, aak.AccessKeyId, aak.AccessKeySecret)
  682. if err != nil {
  683. log.Warnf("unable to set the client for node with providerID %s to retrieve the key skipping SystemDisk Retrieval with err: %v", slimK8sNode.ProviderID, err)
  684. skipSystemDiskRetrieval = true
  685. }
  686. alibaba.clients[slimK8sNode.RegionID] = client
  687. }
  688. optimizedKeyword := ""
  689. if slimK8sNode.IsIoOptimized {
  690. optimizedKeyword = ALIBABA_OPTIMIZE_KEYWORD
  691. } else {
  692. optimizedKeyword = ALIBABA_NON_OPTIMIZE_KEYWORD
  693. }
  694. var diskCategory, diskSizeInGiB, diskPerformanceLevel string
  695. if skipSystemDiskRetrieval {
  696. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  697. }
  698. instanceID := getInstanceIDFromProviderID(slimK8sNode.ProviderID)
  699. slimK8sNode.SystemDisk = getSystemDiskInfoOfANode(instanceID, slimK8sNode.RegionID, client, signer)
  700. if slimK8sNode.SystemDisk != nil {
  701. diskCategory = slimK8sNode.SystemDisk.DiskCategory
  702. diskSizeInGiB = slimK8sNode.SystemDisk.SizeInGiB
  703. diskPerformanceLevel = slimK8sNode.SystemDisk.PerformanceLevel
  704. }
  705. return NewAlibabaNodeKey(slimK8sNode, optimizedKeyword, diskCategory, diskSizeInGiB, diskPerformanceLevel)
  706. }
  707. type AlibabaPVKey struct {
  708. ProviderID string
  709. RegionID string
  710. PVType string
  711. PVSubType string
  712. PVCategory string
  713. PVPerformaceLevel string
  714. StorageClassName string
  715. SizeInGiB string
  716. }
  717. func (alibaba *Alibaba) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
  718. regionID := defaultRegion
  719. // If default Region is not passed default it to cluster region ID.
  720. if defaultRegion == "" {
  721. regionID = alibaba.clusterRegion
  722. }
  723. slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, defaultRegion)
  724. return &AlibabaPVKey{
  725. ProviderID: slimK8sDisk.ProviderID,
  726. RegionID: regionID,
  727. PVType: ALIBABA_PV_CLOUD_DISK_TYPE,
  728. PVSubType: slimK8sDisk.DiskType,
  729. PVCategory: slimK8sDisk.DiskCategory,
  730. PVPerformaceLevel: slimK8sDisk.PerformanceLevel,
  731. StorageClassName: pv.Spec.StorageClassName,
  732. SizeInGiB: slimK8sDisk.SizeInGiB,
  733. }
  734. }
  735. func (alibabaPVKey *AlibabaPVKey) Features() string {
  736. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{alibabaPVKey.RegionID, alibabaPVKey.PVSubType, alibabaPVKey.PVCategory, alibabaPVKey.PVPerformaceLevel, alibabaPVKey.SizeInGiB})
  737. return strings.Join(keyLookup, "::")
  738. }
  739. func (alibabaPVKey *AlibabaPVKey) ID() string {
  740. return alibabaPVKey.ProviderID
  741. }
  742. // Get storage class information for PV.
  743. func (alibabaPVKey *AlibabaPVKey) GetStorageClass() string {
  744. return alibabaPVKey.StorageClassName
  745. }
  746. // Helper functions for alibabaprovider.go
  747. // createDescribePriceACSRequest creates the HTTP GET request for the required resources' Price information,
  748. // When supporting subscription and Premptible resources this HTTP call needs to be modified with PriceUnit information
  749. // When supporting different new type of instances like Compute Optimized, Memory Optimized etc make sure you add the instance type
  750. // in unit test and check if it works or not to create the ack request and processDescribePriceAndCreateAlibabaPricing function
  751. // else more paramters need to be pulled from kubernetes node response or gather infromation from elsewhere and function modified.
  752. func createDescribePriceACSRequest(i interface{}) (*requests.CommonRequest, error) {
  753. request := requests.NewCommonRequest()
  754. request.Method = requests.GET
  755. request.Product = ALIBABA_ECS_PRODUCT_CODE
  756. request.Domain = ALIBABA_ECS_DOMAIN
  757. request.Version = ALIBABA_ECS_VERSION
  758. request.Scheme = requests.HTTPS
  759. request.ApiName = ALIBABA_DESCRIBE_PRICE_API_ACTION
  760. switch i.(type) {
  761. case *SlimK8sNode:
  762. node := i.(*SlimK8sNode)
  763. request.QueryParams["RegionId"] = node.RegionID
  764. request.QueryParams["ResourceType"] = ALIBABA_INSTANCE_RESOURCE_TYPE
  765. request.QueryParams["InstanceType"] = node.InstanceType
  766. request.QueryParams["PriceUnit"] = node.PriceUnit
  767. if node.SystemDisk != nil {
  768. // Only if the required information is present it should be overridden else default it via the API
  769. if node.SystemDisk.DiskCategory != "" {
  770. request.QueryParams["SystemDisk.Category"] = node.SystemDisk.DiskCategory
  771. }
  772. if node.SystemDisk.SizeInGiB != "" {
  773. request.QueryParams["SystemDisk.Size"] = node.SystemDisk.SizeInGiB
  774. }
  775. if node.SystemDisk.PerformanceLevel != "" {
  776. request.QueryParams["SystemDisk.PerformanceLevel"] = node.SystemDisk.PerformanceLevel
  777. }
  778. } else {
  779. // When System Disk information is not available for instance family g6e, r7 and r6e the defaults in
  780. // DescribePrice dont default rightly to cloud_essd for these instances.
  781. if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) {
  782. request.QueryParams["SystemDisk.Category"] = ALIBABA_DISK_CLOUD_ESSD_CATEGORY
  783. }
  784. }
  785. request.TransToAcsRequest()
  786. return request, nil
  787. case *SlimK8sDisk:
  788. disk := i.(*SlimK8sDisk)
  789. request.QueryParams["RegionId"] = disk.RegionID
  790. request.QueryParams["PriceUnit"] = disk.PriceUnit
  791. request.QueryParams["ResourceType"] = ALIBABA_DISK_RESOURCE_TYPE
  792. request.QueryParams[fmt.Sprintf("%s.%d.Size", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.SizeInGiB
  793. request.QueryParams[fmt.Sprintf("%s.%d.Category", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.DiskCategory
  794. // Performance level defaults to PL1 if not present in volume attribute.
  795. if disk.PerformanceLevel != "" {
  796. request.QueryParams[fmt.Sprintf("%s.%d.PerformanceLevel", ALIBABA_DATA_DISK_PREFIX, 1)] = disk.PerformanceLevel
  797. }
  798. request.TransToAcsRequest()
  799. return request, nil
  800. default:
  801. return nil, fmt.Errorf("unsupported ECS type (%T) for DescribePrice at this time", i)
  802. }
  803. }
  804. // createDescribeDisksCSRequest creates the HTTP GET Request to map the system disk to the InstanceID
  805. func createDescribeDisksACSRequest(instanceID, regionID, diskType string) (*requests.CommonRequest, error) {
  806. request := requests.NewCommonRequest()
  807. request.Method = requests.GET
  808. request.Product = ALIBABA_ECS_PRODUCT_CODE
  809. request.Domain = ALIBABA_ECS_DOMAIN
  810. request.Version = ALIBABA_ECS_VERSION
  811. request.Scheme = requests.HTTPS
  812. request.ApiName = ALIBABA_DESCRIBE_DISK_API_ACTION
  813. request.QueryParams["RegionId"] = regionID
  814. request.QueryParams["InstanceId"] = instanceID
  815. request.QueryParams["DiskType"] = diskType
  816. request.TransToAcsRequest()
  817. return request, nil
  818. }
  819. // determineKeyForPricing generate a unique key from SlimK8sNode object that is constructed from v1.Node object and
  820. // SlimK8sDisk that is constructed from v1.PersistentVolume.
  821. func determineKeyForPricing(i interface{}) (string, error) {
  822. if i == nil {
  823. return "", fmt.Errorf("nil component passed to determine key")
  824. }
  825. switch i.(type) {
  826. case *SlimK8sNode:
  827. node := i.(*SlimK8sNode)
  828. var diskCategory, diskSizeInGiB, diskPerformanceLevel string
  829. if node.SystemDisk != nil {
  830. diskCategory = node.SystemDisk.DiskCategory
  831. diskSizeInGiB = node.SystemDisk.SizeInGiB
  832. diskPerformanceLevel = node.SystemDisk.PerformanceLevel
  833. }
  834. if node.IsIoOptimized {
  835. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{node.RegionID, node.InstanceType, node.OSType, ALIBABA_OPTIMIZE_KEYWORD, diskCategory, diskSizeInGiB, diskPerformanceLevel})
  836. return strings.Join(keyLookup, "::"), nil
  837. } else {
  838. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{node.RegionID, node.InstanceType, node.OSType, ALIBABA_NON_OPTIMIZE_KEYWORD, diskCategory, diskSizeInGiB, diskPerformanceLevel})
  839. return strings.Join(keyLookup, "::"), nil
  840. }
  841. case *SlimK8sDisk:
  842. disk := i.(*SlimK8sDisk)
  843. keyLookup := stringutil.DeleteEmptyStringsFromArray([]string{disk.RegionID, disk.DiskType, disk.DiskCategory, disk.PerformanceLevel, disk.SizeInGiB})
  844. return strings.Join(keyLookup, "::"), nil
  845. default:
  846. return "", fmt.Errorf("unsupported ECS type (%T) at this time", i)
  847. }
  848. }
  849. // Below structs are used to unmarshal json response of Alibaba cloud's API DescribePrice
  850. type Price struct {
  851. OriginalPrice float32 `json:"OriginalPrice"`
  852. ReservedInstanceHourPrice float32 `json:"ReservedInstanceHourPrice"`
  853. DiscountPrice float32 `json:"DiscountPrice"`
  854. Currency string `json:"Currency"`
  855. TradePrice float32 `json:"TradePrice"`
  856. }
  857. type PriceInfo struct {
  858. Price Price `json:"Price"`
  859. }
  860. type DescribePriceResponse struct {
  861. RequestId string `json:"RequestId"`
  862. PriceInfo PriceInfo `json:"PriceInfo"`
  863. }
  864. // processDescribePriceAndCreateAlibabaPricing processes the DescribePrice API and generates the pricing information for alibaba node resource and alibaba pv resource that's backed by cloud disk.
  865. func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface{}, signer *signers.AccessKeySigner, custom *CustomPricing) (pricing *AlibabaPricing, err error) {
  866. pricing = &AlibabaPricing{}
  867. var response DescribePriceResponse
  868. if i == nil {
  869. return nil, fmt.Errorf("nil component passed to process the pricing information")
  870. }
  871. switch i.(type) {
  872. case *SlimK8sNode:
  873. node := i.(*SlimK8sNode)
  874. req, err := createDescribePriceACSRequest(node)
  875. if err != nil {
  876. return nil, err
  877. }
  878. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  879. pricing.NodeAttributes = NewAlibabaNodeAttributes(node)
  880. if err != nil || resp.GetHttpStatus() != 200 {
  881. // Can be defaulted to some value here?
  882. return nil, fmt.Errorf("unable to fetch information for node with InstanceType: %v", node.InstanceType)
  883. } else {
  884. // This is where population of Pricing happens
  885. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  886. if err != nil {
  887. return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
  888. }
  889. // TO-DO : Ask in PR How to get the defaults is it equal to AWS/GCP defaults? And what needs to be returned
  890. pricing.Node = &Node{
  891. Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
  892. BaseCPUPrice: custom.CPU,
  893. BaseRAMPrice: custom.RAM,
  894. BaseGPUPrice: custom.GPU,
  895. }
  896. // TO-DO : Currently with Pay-As-You-go Offering TradePrice = HourlyPrice , When support happens to other type HourlyPrice Need to be determined.
  897. pricing.PricingTerms = NewAlibabaPricingTerms(ALIBABA_PAY_AS_YOU_GO_BILLING, NewAlibabaPricingDetails(response.PriceInfo.Price.TradePrice, ALIBABA_HOUR_PRICE_UNIT, response.PriceInfo.Price.TradePrice, response.PriceInfo.Price.Currency))
  898. }
  899. case *SlimK8sDisk:
  900. disk := i.(*SlimK8sDisk)
  901. req, err := createDescribePriceACSRequest(disk)
  902. if err != nil {
  903. return nil, err
  904. }
  905. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  906. if err != nil || resp.GetHttpStatus() != 200 {
  907. return nil, fmt.Errorf("unable to fetch information for disk with DiskType: %v with err: %w", disk.DiskCategory, err)
  908. } else {
  909. // This is where population of Pricing happens
  910. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  911. if err != nil {
  912. return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
  913. }
  914. pricing.PVAttributes = NewAlibabaPVAttributes(disk)
  915. pricing.PV = &PV{
  916. Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
  917. }
  918. // TO-DO : Disk has support for Hour and Month but pricing API is failing for month for disk(Research why?) and same challenge as node pricing no prepaid/postpaid distinction in v1.PersistentVolume object have to look at APIs for th information.
  919. pricing.PricingTerms = NewAlibabaPricingTerms(ALIBABA_PAY_AS_YOU_GO_BILLING, NewAlibabaPricingDetails(response.PriceInfo.Price.TradePrice, ALIBABA_HOUR_PRICE_UNIT, response.PriceInfo.Price.TradePrice, response.PriceInfo.Price.Currency))
  920. }
  921. default:
  922. return nil, fmt.Errorf("unsupported ECS Pricing component of type (%T) at this time", i)
  923. }
  924. return pricing, nil
  925. }
  926. // This function is to get the InstanceFamily from the InstanceType , convention followed in
  927. // instance type is ecs.[FamilyName].[DifferentSize], it gets the familyName , if it is unable to get it
  928. // it lists the instance family name as Unknown.
  929. func getInstanceFamilyFromType(instanceType string) string {
  930. splitinstanceType := strings.Split(instanceType, ".")
  931. if len(splitinstanceType) != 3 {
  932. log.Warnf("unable to find the family of the instance type %s, returning its family type unknown", instanceType)
  933. return ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE
  934. }
  935. if !slices.Contains(alibabaInstanceFamilies, splitinstanceType[1]) {
  936. log.Warnf("currently the instance family type %s is not valid or not tested completely for pricing API", instanceType)
  937. return ALIBABA_NOT_SUPPORTED_INSTANCE_FAMILY_TYPE
  938. }
  939. return splitinstanceType[1]
  940. }
  941. // getInstanceIDFromProviderID returns the instance ID associated with the Node. A *v1.Node providerID in Alibaba cloud
  942. // is of <REGION-ID>.<INSTANCE-ID>. This function returns the Instance ID for the given ProviderID. if its unable to interpret
  943. // it defaults to empty string.
  944. func getInstanceIDFromProviderID(providerID string) string {
  945. if providerID == "" {
  946. return ""
  947. }
  948. splitStrings := strings.Split(providerID, ".")
  949. if len(splitStrings) < 2 {
  950. return ""
  951. }
  952. return splitStrings[1]
  953. }
  954. type Disk struct {
  955. Category string `json:"Category"`
  956. Size int `json:"Size"`
  957. PerformanceLevel string `json:"PerformanceLevel"`
  958. Type string `json:"Type"`
  959. RegionId string `json:"RegionId"`
  960. DiskId string `json:"DiskId"`
  961. DiskChargeType string `json:"DiskChargeType"`
  962. }
  963. type Disks struct {
  964. Disk []*Disk `json:"Disk"`
  965. }
  966. type DescribeDiskResponse struct {
  967. TotalCount int `json:"TotalCount"`
  968. Disks *Disks `json:"Disks"`
  969. }
  970. // getSystemDiskInfoOfANode gets the relevant System disk information associated with the Node given by the instanceID
  971. // in form of a SlimK8sDisk with only relevant information that can adjust the node pricing. If any error occurs return
  972. // an empty disk to not impact any default set at the price retrieval of the node.
  973. func getSystemDiskInfoOfANode(instanceID, regionID string, client *sdk.Client, signer *signers.AccessKeySigner) (systemDisk *SlimK8sDisk) {
  974. systemDisk = &SlimK8sDisk{}
  975. var response DescribeDiskResponse
  976. // if instanceID is empty string return an empty k8s
  977. if instanceID == "" {
  978. return
  979. }
  980. req, err := createDescribeDisksACSRequest(instanceID, regionID, ALIBABA_SYSTEM_DISK_CATEGORY)
  981. // if any error occurs return an empty disk to not impact default pricing.
  982. if err != nil {
  983. log.Warnf("Unable to create Describe Disk Request with err: %v for node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, instanceID)
  984. return
  985. }
  986. resp, err := client.ProcessCommonRequestWithSigner(req, signer)
  987. if err != nil || resp.GetHttpStatus() != 200 {
  988. log.Warnf("Unable to process Describe Disk request with err: %v and errcode: %d for the node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, resp.GetHttpStatus(), instanceID)
  989. return
  990. } else {
  991. // This is where population of Pricing happens
  992. err = json.Unmarshal(resp.GetHttpContentBytes(), &response)
  993. if err != nil {
  994. log.Warnf("Unable to unmarshall Describe Disk response with err: %v for the node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", err, instanceID)
  995. return
  996. }
  997. // Every instance should only have one system disk per Alibaba Cloud documentation https://www.alibabacloud.com/help/en/elastic-compute-service/latest/block-storage-overview-disks,
  998. // if TotalCount is not 1 just return empty and let it not impact default pricing.
  999. if response.TotalCount != 1 {
  1000. log.Warnf("Total count of system disk for node with InstanceID: %s is not 1, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  1001. return
  1002. }
  1003. if response.Disks == nil {
  1004. log.Warnf("Disks information missing for node with InstanceID: %s, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  1005. return
  1006. }
  1007. if len(response.Disks.Disk) < 1 {
  1008. log.Warnf("Total number of system disk for node with InstanceID: %s is less than 1, hence defaulting it to an empty system disk to pass through to defaults", instanceID)
  1009. return
  1010. }
  1011. // TO-DO: When supporting Subscription type disk, you can leverge the disk.DiskChargeType here to map it to subscription type.
  1012. systemDisk := response.Disks.Disk[0]
  1013. return NewSlimK8sDisk(systemDisk.Type, systemDisk.RegionId, ALIBABA_HOUR_PRICE_UNIT, systemDisk.Category, systemDisk.PerformanceLevel, systemDisk.DiskId, "", fmt.Sprintf("%d", systemDisk.Size))
  1014. }
  1015. }
  1016. // generateSlimK8sNodeFromV1Node generates SlimK8sNode struct from v1.Node to fetch pricing information and call alibaba API.
  1017. func generateSlimK8sNodeFromV1Node(node *v1.Node) *SlimK8sNode {
  1018. var regionID, osType, instanceType, providerID, priceUnit, instanceFamily string
  1019. var memorySizeInKiB string // TO-DO: try to convert it into float
  1020. var ok, IsIoOptimized bool
  1021. if regionID, ok = node.Labels["topology.kubernetes.io/region"]; !ok {
  1022. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1023. log.Debugf("No RegionID label for the node: %s", node.Name)
  1024. }
  1025. if osType, ok = node.Labels["beta.kubernetes.io/os"]; !ok {
  1026. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1027. log.Debugf("OS type undetected for the node: %s", node.Name)
  1028. }
  1029. if instanceType, ok = node.Labels["node.kubernetes.io/instance-type"]; !ok {
  1030. // HIGHLY UNLIKELY THAT THIS LABEL WONT BE THERE.
  1031. log.Debugf("Instance Type undetected for the node: %s", node.Name)
  1032. }
  1033. instanceFamily = getInstanceFamilyFromType(instanceType)
  1034. memorySizeInKiB = fmt.Sprintf("%s", node.Status.Capacity.Memory())
  1035. providerID = node.Spec.ProviderID // Alibaba Cloud provider doesnt follow convention of prefix with cloud provider name
  1036. // Looking at current Instance offering , all of the Instances seem to be I/O optimized - https://www.alibabacloud.com/help/en/elastic-compute-service/latest/instance-family
  1037. // Basic price Json has it as part of the key so defaulting to true.
  1038. IsIoOptimized = true
  1039. priceUnit = ALIBABA_HOUR_PRICE_UNIT
  1040. systemDisk := &SlimK8sDisk{}
  1041. return NewSlimK8sNode(instanceType, regionID, priceUnit, memorySizeInKiB, osType, providerID, instanceFamily, IsIoOptimized, systemDisk)
  1042. }
  1043. // getNumericalValueFromResourceQuantity returns the numericalValue of the resourceQuantity
  1044. // An example is: 20Gi returns to 20. If any error occurs it returns the default value used in describePrice API which is 2000.
  1045. func getNumericalValueFromResourceQuantity(quantity string) (value string) {
  1046. // defaulting when any panic or empty string occurs.
  1047. defer func() {
  1048. log.Debugf("unable to determine the size of the PV so defaulting the size to %s", ALIBABA_DEFAULT_DATADISK_SIZE)
  1049. if err := recover(); err != nil {
  1050. value = ALIBABA_DEFAULT_DATADISK_SIZE
  1051. }
  1052. if value == "" {
  1053. value = ALIBABA_DEFAULT_DATADISK_SIZE
  1054. }
  1055. }()
  1056. res := sizeRegEx.FindAllStringSubmatch(quantity, 1)
  1057. value = res[0][1]
  1058. return
  1059. }
  1060. // generateSlimK8sDiskFromV1PV function generates SlimK8sDisk from v1.PersistentVolume
  1061. // to generate slim disk type that can be used to fetch pricing information for Data disk type.
  1062. func generateSlimK8sDiskFromV1PV(pv *v1.PersistentVolume, regionID string) *SlimK8sDisk {
  1063. // All PVs are data disks while local disk are categorized as system disk
  1064. diskType := ALIBABA_DATA_DISK_CATEGORY
  1065. //TO-DO: Disk supports month and hour prices , defaulting to hour
  1066. priceUnit := ALIBABA_HOUR_PRICE_UNIT
  1067. sizeQuantity := fmt.Sprintf("%s", pv.Spec.Capacity.Storage())
  1068. // res := sizeRegEx.FindAllStringSubmatch(sizeQuantity, 1)
  1069. sizeInGiB := getNumericalValueFromResourceQuantity(sizeQuantity)
  1070. providerID := ""
  1071. if pv.Spec.CSI != nil {
  1072. providerID = pv.Spec.CSI.VolumeHandle
  1073. } else {
  1074. providerID = pv.Name // Looks like pv name is same as providerID in Alibaba k8s cluster
  1075. }
  1076. // Performance level being empty string gets defaulted in describePrice to PL1.
  1077. performanceLevel := ""
  1078. diskCategory := ""
  1079. if pv.Spec.CSI != nil {
  1080. if val, ok := pv.Spec.CSI.VolumeAttributes["performanceLevel"]; ok {
  1081. performanceLevel = val
  1082. }
  1083. if val, ok := pv.Spec.CSI.VolumeAttributes["type"]; ok {
  1084. diskCategory = val
  1085. }
  1086. }
  1087. // Highly unlikely that label pv.Spec.CSI.VolumeAttributes["type"] doesn't exist but if occured default to cloud (most basic disk type)
  1088. if diskCategory == "" {
  1089. diskCategory = ALIBABA_DISK_CLOUD_CATEGORY
  1090. }
  1091. return NewSlimK8sDisk(diskType, regionID, priceUnit, diskCategory, performanceLevel, providerID, pv.Spec.StorageClassName, sizeInGiB)
  1092. }
  1093. // determinePVRegion determines associated region for a particular PV based on the following priority, which can be changed and any other path to determine region can be added!
  1094. // if topology.diskplugin.csi.alibabacloud.com/region label/annotation is passed during PV creation return that as the PV region.
  1095. // if topology.diskplugin.csi.alibabacloud.com/zone label/annotation is passed during PV creation determine the region based on this pv label.
  1096. // if neither of the above label/annotation is present check node affinity for the zone affinity and determine the region based on this zone.
  1097. // if nether of the above yields a region , return empty string to default it to cluster region.
  1098. func determinePVRegion(pv *v1.PersistentVolume) string {
  1099. // if "topology.diskplugin.csi.alibabacloud.com/region" is present as a label or annotation return that as the PV region
  1100. if val, ok := pv.Labels[ALIBABA_DISK_TOPOLOGY_REGION_LABEL]; ok {
  1101. log.Debugf("determinePVRegion returned a region value of: %s through label: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_REGION_LABEL, pv.Name)
  1102. return val
  1103. }
  1104. if val, ok := pv.Annotations[ALIBABA_DISK_TOPOLOGY_REGION_LABEL]; ok {
  1105. log.Debugf("determinePVRegion returned a region value of: %s through annotation: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_REGION_LABEL, pv.Name)
  1106. return val
  1107. }
  1108. // if "topology.diskplugin.csi.alibabacloud.com/zone" is present as a label or annotation set it as the PV zone before looking at node affinity to determine the region PV belongs too
  1109. var pvZone string
  1110. if val, ok := pv.Labels[ALIBABA_DISK_TOPOLOGY_ZONE_LABEL]; ok {
  1111. log.Debugf("determinePVRegion will set zone value to: %s through label: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1112. pvZone = val
  1113. }
  1114. if pvZone == "" {
  1115. if val, ok := pv.Annotations[ALIBABA_DISK_TOPOLOGY_ZONE_LABEL]; ok {
  1116. log.Debugf("determinePVRegion will set zone value to: %s through annotation: %s for PV name: %s", val, ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1117. pvZone = val
  1118. }
  1119. }
  1120. if pvZone == "" {
  1121. // zone and regionID labels are optional in Alibaba PV creation, while PV through UI creation put's a zone PV is associated with and the region
  1122. // can be determined from this information. If pv is provision via yaml and the block is missing that's the only time it gets defaulted to clusterRegion.
  1123. if pv.Spec.NodeAffinity != nil {
  1124. nodeAffinity := pv.Spec.NodeAffinity
  1125. if nodeAffinity.Required != nil && nodeAffinity.Required.NodeSelectorTerms != nil {
  1126. for _, nodeSelectorTerm := range nodeAffinity.Required.NodeSelectorTerms {
  1127. matchExpression := nodeSelectorTerm.MatchExpressions
  1128. for _, nodeSelectorRequirement := range matchExpression {
  1129. if nodeSelectorRequirement.Key == ALIBABA_DISK_TOPOLOGY_ZONE_LABEL {
  1130. log.Debugf("determinePVRegion will set zone value to: %s through node affinity label: %s for PV name: %s", nodeSelectorRequirement.Values[0], ALIBABA_DISK_TOPOLOGY_ZONE_LABEL, pv.Name)
  1131. pvZone = nodeSelectorRequirement.Values[0]
  1132. }
  1133. }
  1134. }
  1135. }
  1136. }
  1137. }
  1138. for _, region := range alibabaRegions {
  1139. if strings.Contains(pvZone, region) {
  1140. log.Debugf("determinePVRegion determined region of %s through zone affiliation of the PV %s\n", region, pvZone)
  1141. return region
  1142. }
  1143. }
  1144. return ""
  1145. }