cluster_helpers.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838
  1. package costmodel
  2. import (
  3. "strconv"
  4. "time"
  5. "github.com/opencost/opencost/pkg/cloud/models"
  6. "github.com/opencost/opencost/pkg/cloud/provider"
  7. "github.com/opencost/opencost/pkg/env"
  8. "github.com/opencost/opencost/pkg/log"
  9. "github.com/opencost/opencost/pkg/prom"
  10. )
  11. // mergeTypeMaps takes two maps of (cluster name, node name) -> node type
  12. // and combines them into a single map, preferring the k/v pairs in
  13. // the first map.
  14. func mergeTypeMaps(clusterAndNameToType1, clusterAndNameToType2 map[nodeIdentifierNoProviderID]string) map[nodeIdentifierNoProviderID]string {
  15. merged := map[nodeIdentifierNoProviderID]string{}
  16. for k, v := range clusterAndNameToType2 {
  17. merged[k] = v
  18. }
  19. // This ordering ensures the mappings in the first arg are preferred.
  20. for k, v := range clusterAndNameToType1 {
  21. merged[k] = v
  22. }
  23. return merged
  24. }
  25. func buildCPUCostMap(
  26. resNodeCPUCost []*prom.QueryResult,
  27. cp models.Provider,
  28. preemptible map[NodeIdentifier]bool,
  29. ) (
  30. map[NodeIdentifier]float64,
  31. map[nodeIdentifierNoProviderID]string,
  32. ) {
  33. cpuCostMap := make(map[NodeIdentifier]float64)
  34. clusterAndNameToType := make(map[nodeIdentifierNoProviderID]string)
  35. customPricingEnabled := provider.CustomPricesEnabled(cp)
  36. customPricingConfig, err := cp.GetConfig()
  37. if err != nil {
  38. log.Warnf("ClusterNodes: failed to load custom pricing: %s", err)
  39. }
  40. for _, result := range resNodeCPUCost {
  41. cluster, err := result.GetString(env.GetPromClusterLabel())
  42. if err != nil {
  43. cluster = env.GetClusterID()
  44. }
  45. name, err := result.GetString("node")
  46. if err != nil {
  47. log.Warnf("ClusterNodes: CPU cost data missing node")
  48. continue
  49. }
  50. nodeType, _ := result.GetString("instance_type")
  51. providerID, _ := result.GetString("provider_id")
  52. key := NodeIdentifier{
  53. Cluster: cluster,
  54. Name: name,
  55. ProviderID: provider.ParseID(providerID),
  56. }
  57. keyNon := nodeIdentifierNoProviderID{
  58. Cluster: cluster,
  59. Name: name,
  60. }
  61. var cpuCost float64
  62. if customPricingEnabled && customPricingConfig != nil {
  63. var customCPUStr string
  64. if spot, ok := preemptible[key]; ok && spot {
  65. customCPUStr = customPricingConfig.SpotCPU
  66. } else {
  67. customCPUStr = customPricingConfig.CPU
  68. }
  69. customCPUCost, err := strconv.ParseFloat(customCPUStr, 64)
  70. if err != nil {
  71. log.Warnf("ClusterNodes: error parsing custom CPU price: %s", customCPUStr)
  72. }
  73. cpuCost = customCPUCost
  74. } else {
  75. cpuCost = result.Values[0].Value
  76. }
  77. clusterAndNameToType[keyNon] = nodeType
  78. cpuCostMap[key] = cpuCost
  79. }
  80. return cpuCostMap, clusterAndNameToType
  81. }
  82. func buildRAMCostMap(
  83. resNodeRAMCost []*prom.QueryResult,
  84. cp models.Provider,
  85. preemptible map[NodeIdentifier]bool,
  86. ) (
  87. map[NodeIdentifier]float64,
  88. map[nodeIdentifierNoProviderID]string,
  89. ) {
  90. ramCostMap := make(map[NodeIdentifier]float64)
  91. clusterAndNameToType := make(map[nodeIdentifierNoProviderID]string)
  92. customPricingEnabled := provider.CustomPricesEnabled(cp)
  93. customPricingConfig, err := cp.GetConfig()
  94. if err != nil {
  95. log.Warnf("ClusterNodes: failed to load custom pricing: %s", err)
  96. }
  97. for _, result := range resNodeRAMCost {
  98. cluster, err := result.GetString(env.GetPromClusterLabel())
  99. if err != nil {
  100. cluster = env.GetClusterID()
  101. }
  102. name, err := result.GetString("node")
  103. if err != nil {
  104. log.Warnf("ClusterNodes: RAM cost data missing node")
  105. continue
  106. }
  107. nodeType, _ := result.GetString("instance_type")
  108. providerID, _ := result.GetString("provider_id")
  109. key := NodeIdentifier{
  110. Cluster: cluster,
  111. Name: name,
  112. ProviderID: provider.ParseID(providerID),
  113. }
  114. keyNon := nodeIdentifierNoProviderID{
  115. Cluster: cluster,
  116. Name: name,
  117. }
  118. var ramCost float64
  119. if customPricingEnabled && customPricingConfig != nil {
  120. var customRAMStr string
  121. if spot, ok := preemptible[key]; ok && spot {
  122. customRAMStr = customPricingConfig.SpotRAM
  123. } else {
  124. customRAMStr = customPricingConfig.RAM
  125. }
  126. customRAMCost, err := strconv.ParseFloat(customRAMStr, 64)
  127. if err != nil {
  128. log.Warnf("ClusterNodes: error parsing custom RAM price: %s", customRAMStr)
  129. }
  130. ramCost = customRAMCost / 1024 / 1024 / 1024
  131. } else {
  132. ramCost = result.Values[0].Value
  133. }
  134. clusterAndNameToType[keyNon] = nodeType
  135. ramCostMap[key] = ramCost
  136. }
  137. return ramCostMap, clusterAndNameToType
  138. }
  139. func buildGPUCostMap(
  140. resNodeGPUCost []*prom.QueryResult,
  141. gpuCountMap map[NodeIdentifier]float64,
  142. cp models.Provider,
  143. preemptible map[NodeIdentifier]bool,
  144. ) (
  145. map[NodeIdentifier]float64,
  146. map[nodeIdentifierNoProviderID]string,
  147. ) {
  148. gpuCostMap := make(map[NodeIdentifier]float64)
  149. clusterAndNameToType := make(map[nodeIdentifierNoProviderID]string)
  150. customPricingEnabled := provider.CustomPricesEnabled(cp)
  151. customPricingConfig, err := cp.GetConfig()
  152. if err != nil {
  153. log.Warnf("ClusterNodes: failed to load custom pricing: %s", err)
  154. }
  155. for _, result := range resNodeGPUCost {
  156. cluster, err := result.GetString(env.GetPromClusterLabel())
  157. if err != nil {
  158. cluster = env.GetClusterID()
  159. }
  160. name, err := result.GetString("node")
  161. if err != nil {
  162. log.Warnf("ClusterNodes: GPU cost data missing node")
  163. continue
  164. }
  165. nodeType, _ := result.GetString("instance_type")
  166. providerID, _ := result.GetString("provider_id")
  167. key := NodeIdentifier{
  168. Cluster: cluster,
  169. Name: name,
  170. ProviderID: provider.ParseID(providerID),
  171. }
  172. keyNon := nodeIdentifierNoProviderID{
  173. Cluster: cluster,
  174. Name: name,
  175. }
  176. var gpuCost float64
  177. if customPricingEnabled && customPricingConfig != nil {
  178. var customGPUStr string
  179. if spot, ok := preemptible[key]; ok && spot {
  180. customGPUStr = customPricingConfig.SpotGPU
  181. } else {
  182. customGPUStr = customPricingConfig.GPU
  183. }
  184. customGPUCost, err := strconv.ParseFloat(customGPUStr, 64)
  185. if err != nil {
  186. log.Warnf("ClusterNodes: error parsing custom GPU price: %s", customGPUStr)
  187. }
  188. gpuCost = customGPUCost
  189. } else {
  190. gpuCost = result.Values[0].Value
  191. }
  192. clusterAndNameToType[keyNon] = nodeType
  193. // If gpu count is available use it to multiply gpu cost
  194. if value, ok := gpuCountMap[key]; ok {
  195. gpuCostMap[key] = gpuCost * value
  196. } else {
  197. gpuCostMap[key] = 0
  198. }
  199. }
  200. return gpuCostMap, clusterAndNameToType
  201. }
  202. func buildGPUCountMap(
  203. resNodeGPUCount []*prom.QueryResult,
  204. ) map[NodeIdentifier]float64 {
  205. gpuCountMap := make(map[NodeIdentifier]float64)
  206. for _, result := range resNodeGPUCount {
  207. cluster, err := result.GetString(env.GetPromClusterLabel())
  208. if err != nil {
  209. cluster = env.GetClusterID()
  210. }
  211. name, err := result.GetString("node")
  212. if err != nil {
  213. log.Warnf("ClusterNodes: GPU count data missing node")
  214. continue
  215. }
  216. gpuCount := result.Values[0].Value
  217. providerID, _ := result.GetString("provider_id")
  218. key := NodeIdentifier{
  219. Cluster: cluster,
  220. Name: name,
  221. ProviderID: provider.ParseID(providerID),
  222. }
  223. gpuCountMap[key] = gpuCount
  224. }
  225. return gpuCountMap
  226. }
  227. func buildCPUCoresMap(
  228. resNodeCPUCores []*prom.QueryResult,
  229. ) map[nodeIdentifierNoProviderID]float64 {
  230. m := make(map[nodeIdentifierNoProviderID]float64)
  231. for _, result := range resNodeCPUCores {
  232. cluster, err := result.GetString(env.GetPromClusterLabel())
  233. if err != nil {
  234. cluster = env.GetClusterID()
  235. }
  236. name, err := result.GetString("node")
  237. if err != nil {
  238. log.Warnf("ClusterNodes: CPU cores data missing node")
  239. continue
  240. }
  241. cpuCores := result.Values[0].Value
  242. key := nodeIdentifierNoProviderID{
  243. Cluster: cluster,
  244. Name: name,
  245. }
  246. m[key] = cpuCores
  247. }
  248. return m
  249. }
  250. func buildRAMBytesMap(resNodeRAMBytes []*prom.QueryResult) map[nodeIdentifierNoProviderID]float64 {
  251. m := make(map[nodeIdentifierNoProviderID]float64)
  252. for _, result := range resNodeRAMBytes {
  253. cluster, err := result.GetString(env.GetPromClusterLabel())
  254. if err != nil {
  255. cluster = env.GetClusterID()
  256. }
  257. name, err := result.GetString("node")
  258. if err != nil {
  259. log.Warnf("ClusterNodes: RAM bytes data missing node")
  260. continue
  261. }
  262. ramBytes := result.Values[0].Value
  263. key := nodeIdentifierNoProviderID{
  264. Cluster: cluster,
  265. Name: name,
  266. }
  267. m[key] = ramBytes
  268. }
  269. return m
  270. }
  271. // Mapping of cluster/node=cpu for computing resource efficiency
  272. func buildCPUBreakdownMap(resNodeCPUModeTotal []*prom.QueryResult) map[nodeIdentifierNoProviderID]*ClusterCostsBreakdown {
  273. cpuBreakdownMap := make(map[nodeIdentifierNoProviderID]*ClusterCostsBreakdown)
  274. // Mapping of cluster/node=cpu for computing resource efficiency
  275. clusterNodeCPUTotal := map[nodeIdentifierNoProviderID]float64{}
  276. // Mapping of cluster/node:mode=cpu for computing resource efficiency
  277. clusterNodeModeCPUTotal := map[nodeIdentifierNoProviderID]map[string]float64{}
  278. // Build intermediate structures for CPU usage by (cluster, node) and by
  279. // (cluster, node, mode) for computing resouce efficiency
  280. for _, result := range resNodeCPUModeTotal {
  281. cluster, err := result.GetString(env.GetPromClusterLabel())
  282. if err != nil {
  283. cluster = env.GetClusterID()
  284. }
  285. node, err := result.GetString("kubernetes_node")
  286. if err != nil {
  287. log.DedupedWarningf(5, "ClusterNodes: CPU mode data missing node")
  288. continue
  289. }
  290. mode, err := result.GetString("mode")
  291. if err != nil {
  292. log.Warnf("ClusterNodes: unable to read CPU mode: %s", err)
  293. mode = "other"
  294. }
  295. key := nodeIdentifierNoProviderID{
  296. Cluster: cluster,
  297. Name: node,
  298. }
  299. total := result.Values[0].Value
  300. // Increment total
  301. clusterNodeCPUTotal[key] += total
  302. // Increment mode
  303. if _, ok := clusterNodeModeCPUTotal[key]; !ok {
  304. clusterNodeModeCPUTotal[key] = map[string]float64{}
  305. }
  306. clusterNodeModeCPUTotal[key][mode] += total
  307. }
  308. // Compute resource efficiency from intermediate structures
  309. for key, total := range clusterNodeCPUTotal {
  310. if modeTotals, ok := clusterNodeModeCPUTotal[key]; ok {
  311. for mode, subtotal := range modeTotals {
  312. // Compute percentage for the current cluster, node, mode
  313. pct := 0.0
  314. if total > 0 {
  315. pct = subtotal / total
  316. }
  317. if _, ok := cpuBreakdownMap[key]; !ok {
  318. cpuBreakdownMap[key] = &ClusterCostsBreakdown{}
  319. }
  320. switch mode {
  321. case "idle":
  322. cpuBreakdownMap[key].Idle += pct
  323. case "system":
  324. cpuBreakdownMap[key].System += pct
  325. case "user":
  326. cpuBreakdownMap[key].User += pct
  327. default:
  328. cpuBreakdownMap[key].Other += pct
  329. }
  330. }
  331. }
  332. }
  333. return cpuBreakdownMap
  334. }
  335. func buildOverheadMap(capRam, allocRam, capCPU, allocCPU map[nodeIdentifierNoProviderID]float64) map[nodeIdentifierNoProviderID]*NodeOverhead {
  336. m := make(map[nodeIdentifierNoProviderID]*NodeOverhead, len(capRam))
  337. for identifier, ramCapacity := range capRam {
  338. allocatableRam, ok := allocRam[identifier]
  339. if !ok {
  340. log.Warnf("Could not find allocatable ram for node %s", identifier.Name)
  341. continue
  342. }
  343. overheadBytes := ramCapacity - allocatableRam
  344. m[identifier] = &NodeOverhead{
  345. RamOverheadFraction: overheadBytes / ramCapacity,
  346. }
  347. }
  348. for identifier, cpuCapacity := range capCPU {
  349. allocatableCPU, ok := allocCPU[identifier]
  350. if !ok {
  351. log.Warnf("Could not find allocatable cpu for node %s", identifier.Name)
  352. continue
  353. }
  354. overhead := cpuCapacity - allocatableCPU
  355. if _, found := m[identifier]; found {
  356. m[identifier].CpuOverheadFraction = overhead / cpuCapacity
  357. } else {
  358. m[identifier] = &NodeOverhead{
  359. CpuOverheadFraction: overhead / cpuCapacity,
  360. }
  361. }
  362. }
  363. return m
  364. }
  365. func buildRAMUserPctMap(resNodeRAMUserPct []*prom.QueryResult) map[nodeIdentifierNoProviderID]float64 {
  366. m := make(map[nodeIdentifierNoProviderID]float64)
  367. for _, result := range resNodeRAMUserPct {
  368. cluster, err := result.GetString(env.GetPromClusterLabel())
  369. if err != nil {
  370. cluster = env.GetClusterID()
  371. }
  372. name, err := result.GetString("instance")
  373. if err != nil {
  374. log.Warnf("ClusterNodes: RAM user percent missing node")
  375. continue
  376. }
  377. pct := result.Values[0].Value
  378. key := nodeIdentifierNoProviderID{
  379. Cluster: cluster,
  380. Name: name,
  381. }
  382. m[key] = pct
  383. }
  384. return m
  385. }
  386. func buildRAMSystemPctMap(resNodeRAMSystemPct []*prom.QueryResult) map[nodeIdentifierNoProviderID]float64 {
  387. m := make(map[nodeIdentifierNoProviderID]float64)
  388. for _, result := range resNodeRAMSystemPct {
  389. cluster, err := result.GetString(env.GetPromClusterLabel())
  390. if err != nil {
  391. cluster = env.GetClusterID()
  392. }
  393. name, err := result.GetString("instance")
  394. if err != nil {
  395. log.Warnf("ClusterNodes: RAM system percent missing node")
  396. continue
  397. }
  398. pct := result.Values[0].Value
  399. key := nodeIdentifierNoProviderID{
  400. Cluster: cluster,
  401. Name: name,
  402. }
  403. m[key] = pct
  404. }
  405. return m
  406. }
  407. type activeData struct {
  408. start time.Time
  409. end time.Time
  410. minutes float64
  411. }
  412. func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Duration) map[NodeIdentifier]activeData {
  413. m := make(map[NodeIdentifier]activeData)
  414. for _, result := range resActiveMins {
  415. cluster, err := result.GetString(env.GetPromClusterLabel())
  416. if err != nil {
  417. cluster = env.GetClusterID()
  418. }
  419. name, err := result.GetString("node")
  420. if err != nil {
  421. log.Warnf("ClusterNodes: active mins missing node")
  422. continue
  423. }
  424. providerID, _ := result.GetString("provider_id")
  425. key := NodeIdentifier{
  426. Cluster: cluster,
  427. Name: name,
  428. ProviderID: provider.ParseID(providerID),
  429. }
  430. if len(result.Values) == 0 {
  431. continue
  432. }
  433. s := time.Unix(int64(result.Values[0].Timestamp), 0)
  434. e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0)
  435. mins := e.Sub(s).Minutes()
  436. // TODO niko/assets if mins >= threshold, interpolate for missing data?
  437. m[key] = activeData{
  438. start: s,
  439. end: e,
  440. minutes: mins,
  441. }
  442. }
  443. return m
  444. }
  445. // Determine preemptibility with node labels
  446. // node id -> is preemptible?
  447. func buildPreemptibleMap(
  448. resIsSpot []*prom.QueryResult,
  449. ) map[NodeIdentifier]bool {
  450. m := make(map[NodeIdentifier]bool)
  451. for _, result := range resIsSpot {
  452. nodeName, err := result.GetString("node")
  453. if err != nil {
  454. continue
  455. }
  456. // GCP preemptible label
  457. pre := result.Values[0].Value
  458. cluster, err := result.GetString(env.GetPromClusterLabel())
  459. if err != nil {
  460. cluster = env.GetClusterID()
  461. }
  462. providerID, _ := result.GetString("provider_id")
  463. key := NodeIdentifier{
  464. Cluster: cluster,
  465. Name: nodeName,
  466. ProviderID: provider.ParseID(providerID),
  467. }
  468. // TODO(michaelmdresser): check this condition at merge time?
  469. // if node, ok := nodeMap[key]; pre > 0.0 && ok {
  470. // node.Preemptible = true
  471. // }
  472. m[key] = pre > 0.0
  473. // TODO AWS preemptible
  474. // TODO Azure preemptible
  475. }
  476. return m
  477. }
  478. func buildLabelsMap(
  479. resLabels []*prom.QueryResult,
  480. ) map[nodeIdentifierNoProviderID]map[string]string {
  481. m := make(map[nodeIdentifierNoProviderID]map[string]string)
  482. // Copy labels into node
  483. for _, result := range resLabels {
  484. cluster, err := result.GetString(env.GetPromClusterLabel())
  485. if err != nil {
  486. cluster = env.GetClusterID()
  487. }
  488. node, err := result.GetString("node")
  489. if err != nil {
  490. log.DedupedWarningf(5, "ClusterNodes: label data missing node")
  491. continue
  492. }
  493. key := nodeIdentifierNoProviderID{
  494. Cluster: cluster,
  495. Name: node,
  496. }
  497. m[key] = make(map[string]string)
  498. for name, value := range result.Metric {
  499. if val, ok := value.(string); ok {
  500. m[key][name] = val
  501. }
  502. }
  503. }
  504. return m
  505. }
  506. // checkForKeyAndInitIfMissing inits a key in the provided nodemap if
  507. // it does not exist. Intended to be called ONLY by buildNodeMap
  508. func checkForKeyAndInitIfMissing(
  509. nodeMap map[NodeIdentifier]*Node,
  510. key NodeIdentifier,
  511. clusterAndNameToType map[nodeIdentifierNoProviderID]string,
  512. ) {
  513. if _, ok := nodeMap[key]; !ok {
  514. // default nodeType in case we don't have the mapping
  515. var nodeType string
  516. if t, ok := clusterAndNameToType[nodeIdentifierNoProviderID{
  517. Cluster: key.Cluster,
  518. Name: key.Name,
  519. }]; ok {
  520. nodeType = t
  521. } else {
  522. log.Warnf("ClusterNodes: Type does not exist for node identifier %s", key)
  523. }
  524. nodeMap[key] = &Node{
  525. Cluster: key.Cluster,
  526. Name: key.Name,
  527. NodeType: nodeType,
  528. ProviderID: key.ProviderID,
  529. CPUBreakdown: &ClusterCostsBreakdown{},
  530. RAMBreakdown: &ClusterCostsBreakdown{},
  531. }
  532. }
  533. }
  534. // buildNodeMap creates the main set of node data for ClusterNodes from
  535. // the data maps built from Prometheus queries. Some of the Prometheus
  536. // data has access to the provider_id field and some does not. To get
  537. // around this problem, we use the data that includes provider_id
  538. // to build up the definitive set of nodes and then use the data
  539. // with less-specific identifiers (i.e. without provider_id) to fill
  540. // in the remaining fields.
  541. //
  542. // For example, let's say we have nodes identified like so:
  543. // cluster name/node name/provider_id. For the sake of the example,
  544. // we will also limit data to CPU cost, CPU cores, and preemptibility.
  545. //
  546. // We have CPU cost data that looks like this:
  547. // cluster1/node1/prov_node1_A: $10
  548. // cluster1/node1/prov_node1_B: $8
  549. // cluster1/node2/prov_node2: $15
  550. //
  551. // We have Preemptible data that looks like this:
  552. // cluster1/node1/prov_node1_A: true
  553. // cluster1/node1/prov_node1_B: false
  554. // cluster1/node2/prov_node2_B: false
  555. //
  556. // We have CPU cores data that looks like this:
  557. // cluster1/node1: 4
  558. // cluster1/node2: 6
  559. //
  560. // This function first combines the data that is fully identified,
  561. // creating the following:
  562. // cluster1/node1/prov_node1_A: CPUCost($10), Preemptible(true)
  563. // cluster1/node1/prov_node1_B: CPUCost($8), Preemptible(false)
  564. // cluster1/node2/prov_node2: CPUCost($15), Preemptible(false)
  565. //
  566. // It then uses the less-specific data to extend the specific data,
  567. // making the following:
  568. // cluster1/node1/prov_node1_A: CPUCost($10), Preemptible(true), Cores(4)
  569. // cluster1/node1/prov_node1_B: CPUCost($8), Preemptible(false), Cores(4)
  570. // cluster1/node2/prov_node2: CPUCost($15), Preemptible(false), Cores(6)
  571. //
  572. // In the situation where provider_id doesn't exist for any metrics,
  573. // that is the same as all provider_ids being empty strings. If
  574. // provider_id doesn't exist at all, then we (without having to do
  575. // extra work) easily fall back on identifying nodes only by cluster name
  576. // and node name because the provider_id part of the key will always
  577. // be the empty string.
  578. //
  579. // It is worth nothing that, in this approach, if a node is not present
  580. // in the more specific data but is present in the less-specific data,
  581. // that data is never processed into the final node map. For example,
  582. // let's say the CPU cores map has the following entry:
  583. // cluster1/node8: 6
  584. // But none of the maps with provider_id (CPU cost, RAM cost, etc.)
  585. // have an identifier for cluster1/node8 (regardless of provider_id).
  586. // In this situation, the final node map will not have a cluster1/node8
  587. // entry. This could be fixed by iterating over all of the less specific
  588. // identifiers and, inside that iteration, all of the identifiers in
  589. // the node map, but this would introduce a roughly quadratic time
  590. // complexity.
  591. func buildNodeMap(
  592. cpuCostMap, ramCostMap, gpuCostMap, gpuCountMap map[NodeIdentifier]float64,
  593. cpuCoresMap, ramBytesMap, ramUserPctMap,
  594. ramSystemPctMap map[nodeIdentifierNoProviderID]float64,
  595. cpuBreakdownMap map[nodeIdentifierNoProviderID]*ClusterCostsBreakdown,
  596. activeDataMap map[NodeIdentifier]activeData,
  597. preemptibleMap map[NodeIdentifier]bool,
  598. labelsMap map[nodeIdentifierNoProviderID]map[string]string,
  599. clusterAndNameToType map[nodeIdentifierNoProviderID]string,
  600. res time.Duration,
  601. overheadMap map[nodeIdentifierNoProviderID]*NodeOverhead,
  602. ) map[NodeIdentifier]*Node {
  603. nodeMap := make(map[NodeIdentifier]*Node)
  604. // Initialize the map with the most-specific data:
  605. for id, cost := range cpuCostMap {
  606. checkForKeyAndInitIfMissing(nodeMap, id, clusterAndNameToType)
  607. nodeMap[id].CPUCost = cost
  608. }
  609. for id, cost := range ramCostMap {
  610. checkForKeyAndInitIfMissing(nodeMap, id, clusterAndNameToType)
  611. nodeMap[id].RAMCost = cost
  612. }
  613. for id, cost := range gpuCostMap {
  614. checkForKeyAndInitIfMissing(nodeMap, id, clusterAndNameToType)
  615. nodeMap[id].GPUCost = cost
  616. }
  617. for id, count := range gpuCountMap {
  618. checkForKeyAndInitIfMissing(nodeMap, id, clusterAndNameToType)
  619. nodeMap[id].GPUCount = count
  620. }
  621. for id, preemptible := range preemptibleMap {
  622. checkForKeyAndInitIfMissing(nodeMap, id, clusterAndNameToType)
  623. nodeMap[id].Preemptible = preemptible
  624. }
  625. for id, activeData := range activeDataMap {
  626. checkForKeyAndInitIfMissing(nodeMap, id, clusterAndNameToType)
  627. nodeMap[id].Start = activeData.start
  628. nodeMap[id].End = activeData.end
  629. nodeMap[id].Minutes = nodeMap[id].End.Sub(nodeMap[id].Start).Minutes()
  630. }
  631. // We now merge in data that doesn't have a provider id by looping over
  632. // all keys already added and inserting data according to their
  633. // cluster name/node name combos.
  634. for id, nodePtr := range nodeMap {
  635. clusterAndNameID := nodeIdentifierNoProviderID{
  636. Cluster: id.Cluster,
  637. Name: id.Name,
  638. }
  639. if cores, ok := cpuCoresMap[clusterAndNameID]; ok {
  640. nodePtr.CPUCores = cores
  641. if v, ok := partialCPUMap[nodePtr.NodeType]; ok {
  642. if cores > 0 {
  643. nodePtr.CPUCores = v
  644. adjustmentFactor := v / cores
  645. nodePtr.CPUCost = nodePtr.CPUCost * adjustmentFactor
  646. }
  647. }
  648. }
  649. if ramBytes, ok := ramBytesMap[clusterAndNameID]; ok {
  650. nodePtr.RAMBytes = ramBytes
  651. }
  652. if ramUserPct, ok := ramUserPctMap[clusterAndNameID]; ok {
  653. nodePtr.RAMBreakdown.User = ramUserPct
  654. }
  655. if ramSystemPct, ok := ramSystemPctMap[clusterAndNameID]; ok {
  656. nodePtr.RAMBreakdown.System = ramSystemPct
  657. }
  658. if cpuBreakdown, ok := cpuBreakdownMap[clusterAndNameID]; ok {
  659. nodePtr.CPUBreakdown = cpuBreakdown
  660. }
  661. if labels, ok := labelsMap[clusterAndNameID]; ok {
  662. nodePtr.Labels = labels
  663. }
  664. if overhead, ok := overheadMap[clusterAndNameID]; ok {
  665. nodePtr.Overhead = overhead
  666. } else {
  667. // we were unable to compute overhead for this node
  668. // assume default case of no overhead
  669. nodePtr.Overhead = &NodeOverhead{}
  670. log.Warnf("unable to compute overhead for node %s - defaulting to no overhead", clusterAndNameID.Name)
  671. }
  672. }
  673. return nodeMap
  674. }