asset.go 77 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067
  1. package kubecost
  2. import (
  3. "bytes"
  4. "encoding"
  5. "fmt"
  6. "strings"
  7. "sync"
  8. "time"
  9. "github.com/kubecost/cost-model/pkg/log"
  10. "github.com/kubecost/cost-model/pkg/util/json"
  11. )
  12. // UndefinedKey is used in composing Asset group keys if the group does not have that property defined.
  13. // E.g. if aggregating on Cluster, Assets in the AssetSet where Asset has no cluster will be grouped under key "__undefined__"
  14. const UndefinedKey = "__undefined__"
  15. // Asset defines an entity within a cluster that has a defined cost over a
  16. // given period of time.
  17. type Asset interface {
  18. // Type identifies the kind of Asset, which must always exist and should
  19. // be defined by the underlying type implementing the interface.
  20. Type() AssetType
  21. // Properties are a map of predefined traits, which may or may not exist,
  22. // but must conform to the AssetProperty schema
  23. Properties() *AssetProperties
  24. SetProperties(*AssetProperties)
  25. // Labels are a map of undefined string-to-string values
  26. Labels() AssetLabels
  27. SetLabels(AssetLabels)
  28. // Monetary values
  29. Adjustment() float64
  30. SetAdjustment(float64)
  31. TotalCost() float64
  32. // Temporal values
  33. Start() time.Time
  34. End() time.Time
  35. SetStartEnd(time.Time, time.Time)
  36. Window() Window
  37. ExpandWindow(Window)
  38. Minutes() float64
  39. // Operations and comparisons
  40. Add(Asset) Asset
  41. Clone() Asset
  42. Equal(Asset) bool
  43. // Representations
  44. encoding.BinaryMarshaler
  45. encoding.BinaryUnmarshaler
  46. json.Marshaler
  47. fmt.Stringer
  48. }
  49. // AssetToExternalAllocation converts the given asset to an Allocation, given
  50. // the properties to use to aggregate, and the mapping from Allocation property
  51. // to Asset label. For example, consider this asset:
  52. //
  53. // CURRENT: Asset ETL stores its data ALREADY MAPPED from label to k8s concept. This isn't ideal-- see the TOOD.
  54. // Cloud {
  55. // TotalCost: 10.00,
  56. // Labels{
  57. // "kubernetes_namespace":"monitoring",
  58. // "env":"prod"
  59. // }
  60. // }
  61. //
  62. // Given the following parameters, we expect to return:
  63. //
  64. // 1) single-prop full match
  65. // aggregateBy = ["namespace"]
  66. // => Allocation{Name: "monitoring", ExternalCost: 10.00, TotalCost: 10.00}, nil
  67. //
  68. // 2) multi-prop full match
  69. // aggregateBy = ["namespace", "label:env"]
  70. // allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
  71. // => Allocation{Name: "monitoring/env=prod", ExternalCost: 10.00, TotalCost: 10.00}, nil
  72. //
  73. // 3) multi-prop partial match
  74. // aggregateBy = ["namespace", "label:foo"]
  75. // => Allocation{Name: "monitoring/__unallocated__", ExternalCost: 10.00, TotalCost: 10.00}, nil
  76. //
  77. // 4) no match
  78. // aggregateBy = ["cluster"]
  79. // => nil, err
  80. //
  81. // TODO:
  82. // Cloud {
  83. // TotalCost: 10.00,
  84. // Labels{
  85. // "kubernetes_namespace":"monitoring",
  86. // "env":"prod"
  87. // }
  88. // }
  89. //
  90. // Given the following parameters, we expect to return:
  91. //
  92. // 1) single-prop full match
  93. // aggregateBy = ["namespace"]
  94. // allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
  95. // => Allocation{Name: "monitoring", ExternalCost: 10.00, TotalCost: 10.00}, nil
  96. //
  97. // 2) multi-prop full match
  98. // aggregateBy = ["namespace", "label:env"]
  99. // allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
  100. // => Allocation{Name: "monitoring/env=prod", ExternalCost: 10.00, TotalCost: 10.00}, nil
  101. //
  102. // 3) multi-prop partial match
  103. // aggregateBy = ["namespace", "label:foo"]
  104. // allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
  105. // => Allocation{Name: "monitoring/__unallocated__", ExternalCost: 10.00, TotalCost: 10.00}, nil
  106. //
  107. // 4) no match
  108. // aggregateBy = ["cluster"]
  109. // allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
  110. // => nil, err
  111. //
  112. // (See asset_test.go for assertions of these examples and more.)
  113. func AssetToExternalAllocation(asset Asset, aggregateBy []string, labelConfig *LabelConfig) (*Allocation, error) {
  114. if asset == nil {
  115. return nil, fmt.Errorf("asset is nil")
  116. }
  117. // Use default label config if one is not provided.
  118. if labelConfig == nil {
  119. labelConfig = NewLabelConfig()
  120. }
  121. // names will collect the slash-separated names accrued by iterating over
  122. // aggregateBy and checking the relevant labels.
  123. names := []string{}
  124. // match records whether or not a match was found in the Asset labels,
  125. // such that is can genuinely be turned into an external Allocation.
  126. match := false
  127. // props records the relevant Properties to set on the resultant Allocation
  128. props := AllocationProperties{}
  129. // For each aggregation parameter, try to find a match in the asset's
  130. // labels, using the labelConfig to translate. For an aggregation parameter
  131. // defined by a label (e.g. "label:app") this is simple: look for the label
  132. // and use it (e.g. if "app" is a defined label on the asset, then use its
  133. // value). For an aggregation parameter defined by a non-label property
  134. // (e.g. "namespace") this requires using the labelConfig to look up the
  135. // label name associated with that property and to use the value under that
  136. // label, if set (e.g. if the aggregation property is "namespace" and the
  137. // labelConfig is configured with "namespace_external_label" => "kubens"
  138. // and the asset has label "kubens":"kubecost", then file the asset as an
  139. // external cost under "kubecost").
  140. for _, aggBy := range aggregateBy {
  141. name := labelConfig.GetExternalAllocationName(asset.Labels(), aggBy)
  142. if name == "" {
  143. // No matching label has been defined in the cost-analyzer label config
  144. // relating to the given aggregateBy property.
  145. names = append(names, UnallocatedSuffix)
  146. continue
  147. } else {
  148. names = append(names, name)
  149. match = true
  150. // Set the corresponding property on props
  151. switch aggBy {
  152. case AllocationClusterProp:
  153. props.Cluster = name
  154. case AllocationNodeProp:
  155. props.Node = name
  156. case AllocationNamespaceProp:
  157. props.Namespace = name
  158. case AllocationControllerKindProp:
  159. props.ControllerKind = name
  160. case AllocationControllerProp:
  161. props.Controller = name
  162. case AllocationPodProp:
  163. props.Pod = name
  164. case AllocationContainerProp:
  165. props.Container = name
  166. case AllocationServiceProp:
  167. props.Services = []string{name}
  168. default:
  169. if strings.HasPrefix(aggBy, "label:") {
  170. // Set the corresponding label in props
  171. if props.Labels == nil {
  172. props.Labels = map[string]string{}
  173. }
  174. labelName := strings.TrimPrefix(aggBy, "label:")
  175. labelValue := strings.TrimPrefix(name, labelName+"=")
  176. props.Labels[labelName] = labelValue
  177. }
  178. }
  179. }
  180. }
  181. // If not a signle aggregation property generated a matching label property,
  182. // then consider the asset ineligible to be treated as an external allocation.
  183. if !match {
  184. return nil, fmt.Errorf("asset does not qualify as an external allocation")
  185. }
  186. // Use naming to label as an external allocation. See IsExternal() for more.
  187. names = append(names, ExternalSuffix)
  188. // TODO: external allocation: efficiency?
  189. // TODO: external allocation: resource totals?
  190. return &Allocation{
  191. Name: strings.Join(names, "/"),
  192. Properties: &props,
  193. Window: asset.Window().Clone(),
  194. Start: asset.Start(),
  195. End: asset.End(),
  196. ExternalCost: asset.TotalCost(),
  197. }, nil
  198. }
  199. // key is used to determine uniqueness of an Asset, for instance during Insert
  200. // to determine if two Assets should be combined. Passing `nil` `aggregateBy` indicates
  201. // that all available `AssetProperty` keys should be used. Passing empty `aggregateBy` indicates that
  202. // no key should be used (e.g. to aggregate all assets). Passing one or more `aggregateBy`
  203. // values will key by only those values.
  204. // Valid values of `aggregateBy` elements are strings which are an `AssetProperty`, and strings prefixed
  205. // with `"label:"`.
  206. func key(a Asset, aggregateBy []string) (string, error) {
  207. keys := []string{}
  208. if aggregateBy == nil {
  209. aggregateBy = []string{
  210. string(AssetProviderProp),
  211. string(AssetAccountProp),
  212. string(AssetProjectProp),
  213. string(AssetCategoryProp),
  214. string(AssetClusterProp),
  215. string(AssetTypeProp),
  216. string(AssetServiceProp),
  217. string(AssetProviderIDProp),
  218. string(AssetNameProp),
  219. }
  220. }
  221. for _, s := range aggregateBy {
  222. key := ""
  223. switch true {
  224. case s == string(AssetProviderProp):
  225. key = a.Properties().Provider
  226. case s == string(AssetAccountProp):
  227. key = a.Properties().Account
  228. case s == string(AssetProjectProp):
  229. key = a.Properties().Project
  230. case s == string(AssetClusterProp):
  231. key = a.Properties().Cluster
  232. case s == string(AssetCategoryProp):
  233. key = a.Properties().Category
  234. case s == string(AssetTypeProp):
  235. key = a.Type().String()
  236. case s == string(AssetServiceProp):
  237. key = a.Properties().Service
  238. case s == string(AssetProviderIDProp):
  239. key = a.Properties().ProviderID
  240. case s == string(AssetNameProp):
  241. key = a.Properties().Name
  242. case strings.HasPrefix(s, "label:"):
  243. if labelKey := strings.TrimPrefix(s, "label:"); labelKey != "" {
  244. labelVal := a.Labels()[labelKey]
  245. if labelVal == "" {
  246. key = "__undefined__"
  247. } else {
  248. key = fmt.Sprintf("%s=%s", labelKey, labelVal)
  249. }
  250. } else {
  251. // Don't allow aggregating on label ""
  252. return "", fmt.Errorf("attempted to aggregate on invalid key: %s", s)
  253. }
  254. default:
  255. return "", fmt.Errorf("attempted to aggregate on invalid key: %s", s)
  256. }
  257. if key != "" {
  258. keys = append(keys, key)
  259. } else {
  260. keys = append(keys, UndefinedKey)
  261. }
  262. }
  263. return strings.Join(keys, "/"), nil
  264. }
  265. func toString(a Asset) string {
  266. return fmt.Sprintf("%s{%s}%s=%.2f", a.Type().String(), a.Properties(), a.Window(), a.TotalCost())
  267. }
  268. // AssetLabels is a schema-free mapping of key/value pairs that can be
  269. // attributed to an Asset as a flexible a
  270. type AssetLabels map[string]string
  271. // Clone returns a cloned map of labels
  272. func (al AssetLabels) Clone() AssetLabels {
  273. clone := make(AssetLabels, len(al))
  274. for label, value := range al {
  275. clone[label] = value
  276. }
  277. return clone
  278. }
  279. // Equal returns true only if the two set of labels are exact matches
  280. func (al AssetLabels) Equal(that AssetLabels) bool {
  281. if len(al) != len(that) {
  282. return false
  283. }
  284. for label, value := range al {
  285. if thatValue, ok := that[label]; !ok || thatValue != value {
  286. return false
  287. }
  288. }
  289. return true
  290. }
  291. // Merge retains only the labels shared with the given AssetLabels
  292. func (al AssetLabels) Merge(that AssetLabels) AssetLabels {
  293. result := AssetLabels{}
  294. for label, value := range al {
  295. if thatValue, ok := that[label]; ok && thatValue == value {
  296. result[label] = value
  297. }
  298. }
  299. return result
  300. }
  301. // AssetMatchFunc is a function that can be used to match Assets by
  302. // returning true for any given Asset if a condition is met.
  303. type AssetMatchFunc func(Asset) bool
  304. // AssetType identifies a type of Asset
  305. type AssetType int
  306. const (
  307. // AnyAssetType describes the Any AssetType
  308. AnyAssetType AssetType = iota
  309. // CloudAssetType describes the Cloud AssetType
  310. CloudAssetType
  311. // ClusterManagementAssetType describes the ClusterManagement AssetType
  312. ClusterManagementAssetType
  313. // DiskAssetType describes the Disk AssetType
  314. DiskAssetType
  315. // LoadBalancerAssetType describes the LoadBalancer AssetType
  316. LoadBalancerAssetType
  317. // NetworkAssetType describes the Network AssetType
  318. NetworkAssetType
  319. // NodeAssetType describes the Node AssetType
  320. NodeAssetType
  321. // SharedAssetType describes the Shared AssetType
  322. SharedAssetType
  323. )
  324. // ParseAssetType attempts to parse the given string into an AssetType
  325. func ParseAssetType(text string) (AssetType, error) {
  326. switch strings.TrimSpace(strings.ToLower(text)) {
  327. case "cloud":
  328. return CloudAssetType, nil
  329. case "clustermanagement":
  330. return ClusterManagementAssetType, nil
  331. case "disk":
  332. return DiskAssetType, nil
  333. case "loadbalancer":
  334. return LoadBalancerAssetType, nil
  335. case "network":
  336. return NetworkAssetType, nil
  337. case "node":
  338. return NodeAssetType, nil
  339. case "shared":
  340. return SharedAssetType, nil
  341. }
  342. return AnyAssetType, fmt.Errorf("invalid asset type: %s", text)
  343. }
  344. // String converts the given AssetType to a string
  345. func (at AssetType) String() string {
  346. return [...]string{
  347. "Asset",
  348. "Cloud",
  349. "ClusterManagement",
  350. "Disk",
  351. "LoadBalancer",
  352. "Network",
  353. "Node",
  354. "Shared",
  355. }[at]
  356. }
  357. // Any is the most general Asset, which is usually created as a result of
  358. // adding two Assets of different types.
  359. type Any struct {
  360. labels AssetLabels
  361. properties *AssetProperties
  362. start time.Time
  363. end time.Time
  364. window Window
  365. adjustment float64
  366. Cost float64
  367. }
  368. // NewAsset creates a new Any-type Asset for the given period of time
  369. func NewAsset(start, end time.Time, window Window) *Any {
  370. return &Any{
  371. labels: AssetLabels{},
  372. properties: &AssetProperties{},
  373. start: start,
  374. end: end,
  375. window: window.Clone(),
  376. }
  377. }
  378. // Type returns the Asset's type
  379. func (a *Any) Type() AssetType {
  380. return AnyAssetType
  381. }
  382. // Properties returns the Asset's properties
  383. func (a *Any) Properties() *AssetProperties {
  384. return a.properties
  385. }
  386. // SetProperties sets the Asset's properties
  387. func (a *Any) SetProperties(props *AssetProperties) {
  388. a.properties = props
  389. }
  390. // Labels returns the Asset's labels
  391. func (a *Any) Labels() AssetLabels {
  392. return a.labels
  393. }
  394. // SetLabels sets the Asset's labels
  395. func (a *Any) SetLabels(labels AssetLabels) {
  396. a.labels = labels
  397. }
  398. // Adjustment returns the Asset's cost adjustment
  399. func (a *Any) Adjustment() float64 {
  400. return a.adjustment
  401. }
  402. // SetAdjustment sets the Asset's cost adjustment
  403. func (a *Any) SetAdjustment(adj float64) {
  404. a.adjustment = adj
  405. }
  406. // TotalCost returns the Asset's TotalCost
  407. func (a *Any) TotalCost() float64 {
  408. return a.Cost + a.adjustment
  409. }
  410. // Start returns the Asset's start time within the window
  411. func (a *Any) Start() time.Time {
  412. return a.start
  413. }
  414. // End returns the Asset's end time within the window
  415. func (a *Any) End() time.Time {
  416. return a.end
  417. }
  418. // Minutes returns the number of minutes the Asset was active within the window
  419. func (a *Any) Minutes() float64 {
  420. return a.End().Sub(a.Start()).Minutes()
  421. }
  422. // Window returns the Asset's window
  423. func (a *Any) Window() Window {
  424. return a.window
  425. }
  426. // ExpandWindow expands the Asset's window by the given window
  427. func (a *Any) ExpandWindow(window Window) {
  428. a.window = a.window.Expand(window)
  429. }
  430. // SetStartEnd sets the Asset's Start and End fields
  431. func (a *Any) SetStartEnd(start, end time.Time) {
  432. if a.Window().Contains(start) {
  433. a.start = start
  434. } else {
  435. log.Warningf("Any.SetStartEnd: start %s not in %s", start, a.Window())
  436. }
  437. if a.Window().Contains(end) {
  438. a.end = end
  439. } else {
  440. log.Warningf("Any.SetStartEnd: end %s not in %s", end, a.Window())
  441. }
  442. }
  443. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  444. // as much relevant information as possible (i.e. type, properties, labels).
  445. func (a *Any) Add(that Asset) Asset {
  446. this := a.Clone().(*Any)
  447. props := a.Properties().Merge(that.Properties())
  448. labels := a.Labels().Merge(that.Labels())
  449. start := a.Start()
  450. if that.Start().Before(start) {
  451. start = that.Start()
  452. }
  453. end := a.End()
  454. if that.End().After(end) {
  455. end = that.End()
  456. }
  457. window := a.Window().Expand(that.Window())
  458. this.start = start
  459. this.end = end
  460. this.window = window
  461. this.SetProperties(props)
  462. this.SetLabels(labels)
  463. this.adjustment += that.Adjustment()
  464. this.Cost += (that.TotalCost() - that.Adjustment())
  465. return this
  466. }
  467. // Clone returns a cloned instance of the Asset
  468. func (a *Any) Clone() Asset {
  469. return &Any{
  470. labels: a.labels.Clone(),
  471. properties: a.properties.Clone(),
  472. start: a.start,
  473. end: a.end,
  474. window: a.window.Clone(),
  475. adjustment: a.adjustment,
  476. Cost: a.Cost,
  477. }
  478. }
  479. // Equal returns true if the given Asset is an exact match of the receiver
  480. func (a *Any) Equal(that Asset) bool {
  481. t, ok := that.(*Any)
  482. if !ok {
  483. return false
  484. }
  485. if !a.Labels().Equal(that.Labels()) {
  486. return false
  487. }
  488. if !a.Properties().Equal(that.Properties()) {
  489. return false
  490. }
  491. if !a.start.Equal(t.start) {
  492. return false
  493. }
  494. if !a.end.Equal(t.end) {
  495. return false
  496. }
  497. if !a.window.Equal(t.window) {
  498. return false
  499. }
  500. if a.Cost != t.Cost {
  501. return false
  502. }
  503. return true
  504. }
  505. // MarshalJSON implements json.Marshaler
  506. func (a *Any) MarshalJSON() ([]byte, error) {
  507. buffer := bytes.NewBufferString("{")
  508. jsonEncode(buffer, "properties", a.Properties(), ",")
  509. jsonEncode(buffer, "labels", a.Labels(), ",")
  510. jsonEncode(buffer, "window", a.Window(), ",")
  511. jsonEncodeString(buffer, "start", a.Start().Format(time.RFC3339), ",")
  512. jsonEncodeString(buffer, "end", a.End().Format(time.RFC3339), ",")
  513. jsonEncodeFloat64(buffer, "minutes", a.Minutes(), ",")
  514. jsonEncodeFloat64(buffer, "adjustment", a.Adjustment(), ",")
  515. jsonEncodeFloat64(buffer, "totalCost", a.TotalCost(), "")
  516. buffer.WriteString("}")
  517. return buffer.Bytes(), nil
  518. }
  519. // String implements fmt.Stringer
  520. func (a *Any) String() string {
  521. return toString(a)
  522. }
  523. // Cloud describes a cloud asset
  524. type Cloud struct {
  525. labels AssetLabels
  526. properties *AssetProperties
  527. start time.Time
  528. end time.Time
  529. window Window
  530. adjustment float64
  531. Cost float64
  532. Credit float64 // Credit is a negative value representing dollars credited back to a given line-item
  533. }
  534. // NewCloud returns a new Cloud Asset
  535. func NewCloud(category, providerID string, start, end time.Time, window Window) *Cloud {
  536. properties := &AssetProperties{
  537. Category: category,
  538. ProviderID: providerID,
  539. }
  540. return &Cloud{
  541. labels: AssetLabels{},
  542. properties: properties,
  543. start: start,
  544. end: end,
  545. window: window.Clone(),
  546. }
  547. }
  548. // Type returns the AssetType
  549. func (ca *Cloud) Type() AssetType {
  550. return CloudAssetType
  551. }
  552. // Properties returns the AssetProperties
  553. func (ca *Cloud) Properties() *AssetProperties {
  554. return ca.properties
  555. }
  556. // SetProperties sets the Asset's properties
  557. func (ca *Cloud) SetProperties(props *AssetProperties) {
  558. ca.properties = props
  559. }
  560. // Labels returns the AssetLabels
  561. func (ca *Cloud) Labels() AssetLabels {
  562. return ca.labels
  563. }
  564. // SetLabels sets the Asset's labels
  565. func (ca *Cloud) SetLabels(labels AssetLabels) {
  566. ca.labels = labels
  567. }
  568. // Adjustment returns the Asset's adjustment value
  569. func (ca *Cloud) Adjustment() float64 {
  570. return ca.adjustment
  571. }
  572. // SetAdjustment sets the Asset's adjustment value
  573. func (ca *Cloud) SetAdjustment(adj float64) {
  574. ca.adjustment = adj
  575. }
  576. // TotalCost returns the Asset's total cost
  577. func (ca *Cloud) TotalCost() float64 {
  578. return ca.Cost + ca.adjustment + ca.Credit
  579. }
  580. // Start returns the Asset's precise start time within the window
  581. func (ca *Cloud) Start() time.Time {
  582. return ca.start
  583. }
  584. // End returns the Asset's precise end time within the window
  585. func (ca *Cloud) End() time.Time {
  586. return ca.end
  587. }
  588. // Minutes returns the number of Minutes the Asset ran
  589. func (ca *Cloud) Minutes() float64 {
  590. return ca.End().Sub(ca.Start()).Minutes()
  591. }
  592. // Window returns the window within which the Asset ran
  593. func (ca *Cloud) Window() Window {
  594. return ca.window
  595. }
  596. // ExpandWindow expands the Asset's window by the given window
  597. func (ca *Cloud) ExpandWindow(window Window) {
  598. ca.window = ca.window.Expand(window)
  599. }
  600. // SetStartEnd sets the Asset's Start and End fields
  601. func (ca *Cloud) SetStartEnd(start, end time.Time) {
  602. if ca.Window().Contains(start) {
  603. ca.start = start
  604. } else {
  605. log.Warningf("Cloud.SetStartEnd: start %s not in %s", start, ca.Window())
  606. }
  607. if ca.Window().Contains(end) {
  608. ca.end = end
  609. } else {
  610. log.Warningf("Cloud.SetStartEnd: end %s not in %s", end, ca.Window())
  611. }
  612. }
  613. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  614. // as much relevant information as possible (i.e. type, properties, labels).
  615. func (ca *Cloud) Add(a Asset) Asset {
  616. // Cloud + Cloud = Cloud
  617. if that, ok := a.(*Cloud); ok {
  618. this := ca.Clone().(*Cloud)
  619. this.add(that)
  620. return this
  621. }
  622. props := ca.Properties().Merge(a.Properties())
  623. labels := ca.Labels().Merge(a.Labels())
  624. start := ca.Start()
  625. if a.Start().Before(start) {
  626. start = a.Start()
  627. }
  628. end := ca.End()
  629. if a.End().After(end) {
  630. end = a.End()
  631. }
  632. window := ca.Window().Expand(a.Window())
  633. // Cloud + !Cloud = Any
  634. any := NewAsset(start, end, window)
  635. any.SetProperties(props)
  636. any.SetLabels(labels)
  637. any.adjustment = ca.Adjustment() + a.Adjustment()
  638. any.Cost = (ca.TotalCost() - ca.Adjustment() - ca.Credit) + (a.TotalCost() - a.Adjustment() - ca.Credit)
  639. return any
  640. }
  641. func (ca *Cloud) add(that *Cloud) {
  642. if ca == nil {
  643. ca = that
  644. return
  645. }
  646. props := ca.Properties().Merge(that.Properties())
  647. labels := ca.Labels().Merge(that.Labels())
  648. start := ca.Start()
  649. if that.Start().Before(start) {
  650. start = that.Start()
  651. }
  652. end := ca.End()
  653. if that.End().After(end) {
  654. end = that.End()
  655. }
  656. window := ca.Window().Expand(that.Window())
  657. ca.start = start
  658. ca.end = end
  659. ca.window = window
  660. ca.SetProperties(props)
  661. ca.SetLabels(labels)
  662. ca.adjustment += that.adjustment
  663. ca.Cost += that.Cost
  664. ca.Credit += that.Credit
  665. }
  666. // Clone returns a cloned instance of the Asset
  667. func (ca *Cloud) Clone() Asset {
  668. return &Cloud{
  669. labels: ca.labels.Clone(),
  670. properties: ca.properties.Clone(),
  671. start: ca.start,
  672. end: ca.end,
  673. window: ca.window.Clone(),
  674. adjustment: ca.adjustment,
  675. Cost: ca.Cost,
  676. Credit: ca.Credit,
  677. }
  678. }
  679. // Equal returns true if the given Asset precisely equals the Asset
  680. func (ca *Cloud) Equal(a Asset) bool {
  681. that, ok := a.(*Cloud)
  682. if !ok {
  683. return false
  684. }
  685. if !ca.Labels().Equal(that.Labels()) {
  686. return false
  687. }
  688. if !ca.Properties().Equal(that.Properties()) {
  689. return false
  690. }
  691. if !ca.start.Equal(that.start) {
  692. return false
  693. }
  694. if !ca.end.Equal(that.end) {
  695. return false
  696. }
  697. if !ca.window.Equal(that.window) {
  698. return false
  699. }
  700. if ca.adjustment != that.adjustment {
  701. return false
  702. }
  703. if ca.Cost != that.Cost {
  704. return false
  705. }
  706. if ca.Credit != that.Credit {
  707. return false
  708. }
  709. return true
  710. }
  711. // MarshalJSON implements json.Marshaler
  712. func (ca *Cloud) MarshalJSON() ([]byte, error) {
  713. buffer := bytes.NewBufferString("{")
  714. jsonEncodeString(buffer, "type", ca.Type().String(), ",")
  715. jsonEncode(buffer, "properties", ca.Properties(), ",")
  716. jsonEncode(buffer, "labels", ca.Labels(), ",")
  717. jsonEncode(buffer, "window", ca.Window(), ",")
  718. jsonEncodeString(buffer, "start", ca.Start().Format(time.RFC3339), ",")
  719. jsonEncodeString(buffer, "end", ca.End().Format(time.RFC3339), ",")
  720. jsonEncodeFloat64(buffer, "minutes", ca.Minutes(), ",")
  721. jsonEncodeFloat64(buffer, "adjustment", ca.Adjustment(), ",")
  722. jsonEncodeFloat64(buffer, "credit", ca.Credit, ",")
  723. jsonEncodeFloat64(buffer, "totalCost", ca.TotalCost(), "")
  724. buffer.WriteString("}")
  725. return buffer.Bytes(), nil
  726. }
  727. // String implements fmt.Stringer
  728. func (ca *Cloud) String() string {
  729. return toString(ca)
  730. }
  731. // ClusterManagement describes a provider's cluster management fee
  732. type ClusterManagement struct {
  733. labels AssetLabels
  734. properties *AssetProperties
  735. window Window
  736. Cost float64
  737. }
  738. // NewClusterManagement creates and returns a new ClusterManagement instance
  739. func NewClusterManagement(provider, cluster string, window Window) *ClusterManagement {
  740. properties := &AssetProperties{
  741. Category: ManagementCategory,
  742. Provider: ParseProvider(provider),
  743. Cluster: cluster,
  744. Service: KubernetesService,
  745. }
  746. return &ClusterManagement{
  747. labels: AssetLabels{},
  748. properties: properties,
  749. window: window.Clone(),
  750. }
  751. }
  752. // Type returns the Asset's type
  753. func (cm *ClusterManagement) Type() AssetType {
  754. return ClusterManagementAssetType
  755. }
  756. // Properties returns the Asset's properties
  757. func (cm *ClusterManagement) Properties() *AssetProperties {
  758. return cm.properties
  759. }
  760. // SetProperties sets the Asset's properties
  761. func (cm *ClusterManagement) SetProperties(props *AssetProperties) {
  762. cm.properties = props
  763. }
  764. // Labels returns the Asset's labels
  765. func (cm *ClusterManagement) Labels() AssetLabels {
  766. return cm.labels
  767. }
  768. // SetLabels sets the Asset's properties
  769. func (cm *ClusterManagement) SetLabels(props AssetLabels) {
  770. cm.labels = props
  771. }
  772. // Adjustment does not apply to ClusterManagement
  773. func (cm *ClusterManagement) Adjustment() float64 {
  774. return 0.0
  775. }
  776. // SetAdjustment does not apply to ClusterManagement
  777. func (cm *ClusterManagement) SetAdjustment(float64) {
  778. return
  779. }
  780. // TotalCost returns the Asset's total cost
  781. func (cm *ClusterManagement) TotalCost() float64 {
  782. return cm.Cost
  783. }
  784. // Start returns the Asset's precise start time within the window
  785. func (cm *ClusterManagement) Start() time.Time {
  786. return *cm.window.Start()
  787. }
  788. // End returns the Asset's precise end time within the window
  789. func (cm *ClusterManagement) End() time.Time {
  790. return *cm.window.End()
  791. }
  792. // Minutes returns the number of minutes the Asset ran
  793. func (cm *ClusterManagement) Minutes() float64 {
  794. return cm.Window().Minutes()
  795. }
  796. // Window return the Asset's window
  797. func (cm *ClusterManagement) Window() Window {
  798. return cm.window
  799. }
  800. // ExpandWindow expands the Asset's window by the given window
  801. func (cm *ClusterManagement) ExpandWindow(window Window) {
  802. cm.window = cm.window.Expand(window)
  803. }
  804. // SetStartEnd sets the Asset's Start and End fields (not applicable here)
  805. func (cm *ClusterManagement) SetStartEnd(start, end time.Time) {
  806. return
  807. }
  808. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  809. // as much relevant information as possible (i.e. type, properties, labels).
  810. func (cm *ClusterManagement) Add(a Asset) Asset {
  811. // ClusterManagement + ClusterManagement = ClusterManagement
  812. if that, ok := a.(*ClusterManagement); ok {
  813. this := cm.Clone().(*ClusterManagement)
  814. this.add(that)
  815. return this
  816. }
  817. props := cm.Properties().Merge(a.Properties())
  818. labels := cm.Labels().Merge(a.Labels())
  819. start := cm.Start()
  820. if a.Start().Before(start) {
  821. start = a.Start()
  822. }
  823. end := cm.End()
  824. if a.End().After(end) {
  825. end = a.End()
  826. }
  827. window := cm.Window().Expand(a.Window())
  828. // ClusterManagement + !ClusterManagement = Any
  829. any := NewAsset(start, end, window)
  830. any.SetProperties(props)
  831. any.SetLabels(labels)
  832. any.adjustment = cm.Adjustment() + a.Adjustment()
  833. any.Cost = (cm.TotalCost() - cm.Adjustment()) + (a.TotalCost() - a.Adjustment())
  834. return any
  835. }
  836. func (cm *ClusterManagement) add(that *ClusterManagement) {
  837. if cm == nil {
  838. cm = that
  839. return
  840. }
  841. props := cm.Properties().Merge(that.Properties())
  842. labels := cm.Labels().Merge(that.Labels())
  843. window := cm.Window().Expand(that.Window())
  844. cm.window = window
  845. cm.SetProperties(props)
  846. cm.SetLabels(labels)
  847. cm.Cost += that.Cost
  848. }
  849. // Clone returns a cloned instance of the Asset
  850. func (cm *ClusterManagement) Clone() Asset {
  851. return &ClusterManagement{
  852. labels: cm.labels.Clone(),
  853. properties: cm.properties.Clone(),
  854. window: cm.window.Clone(),
  855. Cost: cm.Cost,
  856. }
  857. }
  858. // Equal returns true if the given Asset exactly matches the Asset
  859. func (cm *ClusterManagement) Equal(a Asset) bool {
  860. that, ok := a.(*ClusterManagement)
  861. if !ok {
  862. return false
  863. }
  864. if !cm.Labels().Equal(that.Labels()) {
  865. return false
  866. }
  867. if !cm.Properties().Equal(that.Properties()) {
  868. return false
  869. }
  870. if !cm.window.Equal(that.window) {
  871. return false
  872. }
  873. if cm.Cost != that.Cost {
  874. return false
  875. }
  876. return true
  877. }
  878. // MarshalJSON implements json.Marshler
  879. func (cm *ClusterManagement) MarshalJSON() ([]byte, error) {
  880. buffer := bytes.NewBufferString("{")
  881. jsonEncodeString(buffer, "type", cm.Type().String(), ",")
  882. jsonEncode(buffer, "properties", cm.Properties(), ",")
  883. jsonEncode(buffer, "labels", cm.Labels(), ",")
  884. jsonEncode(buffer, "window", cm.Window(), ",")
  885. jsonEncodeString(buffer, "start", cm.Start().Format(time.RFC3339), ",")
  886. jsonEncodeString(buffer, "end", cm.End().Format(time.RFC3339), ",")
  887. jsonEncodeFloat64(buffer, "minutes", cm.Minutes(), ",")
  888. jsonEncodeFloat64(buffer, "totalCost", cm.TotalCost(), "")
  889. buffer.WriteString("}")
  890. return buffer.Bytes(), nil
  891. }
  892. // String implements fmt.Stringer
  893. func (cm *ClusterManagement) String() string {
  894. return toString(cm)
  895. }
  896. // Disk represents an in-cluster disk Asset
  897. type Disk struct {
  898. labels AssetLabels
  899. properties *AssetProperties
  900. start time.Time
  901. end time.Time
  902. window Window
  903. adjustment float64
  904. Cost float64
  905. ByteHours float64
  906. Local float64
  907. Breakdown *Breakdown
  908. }
  909. // NewDisk creates and returns a new Disk Asset
  910. func NewDisk(name, cluster, providerID string, start, end time.Time, window Window) *Disk {
  911. properties := &AssetProperties{
  912. Category: StorageCategory,
  913. Name: name,
  914. Cluster: cluster,
  915. ProviderID: providerID,
  916. Service: KubernetesService,
  917. }
  918. return &Disk{
  919. labels: AssetLabels{},
  920. properties: properties,
  921. start: start,
  922. end: end,
  923. window: window,
  924. Breakdown: &Breakdown{},
  925. }
  926. }
  927. // Type returns the AssetType of the Asset
  928. func (d *Disk) Type() AssetType {
  929. return DiskAssetType
  930. }
  931. // Properties returns the Asset's properties
  932. func (d *Disk) Properties() *AssetProperties {
  933. return d.properties
  934. }
  935. // SetProperties sets the Asset's properties
  936. func (d *Disk) SetProperties(props *AssetProperties) {
  937. d.properties = props
  938. }
  939. // Labels returns the Asset's labels
  940. func (d *Disk) Labels() AssetLabels {
  941. return d.labels
  942. }
  943. // SetLabels sets the Asset's labels
  944. func (d *Disk) SetLabels(labels AssetLabels) {
  945. d.labels = labels
  946. }
  947. // Adjustment returns the Asset's cost adjustment
  948. func (d *Disk) Adjustment() float64 {
  949. return d.adjustment
  950. }
  951. // SetAdjustment sets the Asset's cost adjustment
  952. func (d *Disk) SetAdjustment(adj float64) {
  953. d.adjustment = adj
  954. }
  955. // TotalCost returns the Asset's total cost
  956. func (d *Disk) TotalCost() float64 {
  957. return d.Cost + d.adjustment
  958. }
  959. // Start returns the precise start time of the Asset within the window
  960. func (d *Disk) Start() time.Time {
  961. return d.start
  962. }
  963. // End returns the precise start time of the Asset within the window
  964. func (d *Disk) End() time.Time {
  965. return d.end
  966. }
  967. // Minutes returns the number of minutes the Asset ran
  968. func (d *Disk) Minutes() float64 {
  969. diskMins := d.end.Sub(d.start).Minutes()
  970. windowMins := d.window.Minutes()
  971. if diskMins > windowMins {
  972. log.Warningf("Asset ETL: Disk.Minutes exceeds window: %.2f > %.2f", diskMins, windowMins)
  973. diskMins = windowMins
  974. }
  975. if diskMins < 0 {
  976. diskMins = 0
  977. }
  978. return diskMins
  979. }
  980. // Window returns the window within which the Asset
  981. func (d *Disk) Window() Window {
  982. return d.window
  983. }
  984. // ExpandWindow expands the Asset's window by the given window
  985. func (d *Disk) ExpandWindow(window Window) {
  986. d.window = d.window.Expand(window)
  987. }
  988. // SetStartEnd sets the Asset's Start and End fields
  989. func (d *Disk) SetStartEnd(start, end time.Time) {
  990. if d.Window().Contains(start) {
  991. d.start = start
  992. } else {
  993. log.Warningf("Disk.SetStartEnd: start %s not in %s", start, d.Window())
  994. }
  995. if d.Window().Contains(end) {
  996. d.end = end
  997. } else {
  998. log.Warningf("Disk.SetStartEnd: end %s not in %s", end, d.Window())
  999. }
  1000. }
  1001. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  1002. // as much relevant information as possible (i.e. type, properties, labels).
  1003. func (d *Disk) Add(a Asset) Asset {
  1004. // Disk + Disk = Disk
  1005. if that, ok := a.(*Disk); ok {
  1006. this := d.Clone().(*Disk)
  1007. this.add(that)
  1008. return this
  1009. }
  1010. props := d.Properties().Merge(a.Properties())
  1011. labels := d.Labels().Merge(a.Labels())
  1012. start := d.Start()
  1013. if a.Start().Before(start) {
  1014. start = a.Start()
  1015. }
  1016. end := d.End()
  1017. if a.End().After(end) {
  1018. end = a.End()
  1019. }
  1020. window := d.Window().Expand(a.Window())
  1021. // Disk + !Disk = Any
  1022. any := NewAsset(start, end, window)
  1023. any.SetProperties(props)
  1024. any.SetLabels(labels)
  1025. any.adjustment = d.Adjustment() + a.Adjustment()
  1026. any.Cost = (d.TotalCost() - d.Adjustment()) + (a.TotalCost() - a.Adjustment())
  1027. return any
  1028. }
  1029. func (d *Disk) add(that *Disk) {
  1030. if d == nil {
  1031. d = that
  1032. return
  1033. }
  1034. props := d.Properties().Merge(that.Properties())
  1035. labels := d.Labels().Merge(that.Labels())
  1036. d.SetProperties(props)
  1037. d.SetLabels(labels)
  1038. start := d.Start()
  1039. if that.Start().Before(start) {
  1040. start = that.Start()
  1041. }
  1042. end := d.End()
  1043. if that.End().After(end) {
  1044. end = that.End()
  1045. }
  1046. window := d.Window().Expand(that.Window())
  1047. d.start = start
  1048. d.end = end
  1049. d.window = window
  1050. totalCost := d.Cost + that.Cost
  1051. if totalCost > 0.0 {
  1052. d.Breakdown.Idle = (d.Breakdown.Idle*d.Cost + that.Breakdown.Idle*that.Cost) / totalCost
  1053. d.Breakdown.Other = (d.Breakdown.Other*d.Cost + that.Breakdown.Other*that.Cost) / totalCost
  1054. d.Breakdown.System = (d.Breakdown.System*d.Cost + that.Breakdown.System*that.Cost) / totalCost
  1055. d.Breakdown.User = (d.Breakdown.User*d.Cost + that.Breakdown.User*that.Cost) / totalCost
  1056. d.Local = (d.TotalCost()*d.Local + that.TotalCost()*that.Local) / (d.TotalCost() + that.TotalCost())
  1057. } else {
  1058. d.Local = (d.Local + that.Local) / 2.0
  1059. }
  1060. d.adjustment += that.adjustment
  1061. d.Cost += that.Cost
  1062. d.ByteHours += that.ByteHours
  1063. }
  1064. // Clone returns a cloned instance of the Asset
  1065. func (d *Disk) Clone() Asset {
  1066. return &Disk{
  1067. properties: d.properties.Clone(),
  1068. labels: d.labels.Clone(),
  1069. start: d.start,
  1070. end: d.end,
  1071. window: d.window.Clone(),
  1072. adjustment: d.adjustment,
  1073. Cost: d.Cost,
  1074. ByteHours: d.ByteHours,
  1075. Local: d.Local,
  1076. Breakdown: d.Breakdown.Clone(),
  1077. }
  1078. }
  1079. // Equal returns true if the two Assets match exactly
  1080. func (d *Disk) Equal(a Asset) bool {
  1081. that, ok := a.(*Disk)
  1082. if !ok {
  1083. return false
  1084. }
  1085. if !d.Labels().Equal(that.Labels()) {
  1086. return false
  1087. }
  1088. if !d.Properties().Equal(that.Properties()) {
  1089. return false
  1090. }
  1091. if !d.Start().Equal(that.Start()) {
  1092. return false
  1093. }
  1094. if !d.End().Equal(that.End()) {
  1095. return false
  1096. }
  1097. if !d.window.Equal(that.window) {
  1098. return false
  1099. }
  1100. if d.adjustment != that.adjustment {
  1101. return false
  1102. }
  1103. if d.Cost != that.Cost {
  1104. return false
  1105. }
  1106. if d.ByteHours != that.ByteHours {
  1107. return false
  1108. }
  1109. if d.Local != that.Local {
  1110. return false
  1111. }
  1112. if !d.Breakdown.Equal(that.Breakdown) {
  1113. return false
  1114. }
  1115. return true
  1116. }
  1117. // MarshalJSON implements the json.Marshaler interface
  1118. func (d *Disk) MarshalJSON() ([]byte, error) {
  1119. buffer := bytes.NewBufferString("{")
  1120. jsonEncodeString(buffer, "type", d.Type().String(), ",")
  1121. jsonEncode(buffer, "properties", d.Properties(), ",")
  1122. jsonEncode(buffer, "labels", d.Labels(), ",")
  1123. jsonEncode(buffer, "window", d.Window(), ",")
  1124. jsonEncodeString(buffer, "start", d.Start().Format(time.RFC3339), ",")
  1125. jsonEncodeString(buffer, "end", d.End().Format(time.RFC3339), ",")
  1126. jsonEncodeFloat64(buffer, "minutes", d.Minutes(), ",")
  1127. jsonEncodeFloat64(buffer, "byteHours", d.ByteHours, ",")
  1128. jsonEncodeFloat64(buffer, "bytes", d.Bytes(), ",")
  1129. jsonEncode(buffer, "breakdown", d.Breakdown, ",")
  1130. jsonEncodeFloat64(buffer, "adjustment", d.Adjustment(), ",")
  1131. jsonEncodeFloat64(buffer, "totalCost", d.TotalCost(), "")
  1132. buffer.WriteString("}")
  1133. return buffer.Bytes(), nil
  1134. }
  1135. // String implements fmt.Stringer
  1136. func (d *Disk) String() string {
  1137. return toString(d)
  1138. }
  1139. // Bytes returns the number of bytes belonging to the disk. This could be
  1140. // fractional because it's the number of byte*hours divided by the number of
  1141. // hours running; e.g. the sum of a 100GiB disk running for the first 10 hours
  1142. // and a 30GiB disk running for the last 20 hours of the same 24-hour window
  1143. // would produce:
  1144. // (100*10 + 30*20) / 24 = 66.667GiB
  1145. // However, any number of disks running for the full span of a window will
  1146. // report the actual number of bytes of the static disk; e.g. the above
  1147. // scenario for one entire 24-hour window:
  1148. // (100*24 + 30*24) / 24 = (100 + 30) = 130GiB
  1149. func (d *Disk) Bytes() float64 {
  1150. // [b*hr]*([min/hr]*[1/min]) = [b*hr]/[hr] = b
  1151. return d.ByteHours * (60.0 / d.Minutes())
  1152. }
  1153. // Breakdown describes a resource's use as a percentage of various usage types
  1154. type Breakdown struct {
  1155. Idle float64 `json:"idle"`
  1156. Other float64 `json:"other"`
  1157. System float64 `json:"system"`
  1158. User float64 `json:"user"`
  1159. }
  1160. // Clone returns a cloned instance of the Breakdown
  1161. func (b *Breakdown) Clone() *Breakdown {
  1162. if b == nil {
  1163. return nil
  1164. }
  1165. return &Breakdown{
  1166. Idle: b.Idle,
  1167. Other: b.Other,
  1168. System: b.System,
  1169. User: b.User,
  1170. }
  1171. }
  1172. // Equal returns true if the two Breakdowns are exact matches
  1173. func (b *Breakdown) Equal(that *Breakdown) bool {
  1174. if b == nil || that == nil {
  1175. return false
  1176. }
  1177. if b.Idle != that.Idle {
  1178. return false
  1179. }
  1180. if b.Other != that.Other {
  1181. return false
  1182. }
  1183. if b.System != that.System {
  1184. return false
  1185. }
  1186. if b.User != that.User {
  1187. return false
  1188. }
  1189. return true
  1190. }
  1191. // Network is an Asset representing a single node's network costs
  1192. type Network struct {
  1193. properties *AssetProperties
  1194. labels AssetLabels
  1195. start time.Time
  1196. end time.Time
  1197. window Window
  1198. adjustment float64
  1199. Cost float64
  1200. }
  1201. // NewNetwork creates and returns a new Network Asset
  1202. func NewNetwork(name, cluster, providerID string, start, end time.Time, window Window) *Network {
  1203. properties := &AssetProperties{
  1204. Category: NetworkCategory,
  1205. Name: name,
  1206. Cluster: cluster,
  1207. ProviderID: providerID,
  1208. Service: KubernetesService,
  1209. }
  1210. return &Network{
  1211. properties: properties,
  1212. labels: AssetLabels{},
  1213. start: start,
  1214. end: end,
  1215. window: window.Clone(),
  1216. }
  1217. }
  1218. // Type returns the AssetType of the Asset
  1219. func (n *Network) Type() AssetType {
  1220. return NetworkAssetType
  1221. }
  1222. // Properties returns the Asset's properties
  1223. func (n *Network) Properties() *AssetProperties {
  1224. return n.properties
  1225. }
  1226. // SetProperties sets the Asset's properties
  1227. func (n *Network) SetProperties(props *AssetProperties) {
  1228. n.properties = props
  1229. }
  1230. // Labels returns the Asset's labels
  1231. func (n *Network) Labels() AssetLabels {
  1232. return n.labels
  1233. }
  1234. // SetLabels sets the Asset's labels
  1235. func (n *Network) SetLabels(labels AssetLabels) {
  1236. n.labels = labels
  1237. }
  1238. // Adjustment returns the Asset's cost adjustment
  1239. func (n *Network) Adjustment() float64 {
  1240. return n.adjustment
  1241. }
  1242. // SetAdjustment sets the Asset's cost adjustment
  1243. func (n *Network) SetAdjustment(adj float64) {
  1244. n.adjustment = adj
  1245. }
  1246. // TotalCost returns the Asset's total cost
  1247. func (n *Network) TotalCost() float64 {
  1248. return n.Cost + n.adjustment
  1249. }
  1250. // Start returns the precise start time of the Asset within the window
  1251. func (n *Network) Start() time.Time {
  1252. return n.start
  1253. }
  1254. // End returns the precise end time of the Asset within the window
  1255. func (n *Network) End() time.Time {
  1256. return n.end
  1257. }
  1258. // Minutes returns the number of minutes the Asset ran within the window
  1259. func (n *Network) Minutes() float64 {
  1260. netMins := n.end.Sub(n.start).Minutes()
  1261. windowMins := n.window.Minutes()
  1262. if netMins > windowMins {
  1263. log.Warningf("Asset ETL: Network.Minutes exceeds window: %.2f > %.2f", netMins, windowMins)
  1264. netMins = windowMins
  1265. }
  1266. if netMins < 0 {
  1267. netMins = 0
  1268. }
  1269. return netMins
  1270. }
  1271. // Window returns the window within which the Asset ran
  1272. func (n *Network) Window() Window {
  1273. return n.window
  1274. }
  1275. // ExpandWindow expands the Asset's window by the given window
  1276. func (n *Network) ExpandWindow(window Window) {
  1277. n.window = n.window.Expand(window)
  1278. }
  1279. // SetStartEnd sets the Asset's Start and End fields
  1280. func (n *Network) SetStartEnd(start, end time.Time) {
  1281. if n.Window().Contains(start) {
  1282. n.start = start
  1283. } else {
  1284. log.Warningf("Disk.SetStartEnd: start %s not in %s", start, n.Window())
  1285. }
  1286. if n.Window().Contains(end) {
  1287. n.end = end
  1288. } else {
  1289. log.Warningf("Disk.SetStartEnd: end %s not in %s", end, n.Window())
  1290. }
  1291. }
  1292. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  1293. // as much relevant information as possible (i.e. type, properties, labels).
  1294. func (n *Network) Add(a Asset) Asset {
  1295. // Network + Network = Network
  1296. if that, ok := a.(*Network); ok {
  1297. this := n.Clone().(*Network)
  1298. this.add(that)
  1299. return this
  1300. }
  1301. props := n.Properties().Merge(a.Properties())
  1302. labels := n.Labels().Merge(a.Labels())
  1303. start := n.Start()
  1304. if a.Start().Before(start) {
  1305. start = a.Start()
  1306. }
  1307. end := n.End()
  1308. if a.End().After(end) {
  1309. end = a.End()
  1310. }
  1311. window := n.Window().Expand(a.Window())
  1312. // Network + !Network = Any
  1313. any := NewAsset(start, end, window)
  1314. any.SetProperties(props)
  1315. any.SetLabels(labels)
  1316. any.adjustment = n.Adjustment() + a.Adjustment()
  1317. any.Cost = (n.TotalCost() - n.Adjustment()) + (a.TotalCost() - a.Adjustment())
  1318. return any
  1319. }
  1320. func (n *Network) add(that *Network) {
  1321. if n == nil {
  1322. n = that
  1323. return
  1324. }
  1325. props := n.Properties().Merge(that.Properties())
  1326. labels := n.Labels().Merge(that.Labels())
  1327. n.SetProperties(props)
  1328. n.SetLabels(labels)
  1329. start := n.Start()
  1330. if that.Start().Before(start) {
  1331. start = that.Start()
  1332. }
  1333. end := n.End()
  1334. if that.End().After(end) {
  1335. end = that.End()
  1336. }
  1337. window := n.Window().Expand(that.Window())
  1338. n.start = start
  1339. n.end = end
  1340. n.window = window
  1341. n.Cost += that.Cost
  1342. n.adjustment += that.adjustment
  1343. }
  1344. // Clone returns a deep copy of the given Network
  1345. func (n *Network) Clone() Asset {
  1346. if n == nil {
  1347. return nil
  1348. }
  1349. return &Network{
  1350. properties: n.properties.Clone(),
  1351. labels: n.labels.Clone(),
  1352. start: n.start,
  1353. end: n.end,
  1354. window: n.window.Clone(),
  1355. adjustment: n.adjustment,
  1356. Cost: n.Cost,
  1357. }
  1358. }
  1359. // Equal returns true if the tow Assets match exactly
  1360. func (n *Network) Equal(a Asset) bool {
  1361. that, ok := a.(*Network)
  1362. if !ok {
  1363. return false
  1364. }
  1365. if !n.Labels().Equal(that.Labels()) {
  1366. return false
  1367. }
  1368. if !n.Properties().Equal(that.Properties()) {
  1369. return false
  1370. }
  1371. if !n.Start().Equal(that.Start()) {
  1372. return false
  1373. }
  1374. if !n.End().Equal(that.End()) {
  1375. return false
  1376. }
  1377. if !n.window.Equal(that.window) {
  1378. return false
  1379. }
  1380. if n.adjustment != that.adjustment {
  1381. return false
  1382. }
  1383. if n.Cost != that.Cost {
  1384. return false
  1385. }
  1386. return true
  1387. }
  1388. // MarshalJSON implements json.Marshal interface
  1389. func (n *Network) MarshalJSON() ([]byte, error) {
  1390. buffer := bytes.NewBufferString("{")
  1391. jsonEncodeString(buffer, "type", n.Type().String(), ",")
  1392. jsonEncode(buffer, "properties", n.Properties(), ",")
  1393. jsonEncode(buffer, "labels", n.Labels(), ",")
  1394. jsonEncode(buffer, "window", n.Window(), ",")
  1395. jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
  1396. jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
  1397. jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
  1398. jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
  1399. jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
  1400. buffer.WriteString("}")
  1401. return buffer.Bytes(), nil
  1402. }
  1403. // String implements fmt.Stringer
  1404. func (n *Network) String() string {
  1405. return toString(n)
  1406. }
  1407. // Node is an Asset representing a single node in a cluster
  1408. type Node struct {
  1409. properties *AssetProperties
  1410. labels AssetLabels
  1411. start time.Time
  1412. end time.Time
  1413. window Window
  1414. adjustment float64
  1415. NodeType string
  1416. CPUCoreHours float64
  1417. RAMByteHours float64
  1418. GPUHours float64
  1419. CPUBreakdown *Breakdown
  1420. RAMBreakdown *Breakdown
  1421. CPUCost float64
  1422. GPUCost float64
  1423. GPUCount float64
  1424. RAMCost float64
  1425. Discount float64
  1426. Preemptible float64
  1427. }
  1428. // NewNode creates and returns a new Node Asset
  1429. func NewNode(name, cluster, providerID string, start, end time.Time, window Window) *Node {
  1430. properties := &AssetProperties{
  1431. Category: ComputeCategory,
  1432. Name: name,
  1433. Cluster: cluster,
  1434. ProviderID: providerID,
  1435. Service: KubernetesService,
  1436. }
  1437. return &Node{
  1438. properties: properties,
  1439. labels: AssetLabels{},
  1440. start: start,
  1441. end: end,
  1442. window: window.Clone(),
  1443. CPUBreakdown: &Breakdown{},
  1444. RAMBreakdown: &Breakdown{},
  1445. }
  1446. }
  1447. // Type returns the AssetType of the Asset
  1448. func (n *Node) Type() AssetType {
  1449. return NodeAssetType
  1450. }
  1451. // Properties returns the Asset's properties
  1452. func (n *Node) Properties() *AssetProperties {
  1453. return n.properties
  1454. }
  1455. // SetProperties sets the Asset's properties
  1456. func (n *Node) SetProperties(props *AssetProperties) {
  1457. n.properties = props
  1458. }
  1459. // Labels returns the Asset's labels
  1460. func (n *Node) Labels() AssetLabels {
  1461. return n.labels
  1462. }
  1463. // SetLabels sets the Asset's labels
  1464. func (n *Node) SetLabels(labels AssetLabels) {
  1465. n.labels = labels
  1466. }
  1467. // Adjustment returns the Asset's cost adjustment
  1468. func (n *Node) Adjustment() float64 {
  1469. return n.adjustment
  1470. }
  1471. // SetAdjustment sets the Asset's cost adjustment
  1472. func (n *Node) SetAdjustment(adj float64) {
  1473. n.adjustment = adj
  1474. }
  1475. // TotalCost returns the Asset's total cost
  1476. func (n *Node) TotalCost() float64 {
  1477. return ((n.CPUCost + n.RAMCost) * (1.0 - n.Discount)) + n.GPUCost + n.adjustment
  1478. }
  1479. // Start returns the precise start time of the Asset within the window
  1480. func (n *Node) Start() time.Time {
  1481. return n.start
  1482. }
  1483. // End returns the precise end time of the Asset within the window
  1484. func (n *Node) End() time.Time {
  1485. return n.end
  1486. }
  1487. // Minutes returns the number of minutes the Asset ran within the window
  1488. func (n *Node) Minutes() float64 {
  1489. nodeMins := n.end.Sub(n.start).Minutes()
  1490. windowMins := n.window.Minutes()
  1491. if nodeMins > windowMins {
  1492. log.Warningf("Asset ETL: Node.Minutes exceeds window: %.2f > %.2f", nodeMins, windowMins)
  1493. nodeMins = windowMins
  1494. }
  1495. if nodeMins < 0 {
  1496. nodeMins = 0
  1497. }
  1498. return nodeMins
  1499. }
  1500. // Window returns the window within which the Asset ran
  1501. func (n *Node) Window() Window {
  1502. return n.window
  1503. }
  1504. // ExpandWindow expands the Asset's window by the given window
  1505. func (n *Node) ExpandWindow(window Window) {
  1506. n.window = n.window.Expand(window)
  1507. }
  1508. // SetStartEnd sets the Asset's Start and End fields
  1509. func (n *Node) SetStartEnd(start, end time.Time) {
  1510. if n.Window().Contains(start) {
  1511. n.start = start
  1512. } else {
  1513. log.Warningf("Disk.SetStartEnd: start %s not in %s", start, n.Window())
  1514. }
  1515. if n.Window().Contains(end) {
  1516. n.end = end
  1517. } else {
  1518. log.Warningf("Disk.SetStartEnd: end %s not in %s", end, n.Window())
  1519. }
  1520. }
  1521. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  1522. // as much relevant information as possible (i.e. type, properties, labels).
  1523. func (n *Node) Add(a Asset) Asset {
  1524. // Node + Node = Node
  1525. if that, ok := a.(*Node); ok {
  1526. this := n.Clone().(*Node)
  1527. this.add(that)
  1528. return this
  1529. }
  1530. props := n.Properties().Merge(a.Properties())
  1531. labels := n.Labels().Merge(a.Labels())
  1532. start := n.Start()
  1533. if a.Start().Before(start) {
  1534. start = a.Start()
  1535. }
  1536. end := n.End()
  1537. if a.End().After(end) {
  1538. end = a.End()
  1539. }
  1540. window := n.Window().Expand(a.Window())
  1541. // Node + !Node = Any
  1542. any := NewAsset(start, end, window)
  1543. any.SetProperties(props)
  1544. any.SetLabels(labels)
  1545. any.adjustment = n.Adjustment() + a.Adjustment()
  1546. any.Cost = (n.TotalCost() - n.Adjustment()) + (a.TotalCost() - a.Adjustment())
  1547. return any
  1548. }
  1549. func (n *Node) add(that *Node) {
  1550. if n == nil {
  1551. n = that
  1552. return
  1553. }
  1554. props := n.Properties().Merge(that.Properties())
  1555. labels := n.Labels().Merge(that.Labels())
  1556. n.SetProperties(props)
  1557. n.SetLabels(labels)
  1558. if n.NodeType != that.NodeType {
  1559. n.NodeType = ""
  1560. }
  1561. start := n.Start()
  1562. if that.Start().Before(start) {
  1563. start = that.Start()
  1564. }
  1565. end := n.End()
  1566. if that.End().After(end) {
  1567. end = that.End()
  1568. }
  1569. window := n.Window().Expand(that.Window())
  1570. n.start = start
  1571. n.end = end
  1572. n.window = window
  1573. // Order of operations for node costs is:
  1574. // Discount(CPU + RAM) + GPU + Adjustment
  1575. // Combining discounts, then involves weighting each discount by each
  1576. // respective (CPU + RAM) cost. Combining preemptible, on the other
  1577. // hand, is done with all three (but not Adjustment, which can change
  1578. // without triggering a re-computation of Preemtible).
  1579. disc := (n.CPUCost+n.RAMCost)*(1.0-n.Discount) + (that.CPUCost+that.RAMCost)*(1.0-that.Discount)
  1580. nonDisc := (n.CPUCost + n.RAMCost) + (that.CPUCost + that.RAMCost)
  1581. if nonDisc > 0 {
  1582. n.Discount = 1.0 - (disc / nonDisc)
  1583. } else {
  1584. n.Discount = (n.Discount + that.Discount) / 2.0
  1585. }
  1586. nNoAdj := n.TotalCost() - n.Adjustment()
  1587. thatNoAdj := that.TotalCost() - that.Adjustment()
  1588. if (nNoAdj + thatNoAdj) > 0 {
  1589. n.Preemptible = (nNoAdj*n.Preemptible + thatNoAdj*that.Preemptible) / (nNoAdj + thatNoAdj)
  1590. } else {
  1591. n.Preemptible = (n.Preemptible + that.Preemptible) / 2.0
  1592. }
  1593. totalCPUCost := n.CPUCost + that.CPUCost
  1594. if totalCPUCost > 0.0 {
  1595. n.CPUBreakdown.Idle = (n.CPUBreakdown.Idle*n.CPUCost + that.CPUBreakdown.Idle*that.CPUCost) / totalCPUCost
  1596. n.CPUBreakdown.Other = (n.CPUBreakdown.Other*n.CPUCost + that.CPUBreakdown.Other*that.CPUCost) / totalCPUCost
  1597. n.CPUBreakdown.System = (n.CPUBreakdown.System*n.CPUCost + that.CPUBreakdown.System*that.CPUCost) / totalCPUCost
  1598. n.CPUBreakdown.User = (n.CPUBreakdown.User*n.CPUCost + that.CPUBreakdown.User*that.CPUCost) / totalCPUCost
  1599. }
  1600. totalRAMCost := n.RAMCost + that.RAMCost
  1601. if totalRAMCost > 0.0 {
  1602. n.RAMBreakdown.Idle = (n.RAMBreakdown.Idle*n.RAMCost + that.RAMBreakdown.Idle*that.RAMCost) / totalRAMCost
  1603. n.RAMBreakdown.Other = (n.RAMBreakdown.Other*n.RAMCost + that.RAMBreakdown.Other*that.RAMCost) / totalRAMCost
  1604. n.RAMBreakdown.System = (n.RAMBreakdown.System*n.RAMCost + that.RAMBreakdown.System*that.RAMCost) / totalRAMCost
  1605. n.RAMBreakdown.User = (n.RAMBreakdown.User*n.RAMCost + that.RAMBreakdown.User*that.RAMCost) / totalRAMCost
  1606. }
  1607. n.CPUCoreHours += that.CPUCoreHours
  1608. n.RAMByteHours += that.RAMByteHours
  1609. n.GPUHours += that.GPUHours
  1610. n.CPUCost += that.CPUCost
  1611. n.GPUCost += that.GPUCost
  1612. n.RAMCost += that.RAMCost
  1613. n.adjustment += that.adjustment
  1614. }
  1615. // Clone returns a deep copy of the given Node
  1616. func (n *Node) Clone() Asset {
  1617. if n == nil {
  1618. return nil
  1619. }
  1620. return &Node{
  1621. properties: n.properties.Clone(),
  1622. labels: n.labels.Clone(),
  1623. start: n.start,
  1624. end: n.end,
  1625. window: n.window.Clone(),
  1626. adjustment: n.adjustment,
  1627. NodeType: n.NodeType,
  1628. CPUCoreHours: n.CPUCoreHours,
  1629. RAMByteHours: n.RAMByteHours,
  1630. GPUHours: n.GPUHours,
  1631. CPUBreakdown: n.CPUBreakdown.Clone(),
  1632. RAMBreakdown: n.RAMBreakdown.Clone(),
  1633. CPUCost: n.CPUCost,
  1634. GPUCost: n.GPUCost,
  1635. GPUCount: n.GPUCount,
  1636. RAMCost: n.RAMCost,
  1637. Preemptible: n.Preemptible,
  1638. Discount: n.Discount,
  1639. }
  1640. }
  1641. // Equal returns true if the tow Assets match exactly
  1642. func (n *Node) Equal(a Asset) bool {
  1643. that, ok := a.(*Node)
  1644. if !ok {
  1645. return false
  1646. }
  1647. if !n.Labels().Equal(that.Labels()) {
  1648. return false
  1649. }
  1650. if !n.Properties().Equal(that.Properties()) {
  1651. return false
  1652. }
  1653. if !n.Start().Equal(that.Start()) {
  1654. return false
  1655. }
  1656. if !n.End().Equal(that.End()) {
  1657. return false
  1658. }
  1659. if !n.window.Equal(that.window) {
  1660. return false
  1661. }
  1662. if n.adjustment != that.adjustment {
  1663. return false
  1664. }
  1665. if n.NodeType != that.NodeType {
  1666. return false
  1667. }
  1668. if n.CPUCoreHours != that.CPUCoreHours {
  1669. return false
  1670. }
  1671. if n.RAMByteHours != that.RAMByteHours {
  1672. return false
  1673. }
  1674. if n.GPUHours != that.GPUHours {
  1675. return false
  1676. }
  1677. if !n.CPUBreakdown.Equal(that.CPUBreakdown) {
  1678. return false
  1679. }
  1680. if !n.RAMBreakdown.Equal(that.RAMBreakdown) {
  1681. return false
  1682. }
  1683. if n.CPUCost != that.CPUCost {
  1684. return false
  1685. }
  1686. if n.GPUCost != that.GPUCost {
  1687. return false
  1688. }
  1689. if n.RAMCost != that.RAMCost {
  1690. return false
  1691. }
  1692. if n.Discount != that.Discount {
  1693. return false
  1694. }
  1695. if n.Preemptible != that.Preemptible {
  1696. return false
  1697. }
  1698. return true
  1699. }
  1700. // MarshalJSON implements json.Marshal interface
  1701. func (n *Node) MarshalJSON() ([]byte, error) {
  1702. buffer := bytes.NewBufferString("{")
  1703. jsonEncodeString(buffer, "type", n.Type().String(), ",")
  1704. jsonEncode(buffer, "properties", n.Properties(), ",")
  1705. jsonEncode(buffer, "labels", n.Labels(), ",")
  1706. jsonEncode(buffer, "window", n.Window(), ",")
  1707. jsonEncodeString(buffer, "start", n.Start().Format(time.RFC3339), ",")
  1708. jsonEncodeString(buffer, "end", n.End().Format(time.RFC3339), ",")
  1709. jsonEncodeFloat64(buffer, "minutes", n.Minutes(), ",")
  1710. jsonEncodeString(buffer, "nodeType", n.NodeType, ",")
  1711. jsonEncodeFloat64(buffer, "cpuCores", n.CPUCores(), ",")
  1712. jsonEncodeFloat64(buffer, "ramBytes", n.RAMBytes(), ",")
  1713. jsonEncodeFloat64(buffer, "cpuCoreHours", n.CPUCoreHours, ",")
  1714. jsonEncodeFloat64(buffer, "ramByteHours", n.RAMByteHours, ",")
  1715. jsonEncodeFloat64(buffer, "GPUHours", n.GPUHours, ",")
  1716. jsonEncode(buffer, "cpuBreakdown", n.CPUBreakdown, ",")
  1717. jsonEncode(buffer, "ramBreakdown", n.RAMBreakdown, ",")
  1718. jsonEncodeFloat64(buffer, "preemptible", n.Preemptible, ",")
  1719. jsonEncodeFloat64(buffer, "discount", n.Discount, ",")
  1720. jsonEncodeFloat64(buffer, "cpuCost", n.CPUCost, ",")
  1721. jsonEncodeFloat64(buffer, "gpuCost", n.GPUCost, ",")
  1722. jsonEncodeFloat64(buffer, "gpuCount", n.GPUs(), ",")
  1723. jsonEncodeFloat64(buffer, "ramCost", n.RAMCost, ",")
  1724. jsonEncodeFloat64(buffer, "adjustment", n.Adjustment(), ",")
  1725. jsonEncodeFloat64(buffer, "totalCost", n.TotalCost(), "")
  1726. buffer.WriteString("}")
  1727. return buffer.Bytes(), nil
  1728. }
  1729. // String implements fmt.Stringer
  1730. func (n *Node) String() string {
  1731. return toString(n)
  1732. }
  1733. // IsPreemptible returns true if the node is 100% preemptible. It's possible
  1734. // to be "partially preemptible" by adding a preemptible node with a
  1735. // non-preemptible node.
  1736. func (n *Node) IsPreemptible() bool {
  1737. return n.Preemptible == 1.0
  1738. }
  1739. // CPUCores returns the number of cores belonging to the node. This could be
  1740. // fractional because it's the number of core*hours divided by the number of
  1741. // hours running; e.g. the sum of a 4-core node running for the first 10 hours
  1742. // and a 3-core node running for the last 20 hours of the same 24-hour window
  1743. // would produce:
  1744. // (4*10 + 3*20) / 24 = 4.167 cores
  1745. // However, any number of cores running for the full span of a window will
  1746. // report the actual number of cores of the static node; e.g. the above
  1747. // scenario for one entire 24-hour window:
  1748. // (4*24 + 3*24) / 24 = (4 + 3) = 7 cores
  1749. func (n *Node) CPUCores() float64 {
  1750. // [core*hr]*([min/hr]*[1/min]) = [core*hr]/[hr] = core
  1751. return n.CPUCoreHours * (60.0 / n.Minutes())
  1752. }
  1753. // RAMBytes returns the amount of RAM belonging to the node. This could be
  1754. // fractional because it's the number of byte*hours divided by the number of
  1755. // hours running; e.g. the sum of a 12GiB-RAM node running for the first 10 hours
  1756. // and a 16GiB-RAM node running for the last 20 hours of the same 24-hour window
  1757. // would produce:
  1758. // (12*10 + 16*20) / 24 = 18.333GiB RAM
  1759. // However, any number of bytes running for the full span of a window will
  1760. // report the actual number of bytes of the static node; e.g. the above
  1761. // scenario for one entire 24-hour window:
  1762. // (12*24 + 16*24) / 24 = (12 + 16) = 28GiB RAM
  1763. func (n *Node) RAMBytes() float64 {
  1764. // [b*hr]*([min/hr]*[1/min]) = [b*hr]/[hr] = b
  1765. return n.RAMByteHours * (60.0 / n.Minutes())
  1766. }
  1767. // GPUs returns the amount of GPUs belonging to the node. This could be
  1768. // fractional because it's the number of gpu*hours divided by the number of
  1769. // hours running; e.g. the sum of a 2 gpu node running for the first 10 hours
  1770. // and a 1 gpu node running for the last 20 hours of the same 24-hour window
  1771. // would produce:
  1772. // (2*10 + 1*20) / 24 = 1.667 GPUs
  1773. // However, any number of GPUs running for the full span of a window will
  1774. // report the actual number of GPUs of the static node; e.g. the above
  1775. // scenario for one entire 24-hour window:
  1776. // (2*24 + 1*24) / 24 = (2 + 1) = 3 GPUs
  1777. func (n *Node) GPUs() float64 {
  1778. // [b*hr]*([min/hr]*[1/min]) = [b*hr]/[hr] = b
  1779. return n.GPUHours * (60.0 / n.Minutes())
  1780. }
  1781. // LoadBalancer is an Asset representing a single load balancer in a cluster
  1782. // TODO: add GB of ingress processed, numForwardingRules once we start recording those to prometheus metric
  1783. type LoadBalancer struct {
  1784. properties *AssetProperties
  1785. labels AssetLabels
  1786. start time.Time
  1787. end time.Time
  1788. window Window
  1789. adjustment float64
  1790. Cost float64
  1791. }
  1792. // NewLoadBalancer instantiates and returns a new LoadBalancer
  1793. func NewLoadBalancer(name, cluster, providerID string, start, end time.Time, window Window) *LoadBalancer {
  1794. properties := &AssetProperties{
  1795. Category: NetworkCategory,
  1796. Name: name,
  1797. Cluster: cluster,
  1798. ProviderID: providerID,
  1799. Service: KubernetesService,
  1800. }
  1801. return &LoadBalancer{
  1802. properties: properties,
  1803. labels: AssetLabels{},
  1804. start: start,
  1805. end: end,
  1806. window: window,
  1807. }
  1808. }
  1809. // Type returns the AssetType of the Asset
  1810. func (lb *LoadBalancer) Type() AssetType {
  1811. return LoadBalancerAssetType
  1812. }
  1813. // Properties returns the Asset's properties
  1814. func (lb *LoadBalancer) Properties() *AssetProperties {
  1815. return lb.properties
  1816. }
  1817. // SetProperties sets the Asset's properties
  1818. func (lb *LoadBalancer) SetProperties(props *AssetProperties) {
  1819. lb.properties = props
  1820. }
  1821. // Labels returns the Asset's labels
  1822. func (lb *LoadBalancer) Labels() AssetLabels {
  1823. return lb.labels
  1824. }
  1825. // SetLabels sets the Asset's labels
  1826. func (lb *LoadBalancer) SetLabels(labels AssetLabels) {
  1827. lb.labels = labels
  1828. }
  1829. // Adjustment returns the Asset's cost adjustment
  1830. func (lb *LoadBalancer) Adjustment() float64 {
  1831. return lb.adjustment
  1832. }
  1833. // SetAdjustment sets the Asset's cost adjustment
  1834. func (lb *LoadBalancer) SetAdjustment(adj float64) {
  1835. lb.adjustment = adj
  1836. }
  1837. // TotalCost returns the total cost of the Asset
  1838. func (lb *LoadBalancer) TotalCost() float64 {
  1839. return lb.Cost + lb.adjustment
  1840. }
  1841. // Start returns the preceise start point of the Asset within the window
  1842. func (lb *LoadBalancer) Start() time.Time {
  1843. return lb.start
  1844. }
  1845. // End returns the preceise end point of the Asset within the window
  1846. func (lb *LoadBalancer) End() time.Time {
  1847. return lb.end
  1848. }
  1849. // Minutes returns the number of minutes the Asset ran within the window
  1850. func (lb *LoadBalancer) Minutes() float64 {
  1851. return lb.end.Sub(lb.start).Minutes()
  1852. }
  1853. // Window returns the window within which the Asset ran
  1854. func (lb *LoadBalancer) Window() Window {
  1855. return lb.window
  1856. }
  1857. // ExpandWindow expands the Asset's window by the given window
  1858. func (lb *LoadBalancer) ExpandWindow(w Window) {
  1859. lb.window = lb.window.Expand(w)
  1860. }
  1861. // SetStartEnd sets the Asset's Start and End fields
  1862. func (lb *LoadBalancer) SetStartEnd(start, end time.Time) {
  1863. if lb.Window().Contains(start) {
  1864. lb.start = start
  1865. } else {
  1866. log.Warningf("Disk.SetStartEnd: start %s not in %s", start, lb.Window())
  1867. }
  1868. if lb.Window().Contains(end) {
  1869. lb.end = end
  1870. } else {
  1871. log.Warningf("Disk.SetStartEnd: end %s not in %s", end, lb.Window())
  1872. }
  1873. }
  1874. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  1875. // as much relevant information as possible (i.e. type, properties, labels).
  1876. func (lb *LoadBalancer) Add(a Asset) Asset {
  1877. // LoadBalancer + LoadBalancer = LoadBalancer
  1878. if that, ok := a.(*LoadBalancer); ok {
  1879. this := lb.Clone().(*LoadBalancer)
  1880. this.add(that)
  1881. return this
  1882. }
  1883. props := lb.Properties().Merge(a.Properties())
  1884. labels := lb.Labels().Merge(a.Labels())
  1885. start := lb.Start()
  1886. if a.Start().Before(start) {
  1887. start = a.Start()
  1888. }
  1889. end := lb.End()
  1890. if a.End().After(end) {
  1891. end = a.End()
  1892. }
  1893. window := lb.Window().Expand(a.Window())
  1894. // LoadBalancer + !LoadBalancer = Any
  1895. any := NewAsset(start, end, window)
  1896. any.SetProperties(props)
  1897. any.SetLabels(labels)
  1898. any.adjustment = lb.Adjustment() + a.Adjustment()
  1899. any.Cost = (lb.TotalCost() - lb.Adjustment()) + (a.TotalCost() - a.Adjustment())
  1900. return any
  1901. }
  1902. func (lb *LoadBalancer) add(that *LoadBalancer) {
  1903. if lb == nil {
  1904. lb = that
  1905. return
  1906. }
  1907. props := lb.Properties().Merge(that.Properties())
  1908. labels := lb.Labels().Merge(that.Labels())
  1909. lb.SetProperties(props)
  1910. lb.SetLabels(labels)
  1911. start := lb.Start()
  1912. if that.Start().Before(start) {
  1913. start = that.Start()
  1914. }
  1915. end := lb.End()
  1916. if that.End().After(end) {
  1917. end = that.End()
  1918. }
  1919. window := lb.Window().Expand(that.Window())
  1920. lb.start = start
  1921. lb.end = end
  1922. lb.window = window
  1923. lb.Cost += that.Cost
  1924. lb.adjustment += that.adjustment
  1925. }
  1926. // Clone returns a cloned instance of the given Asset
  1927. func (lb *LoadBalancer) Clone() Asset {
  1928. return &LoadBalancer{
  1929. properties: lb.properties.Clone(),
  1930. labels: lb.labels.Clone(),
  1931. start: lb.start,
  1932. end: lb.end,
  1933. window: lb.window.Clone(),
  1934. adjustment: lb.adjustment,
  1935. Cost: lb.Cost,
  1936. }
  1937. }
  1938. // Equal returns true if the tow Assets match precisely
  1939. func (lb *LoadBalancer) Equal(a Asset) bool {
  1940. that, ok := a.(*LoadBalancer)
  1941. if !ok {
  1942. return false
  1943. }
  1944. if !lb.Labels().Equal(that.Labels()) {
  1945. return false
  1946. }
  1947. if !lb.Properties().Equal(that.Properties()) {
  1948. return false
  1949. }
  1950. if !lb.Start().Equal(that.Start()) {
  1951. return false
  1952. }
  1953. if !lb.End().Equal(that.End()) {
  1954. return false
  1955. }
  1956. if !lb.window.Equal(that.window) {
  1957. return false
  1958. }
  1959. if lb.adjustment != that.adjustment {
  1960. return false
  1961. }
  1962. if lb.Cost != that.Cost {
  1963. return false
  1964. }
  1965. return true
  1966. }
  1967. // MarshalJSON implements json.Marshal
  1968. func (lb *LoadBalancer) MarshalJSON() ([]byte, error) {
  1969. buffer := bytes.NewBufferString("{")
  1970. jsonEncodeString(buffer, "type", lb.Type().String(), ",")
  1971. jsonEncode(buffer, "properties", lb.Properties(), ",")
  1972. jsonEncode(buffer, "labels", lb.Labels(), ",")
  1973. jsonEncode(buffer, "window", lb.Window(), ",")
  1974. jsonEncodeString(buffer, "start", lb.Start().Format(time.RFC3339), ",")
  1975. jsonEncodeString(buffer, "end", lb.End().Format(time.RFC3339), ",")
  1976. jsonEncodeFloat64(buffer, "minutes", lb.Minutes(), ",")
  1977. jsonEncodeFloat64(buffer, "adjustment", lb.Adjustment(), ",")
  1978. jsonEncodeFloat64(buffer, "totalCost", lb.TotalCost(), "")
  1979. buffer.WriteString("}")
  1980. return buffer.Bytes(), nil
  1981. }
  1982. // String implements fmt.Stringer
  1983. func (lb *LoadBalancer) String() string {
  1984. return toString(lb)
  1985. }
  1986. // SharedAsset is an Asset representing a shared cost
  1987. type SharedAsset struct {
  1988. properties *AssetProperties
  1989. labels AssetLabels
  1990. window Window
  1991. Cost float64
  1992. }
  1993. // NewSharedAsset creates and returns a new SharedAsset
  1994. func NewSharedAsset(name string, window Window) *SharedAsset {
  1995. properties := &AssetProperties{
  1996. Name: name,
  1997. Category: SharedCategory,
  1998. Service: OtherCategory,
  1999. }
  2000. return &SharedAsset{
  2001. properties: properties,
  2002. labels: AssetLabels{},
  2003. window: window.Clone(),
  2004. }
  2005. }
  2006. // Type returns the AssetType of the Asset
  2007. func (sa *SharedAsset) Type() AssetType {
  2008. return SharedAssetType
  2009. }
  2010. // Properties returns the Asset's properties
  2011. func (sa *SharedAsset) Properties() *AssetProperties {
  2012. return sa.properties
  2013. }
  2014. // SetProperties sets the Asset's properties
  2015. func (sa *SharedAsset) SetProperties(props *AssetProperties) {
  2016. sa.properties = props
  2017. }
  2018. // Labels returns the Asset's labels
  2019. func (sa *SharedAsset) Labels() AssetLabels {
  2020. return sa.labels
  2021. }
  2022. // SetLabels sets the Asset's labels
  2023. func (sa *SharedAsset) SetLabels(labels AssetLabels) {
  2024. sa.labels = labels
  2025. }
  2026. // Adjustment is not relevant to SharedAsset, but required to implement Asset
  2027. func (sa *SharedAsset) Adjustment() float64 {
  2028. return 0.0
  2029. }
  2030. // SetAdjustment is not relevant to SharedAsset, but required to implement Asset
  2031. func (sa *SharedAsset) SetAdjustment(float64) {
  2032. return
  2033. }
  2034. // TotalCost returns the Asset's total cost
  2035. func (sa *SharedAsset) TotalCost() float64 {
  2036. return sa.Cost
  2037. }
  2038. // Start returns the start time of the Asset
  2039. func (sa *SharedAsset) Start() time.Time {
  2040. return *sa.window.start
  2041. }
  2042. // End returns the end time of the Asset
  2043. func (sa *SharedAsset) End() time.Time {
  2044. return *sa.window.end
  2045. }
  2046. // Minutes returns the number of minutes the SharedAsset ran within the window
  2047. func (sa *SharedAsset) Minutes() float64 {
  2048. return sa.window.Minutes()
  2049. }
  2050. // Window returns the window within the SharedAsset ran
  2051. func (sa *SharedAsset) Window() Window {
  2052. return sa.window
  2053. }
  2054. // ExpandWindow expands the Asset's window
  2055. func (sa *SharedAsset) ExpandWindow(w Window) {
  2056. sa.window = sa.window.Expand(w)
  2057. }
  2058. // SetStartEnd sets the Asset's Start and End fields (not applicable here)
  2059. func (sa *SharedAsset) SetStartEnd(start, end time.Time) {
  2060. return
  2061. }
  2062. // Add sums the Asset with the given Asset to produce a new Asset, maintaining
  2063. // as much relevant information as possible (i.e. type, properties, labels).
  2064. func (sa *SharedAsset) Add(a Asset) Asset {
  2065. // SharedAsset + SharedAsset = SharedAsset
  2066. if that, ok := a.(*SharedAsset); ok {
  2067. this := sa.Clone().(*SharedAsset)
  2068. this.add(that)
  2069. return this
  2070. }
  2071. props := sa.Properties().Merge(a.Properties())
  2072. labels := sa.Labels().Merge(a.Labels())
  2073. start := sa.Start()
  2074. if a.Start().Before(start) {
  2075. start = a.Start()
  2076. }
  2077. end := sa.End()
  2078. if a.End().After(end) {
  2079. end = a.End()
  2080. }
  2081. window := sa.Window().Expand(a.Window())
  2082. // SharedAsset + !SharedAsset = Any
  2083. any := NewAsset(start, end, window)
  2084. any.SetProperties(props)
  2085. any.SetLabels(labels)
  2086. any.adjustment = sa.Adjustment() + a.Adjustment()
  2087. any.Cost = (sa.TotalCost() - sa.Adjustment()) + (a.TotalCost() - a.Adjustment())
  2088. return any
  2089. }
  2090. func (sa *SharedAsset) add(that *SharedAsset) {
  2091. if sa == nil {
  2092. sa = that
  2093. return
  2094. }
  2095. props := sa.Properties().Merge(that.Properties())
  2096. labels := sa.Labels().Merge(that.Labels())
  2097. sa.SetProperties(props)
  2098. sa.SetLabels(labels)
  2099. window := sa.Window().Expand(that.Window())
  2100. sa.window = window
  2101. sa.Cost += that.Cost
  2102. }
  2103. // Clone returns a deep copy of the given SharedAsset
  2104. func (sa *SharedAsset) Clone() Asset {
  2105. if sa == nil {
  2106. return nil
  2107. }
  2108. return &SharedAsset{
  2109. properties: sa.properties.Clone(),
  2110. labels: sa.labels.Clone(),
  2111. window: sa.window.Clone(),
  2112. Cost: sa.Cost,
  2113. }
  2114. }
  2115. // Equal returns true if the two Assets are exact matches
  2116. func (sa *SharedAsset) Equal(a Asset) bool {
  2117. that, ok := a.(*SharedAsset)
  2118. if !ok {
  2119. return false
  2120. }
  2121. if !sa.Labels().Equal(that.Labels()) {
  2122. return false
  2123. }
  2124. if !sa.Properties().Equal(that.Properties()) {
  2125. return false
  2126. }
  2127. if !sa.Start().Equal(that.Start()) {
  2128. return false
  2129. }
  2130. if !sa.End().Equal(that.End()) {
  2131. return false
  2132. }
  2133. if !sa.window.Equal(that.window) {
  2134. return false
  2135. }
  2136. if sa.Cost != that.Cost {
  2137. return false
  2138. }
  2139. return true
  2140. }
  2141. // MarshalJSON implements json.Marshaler
  2142. func (sa *SharedAsset) MarshalJSON() ([]byte, error) {
  2143. buffer := bytes.NewBufferString("{")
  2144. jsonEncodeString(buffer, "type", sa.Type().String(), ",")
  2145. jsonEncode(buffer, "properties", sa.Properties(), ",")
  2146. jsonEncode(buffer, "labels", sa.Labels(), ",")
  2147. jsonEncode(buffer, "properties", sa.Properties(), ",")
  2148. jsonEncode(buffer, "labels", sa.Labels(), ",")
  2149. jsonEncode(buffer, "window", sa.Window(), ",")
  2150. jsonEncodeString(buffer, "start", sa.Start().Format(time.RFC3339), ",")
  2151. jsonEncodeString(buffer, "end", sa.End().Format(time.RFC3339), ",")
  2152. jsonEncodeFloat64(buffer, "minutes", sa.Minutes(), ",")
  2153. jsonEncodeFloat64(buffer, "totalCost", sa.TotalCost(), "")
  2154. buffer.WriteString("}")
  2155. return buffer.Bytes(), nil
  2156. }
  2157. // String implements fmt.Stringer
  2158. func (sa *SharedAsset) String() string {
  2159. return toString(sa)
  2160. }
  2161. // AssetSet stores a set of Assets, each with a unique name, that share
  2162. // a window. An AssetSet is mutable, so treat it like a threadsafe map.
  2163. type AssetSet struct {
  2164. sync.RWMutex
  2165. aggregateBy []string
  2166. assets map[string]Asset
  2167. FromSource string // stores the name of the source used to compute the data
  2168. Window Window
  2169. Warnings []string
  2170. Errors []string
  2171. }
  2172. // NewAssetSet instantiates a new AssetSet and, optionally, inserts
  2173. // the given list of Assets
  2174. func NewAssetSet(start, end time.Time, assets ...Asset) *AssetSet {
  2175. as := &AssetSet{
  2176. assets: map[string]Asset{},
  2177. Window: NewWindow(&start, &end),
  2178. }
  2179. for _, a := range assets {
  2180. as.Insert(a)
  2181. }
  2182. return as
  2183. }
  2184. // AggregateBy aggregates the Assets in the AssetSet by the given list of
  2185. // AssetProperties, such that each asset is binned by a key determined by its
  2186. // relevant property values.
  2187. func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOptions) error {
  2188. if opts == nil {
  2189. opts = &AssetAggregationOptions{}
  2190. }
  2191. if as.IsEmpty() && len(opts.SharedHourlyCosts) == 0 {
  2192. return nil
  2193. }
  2194. as.Lock()
  2195. defer as.Unlock()
  2196. aggSet := NewAssetSet(as.Start(), as.End())
  2197. aggSet.aggregateBy = aggregateBy
  2198. // Compute hours of the given AssetSet, and if it ends in the future,
  2199. // adjust the hours accordingly
  2200. hours := as.Window.Minutes() / 60.0
  2201. diff := time.Since(as.End())
  2202. if diff < 0.0 {
  2203. hours += diff.Hours()
  2204. }
  2205. // Insert a shared asset for each shared cost
  2206. for name, hourlyCost := range opts.SharedHourlyCosts {
  2207. sa := NewSharedAsset(name, as.Window.Clone())
  2208. sa.Cost = hourlyCost * hours
  2209. err := aggSet.Insert(sa)
  2210. if err != nil {
  2211. return err
  2212. }
  2213. }
  2214. // Delete the Assets that don't pass each filter
  2215. for _, ff := range opts.FilterFuncs {
  2216. for key, asset := range as.assets {
  2217. if !ff(asset) {
  2218. delete(as.assets, key)
  2219. }
  2220. }
  2221. }
  2222. // Insert each asset into the new set, which will be keyed by the `aggregateBy`
  2223. // on aggSet, resulting in aggregation.
  2224. for _, asset := range as.assets {
  2225. err := aggSet.Insert(asset)
  2226. if err != nil {
  2227. return err
  2228. }
  2229. }
  2230. // Assign the aggregated values back to the original set
  2231. as.assets = aggSet.assets
  2232. as.aggregateBy = aggregateBy
  2233. return nil
  2234. }
  2235. // Clone returns a new AssetSet with a deep copy of the given
  2236. // AssetSet's assets.
  2237. func (as *AssetSet) Clone() *AssetSet {
  2238. if as == nil {
  2239. return nil
  2240. }
  2241. as.RLock()
  2242. defer as.RUnlock()
  2243. var aggregateBy []string
  2244. if as.aggregateBy != nil {
  2245. aggregateBy = append([]string{}, as.aggregateBy...)
  2246. }
  2247. assets := map[string]Asset{}
  2248. for k, v := range as.assets {
  2249. assets[k] = v.Clone()
  2250. }
  2251. s := as.Start()
  2252. e := as.End()
  2253. return &AssetSet{
  2254. Window: NewWindow(&s, &e),
  2255. aggregateBy: aggregateBy,
  2256. assets: assets,
  2257. }
  2258. }
  2259. // Each invokes the given function for each Asset in the set
  2260. func (as *AssetSet) Each(f func(string, Asset)) {
  2261. if as == nil {
  2262. return
  2263. }
  2264. for k, a := range as.assets {
  2265. f(k, a)
  2266. }
  2267. }
  2268. // End returns the end time of the AssetSet's window
  2269. func (as *AssetSet) End() time.Time {
  2270. return *as.Window.End()
  2271. }
  2272. // FindMatch attempts to find a match in the AssetSet for the given Asset on
  2273. // the provided properties and labels. If a match is not found, FindMatch
  2274. // returns nil and a Not Found error.
  2275. func (as *AssetSet) FindMatch(query Asset, aggregateBy []string) (Asset, error) {
  2276. as.RLock()
  2277. defer as.RUnlock()
  2278. matchKey, err := key(query, aggregateBy)
  2279. if err != nil {
  2280. return nil, err
  2281. }
  2282. for _, asset := range as.assets {
  2283. if k, err := key(asset, aggregateBy); err != nil {
  2284. return nil, err
  2285. } else if k == matchKey {
  2286. return asset, nil
  2287. }
  2288. }
  2289. return nil, fmt.Errorf("Asset not found to match %s on %v", query, aggregateBy)
  2290. }
  2291. // ReconciliationMatch attempts to find an exact match in the AssetSet on
  2292. // (Category, ProviderID). If a match is found, it returns the Asset with the
  2293. // intent to adjust it. If no match exists, it attempts to find one on only
  2294. // (ProviderID). If that match is found, it returns the Asset with the intent
  2295. // to insert the associated Cloud cost.
  2296. func (as *AssetSet) ReconciliationMatch(query Asset) (Asset, bool, error) {
  2297. as.RLock()
  2298. defer as.RUnlock()
  2299. // Full match means matching on (Category, ProviderID)
  2300. fullMatchProps := []string{string(AssetCategoryProp), string(AssetProviderIDProp)}
  2301. fullMatchKey, err := key(query, fullMatchProps)
  2302. // This should never happen because we are using enumerated properties,
  2303. // but the check is here in case that changes
  2304. if err != nil {
  2305. return nil, false, err
  2306. }
  2307. // Partial match means matching only on (ProviderID)
  2308. providerIDMatchProps := []string{string(AssetProviderIDProp)}
  2309. providerIDMatchKey, err := key(query, providerIDMatchProps)
  2310. // This should never happen because we are using enumerated properties,
  2311. // but the check is here in case that changes
  2312. if err != nil {
  2313. return nil, false, err
  2314. }
  2315. var providerIDMatch Asset
  2316. for _, asset := range as.assets {
  2317. if k, err := key(asset, fullMatchProps); err != nil {
  2318. return nil, false, err
  2319. } else if k == fullMatchKey {
  2320. log.DedupedInfof(10, "Asset ETL: Reconciliation[rcnw]: ReconcileRange Match: %s", fullMatchKey)
  2321. return asset, true, nil
  2322. }
  2323. if k, err := key(asset, providerIDMatchProps); err != nil {
  2324. return nil, false, err
  2325. } else if k == providerIDMatchKey {
  2326. // Found a partial match. Save it until after all other options
  2327. // have been checked for full matches.
  2328. providerIDMatch = asset
  2329. }
  2330. }
  2331. // No full match was found, so return partial match, if found.
  2332. if providerIDMatch != nil {
  2333. return providerIDMatch, false, nil
  2334. }
  2335. return nil, false, fmt.Errorf("Asset not found to match %s", query)
  2336. }
  2337. // Get returns the Asset in the AssetSet at the given key, or nil and false
  2338. // if no Asset exists for the given key
  2339. func (as *AssetSet) Get(key string) (Asset, bool) {
  2340. as.RLock()
  2341. defer as.RUnlock()
  2342. if a, ok := as.assets[key]; ok {
  2343. return a, true
  2344. }
  2345. return nil, false
  2346. }
  2347. // Insert inserts the given Asset into the AssetSet, using the AssetSet's
  2348. // configured properties to determine the key under which the Asset will
  2349. // be inserted.
  2350. func (as *AssetSet) Insert(asset Asset) error {
  2351. if as == nil {
  2352. return fmt.Errorf("cannot Insert into nil AssetSet")
  2353. }
  2354. as.Lock()
  2355. defer as.Unlock()
  2356. if as.assets == nil {
  2357. as.assets = map[string]Asset{}
  2358. }
  2359. // Determine key into which to Insert the Asset.
  2360. k, err := key(asset, as.aggregateBy)
  2361. if err != nil {
  2362. return err
  2363. }
  2364. // Add the given Asset to the existing entry, if there is one;
  2365. // otherwise just set directly into assets
  2366. if _, ok := as.assets[k]; !ok {
  2367. as.assets[k] = asset
  2368. } else {
  2369. as.assets[k] = as.assets[k].Add(asset)
  2370. }
  2371. // Expand the window, just to be safe. It's possible that the asset will
  2372. // be set into the map without expanding it to the AssetSet's window.
  2373. as.assets[k].ExpandWindow(as.Window)
  2374. return nil
  2375. }
  2376. // IsEmpty returns true if the AssetSet is nil, or if it contains
  2377. // zero assets.
  2378. func (as *AssetSet) IsEmpty() bool {
  2379. if as == nil || len(as.assets) == 0 {
  2380. return true
  2381. }
  2382. as.RLock()
  2383. defer as.RUnlock()
  2384. return as.assets == nil || len(as.assets) == 0
  2385. }
  2386. func (as *AssetSet) Length() int {
  2387. if as == nil {
  2388. return 0
  2389. }
  2390. as.RLock()
  2391. defer as.RUnlock()
  2392. return len(as.assets)
  2393. }
  2394. // Map clones and returns a map of the AssetSet's Assets
  2395. func (as *AssetSet) Map() map[string]Asset {
  2396. if as.IsEmpty() {
  2397. return map[string]Asset{}
  2398. }
  2399. return as.Clone().assets
  2400. }
  2401. // MarshalJSON JSON-encodes the AssetSet
  2402. func (as *AssetSet) MarshalJSON() ([]byte, error) {
  2403. as.RLock()
  2404. defer as.RUnlock()
  2405. return json.Marshal(as.assets)
  2406. }
  2407. func (as *AssetSet) Set(asset Asset, aggregateBy []string) error {
  2408. if as.IsEmpty() {
  2409. as.Lock()
  2410. as.assets = map[string]Asset{}
  2411. as.Unlock()
  2412. }
  2413. as.Lock()
  2414. defer as.Unlock()
  2415. // Expand the window to match the AssetSet, then set it
  2416. asset.ExpandWindow(as.Window)
  2417. k, err := key(asset, aggregateBy)
  2418. if err != nil {
  2419. return err
  2420. }
  2421. as.assets[k] = asset
  2422. return nil
  2423. }
  2424. func (as *AssetSet) Start() time.Time {
  2425. return *as.Window.Start()
  2426. }
  2427. func (as *AssetSet) TotalCost() float64 {
  2428. tc := 0.0
  2429. as.Lock()
  2430. defer as.Unlock()
  2431. for _, a := range as.assets {
  2432. tc += a.TotalCost()
  2433. }
  2434. return tc
  2435. }
  2436. func (as *AssetSet) UTCOffset() time.Duration {
  2437. _, zone := as.Start().Zone()
  2438. return time.Duration(zone) * time.Second
  2439. }
  2440. func (as *AssetSet) accumulate(that *AssetSet) (*AssetSet, error) {
  2441. if as == nil {
  2442. return that, nil
  2443. }
  2444. if that == nil {
  2445. return as, nil
  2446. }
  2447. // In the case of an AssetSetRange with empty entries, we may end up with
  2448. // an incoming `as` without an `aggregateBy`, even though we are tring to
  2449. // aggregate here. This handles that case by assigning the correct `aggregateBy`.
  2450. if !sameContents(as.aggregateBy, that.aggregateBy) {
  2451. if len(as.aggregateBy) == 0 {
  2452. as.aggregateBy = that.aggregateBy
  2453. }
  2454. }
  2455. // Set start, end to min(start), max(end)
  2456. start := as.Start()
  2457. end := as.End()
  2458. if that.Start().Before(start) {
  2459. start = that.Start()
  2460. }
  2461. if that.End().After(end) {
  2462. end = that.End()
  2463. }
  2464. if as.IsEmpty() && that.IsEmpty() {
  2465. return NewAssetSet(start, end), nil
  2466. }
  2467. acc := NewAssetSet(start, end)
  2468. acc.aggregateBy = as.aggregateBy
  2469. as.RLock()
  2470. defer as.RUnlock()
  2471. that.RLock()
  2472. defer that.RUnlock()
  2473. for _, asset := range as.assets {
  2474. err := acc.Insert(asset)
  2475. if err != nil {
  2476. return nil, err
  2477. }
  2478. }
  2479. for _, asset := range that.assets {
  2480. err := acc.Insert(asset)
  2481. if err != nil {
  2482. return nil, err
  2483. }
  2484. }
  2485. return acc, nil
  2486. }
  2487. // AssetSetRange is a thread-safe slice of AssetSets. It is meant to
  2488. // be used such that the AssetSets held are consecutive and coherent with
  2489. // respect to using the same aggregation properties, UTC offset, and
  2490. // resolution. However these rules are not necessarily enforced, so use wisely.
  2491. type AssetSetRange struct {
  2492. sync.RWMutex
  2493. assets []*AssetSet
  2494. FromStore string // stores the name of the store used to retrieve the data
  2495. }
  2496. // NewAssetSetRange instantiates a new range composed of the given
  2497. // AssetSets in the order provided.
  2498. func NewAssetSetRange(assets ...*AssetSet) *AssetSetRange {
  2499. return &AssetSetRange{
  2500. assets: assets,
  2501. }
  2502. }
  2503. // Accumulate sums each AssetSet in the given range, returning a single cumulative
  2504. // AssetSet for the entire range.
  2505. func (asr *AssetSetRange) Accumulate() (*AssetSet, error) {
  2506. var assetSet *AssetSet
  2507. var err error
  2508. asr.RLock()
  2509. defer asr.RUnlock()
  2510. for _, as := range asr.assets {
  2511. assetSet, err = assetSet.accumulate(as)
  2512. if err != nil {
  2513. return nil, err
  2514. }
  2515. }
  2516. return assetSet, nil
  2517. }
  2518. type AssetAggregationOptions struct {
  2519. SharedHourlyCosts map[string]float64
  2520. FilterFuncs []AssetMatchFunc
  2521. }
  2522. func (asr *AssetSetRange) AggregateBy(aggregateBy []string, opts *AssetAggregationOptions) error {
  2523. aggRange := &AssetSetRange{assets: []*AssetSet{}}
  2524. asr.Lock()
  2525. defer asr.Unlock()
  2526. for _, as := range asr.assets {
  2527. err := as.AggregateBy(aggregateBy, opts)
  2528. if err != nil {
  2529. return err
  2530. }
  2531. aggRange.assets = append(aggRange.assets, as)
  2532. }
  2533. asr.assets = aggRange.assets
  2534. return nil
  2535. }
  2536. func (asr *AssetSetRange) Append(that *AssetSet) {
  2537. asr.Lock()
  2538. defer asr.Unlock()
  2539. asr.assets = append(asr.assets, that)
  2540. }
  2541. // Each invokes the given function for each AssetSet in the range
  2542. func (asr *AssetSetRange) Each(f func(int, *AssetSet)) {
  2543. if asr == nil {
  2544. return
  2545. }
  2546. for i, as := range asr.assets {
  2547. f(i, as)
  2548. }
  2549. }
  2550. func (asr *AssetSetRange) Get(i int) (*AssetSet, error) {
  2551. if i < 0 || i >= len(asr.assets) {
  2552. return nil, fmt.Errorf("AssetSetRange: index out of range: %d", i)
  2553. }
  2554. asr.RLock()
  2555. defer asr.RUnlock()
  2556. return asr.assets[i], nil
  2557. }
  2558. func (asr *AssetSetRange) Length() int {
  2559. if asr == nil || asr.assets == nil {
  2560. return 0
  2561. }
  2562. asr.RLock()
  2563. defer asr.RUnlock()
  2564. return len(asr.assets)
  2565. }
  2566. func (asr *AssetSetRange) MarshalJSON() ([]byte, error) {
  2567. asr.RLock()
  2568. defer asr.RUnlock()
  2569. return json.Marshal(asr.assets)
  2570. }
  2571. func (asr *AssetSetRange) UTCOffset() time.Duration {
  2572. if asr.Length() == 0 {
  2573. return 0
  2574. }
  2575. as, err := asr.Get(0)
  2576. if err != nil {
  2577. return 0
  2578. }
  2579. return as.UTCOffset()
  2580. }
  2581. // Window returns the full window that the AssetSetRange spans, from the
  2582. // start of the first AssetSet to the end of the last one.
  2583. func (asr *AssetSetRange) Window() Window {
  2584. if asr == nil || asr.Length() == 0 {
  2585. return NewWindow(nil, nil)
  2586. }
  2587. start := asr.assets[0].Start()
  2588. end := asr.assets[asr.Length()-1].End()
  2589. return NewWindow(&start, &end)
  2590. }
  2591. // Returns true if string slices a and b contain all of the same strings, in any order.
  2592. func sameContents(a, b []string) bool {
  2593. if len(a) != len(b) {
  2594. return false
  2595. }
  2596. for i := range a {
  2597. if !contains(b, a[i]) {
  2598. return false
  2599. }
  2600. }
  2601. return true
  2602. }
  2603. func contains(slice []string, item string) bool {
  2604. for _, element := range slice {
  2605. if element == item {
  2606. return true
  2607. }
  2608. }
  2609. return false
  2610. }