metric_codecs.go 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317
  1. ////////////////////////////////////////////////////////////////////////////////
  2. //
  3. // DO NOT MODIFY
  4. //
  5. // ┻━┻ ︵ヽ(`Д´)ノ︵ ┻━┻
  6. //
  7. //
  8. // This source file was automatically generated by bingen.
  9. //
  10. ////////////////////////////////////////////////////////////////////////////////
  11. package metric
  12. import (
  13. "cmp"
  14. "fmt"
  15. "io"
  16. "iter"
  17. "os"
  18. "reflect"
  19. "slices"
  20. "strings"
  21. "sync"
  22. "time"
  23. "unsafe"
  24. util "github.com/opencost/opencost/core/pkg/util"
  25. )
  26. const (
  27. // GeneratorPackageName is the package the generator is targetting
  28. GeneratorPackageName string = "metric"
  29. StringHeaderSize = int64(unsafe.Sizeof(""))
  30. // BinaryTagStringTable is written and/or read prior to the existence of a string
  31. // table (where each index is encoded as a string entry in the resource
  32. BinaryTagStringTable string = "BGST"
  33. // DefaultCodecVersion is used for any resources listed in the Default version set
  34. DefaultCodecVersion uint8 = 1
  35. )
  36. //--------------------------------------------------------------------------
  37. // Configuration
  38. //--------------------------------------------------------------------------
  39. var (
  40. bingenConfigLock sync.RWMutex
  41. bingenConfig *BingenConfiguration = DefaultBingenConfiguration()
  42. )
  43. // BingenConfiguration is used to set any custom configuration in the way files are encoded
  44. // or decoded.
  45. type BingenConfiguration struct {
  46. // FileBackedStringTableEnabled enables the use of file-backed string tables for streaming
  47. // bingen decoding.
  48. FileBackedStringTableEnabled bool
  49. // FileBackedStringTableDir is the directory to write the string table files for reading.
  50. FileBackedStringTableDir string
  51. // FileBackedStringTableMemoMaxBytes limits in-memory memoization for file-backed table lookups.
  52. // 0 disables memoization.
  53. FileBackedStringTableMemoMaxBytes int64
  54. }
  55. // DefaultBingenConfiguration creates the default implementation of the bingen configuration
  56. // and returns it.
  57. func DefaultBingenConfiguration() *BingenConfiguration {
  58. return &BingenConfiguration{
  59. FileBackedStringTableEnabled: false,
  60. FileBackedStringTableDir: os.TempDir(),
  61. FileBackedStringTableMemoMaxBytes: 0,
  62. }
  63. }
  64. // ConfigureBingen accepts a new *BingenConfiguration instance which updates the internal decoder
  65. // and encoder behavior.
  66. func ConfigureBingen(config *BingenConfiguration) {
  67. bingenConfigLock.Lock()
  68. defer bingenConfigLock.Unlock()
  69. if config == nil {
  70. config = DefaultBingenConfiguration()
  71. }
  72. bingenConfig = config
  73. }
  74. // IsBingenFileBackedStringTableEnabled accessor for file backed string table configuration
  75. func IsBingenFileBackedStringTableEnabled() bool {
  76. bingenConfigLock.RLock()
  77. defer bingenConfigLock.RUnlock()
  78. return bingenConfig.FileBackedStringTableEnabled
  79. }
  80. // BingenFileBackedStringTableDir returns the directory configured for file backed string tables.
  81. func BingenFileBackedStringTableDir() string {
  82. bingenConfigLock.RLock()
  83. defer bingenConfigLock.RUnlock()
  84. return bingenConfig.FileBackedStringTableDir
  85. }
  86. // BingenFileBackedStringTableMemoMaxBytes returns the maximum bytes used for file-backed memo cache.
  87. func BingenFileBackedStringTableMemoMaxBytes() int64 {
  88. bingenConfigLock.RLock()
  89. defer bingenConfigLock.RUnlock()
  90. return bingenConfig.FileBackedStringTableMemoMaxBytes
  91. }
  92. //--------------------------------------------------------------------------
  93. // Type Map
  94. //--------------------------------------------------------------------------
  95. // Generated type map for resolving interface implementations to to concrete types
  96. var typeMap map[string]reflect.Type = map[string]reflect.Type{
  97. "Update": reflect.TypeFor[Update](),
  98. "UpdateSet": reflect.TypeFor[UpdateSet](),
  99. }
  100. //--------------------------------------------------------------------------
  101. // Type Helpers
  102. //--------------------------------------------------------------------------
  103. // isBinaryTag returns true when the first bytes in the provided binary matches the tag
  104. func isBinaryTag(data []byte, tag string) bool {
  105. if len(data) < len(tag) {
  106. return false
  107. }
  108. return string(data[:len(tag)]) == tag
  109. }
  110. // isReaderBinaryTag is used to peek the header for an io.Reader Buffer
  111. func isReaderBinaryTag(buff *util.Buffer, tag string) bool {
  112. data, err := buff.Peek(len(tag))
  113. if err != nil && err != io.EOF {
  114. panic(fmt.Sprintf("called Peek() on a non buffered reader: %s", err))
  115. }
  116. if len(data) < len(tag) {
  117. return false
  118. }
  119. return string(data[:len(tag)]) == tag
  120. }
  121. // typeToString determines the basic properties of the type, the qualifier, package path, and
  122. // type name, and returns the qualified type
  123. func typeToString(f interface{}) string {
  124. qual := ""
  125. t := reflect.TypeOf(f)
  126. if t.Kind() == reflect.Ptr {
  127. t = t.Elem()
  128. qual = "*"
  129. }
  130. return fmt.Sprintf("%s%s.%s", qual, t.PkgPath(), t.Name())
  131. }
  132. // resolveType uses the name of a type and returns the package, base type name, and whether
  133. // or not it's a pointer.
  134. func resolveType(t string) (pkg string, name string, isPtr bool) {
  135. isPtr = t[:1] == "*"
  136. if isPtr {
  137. t = t[1:]
  138. }
  139. slashIndex := strings.LastIndex(t, "/")
  140. if slashIndex >= 0 {
  141. t = t[slashIndex+1:]
  142. }
  143. parts := strings.Split(t, ".")
  144. if parts[0] == GeneratorPackageName {
  145. parts[0] = ""
  146. }
  147. pkg = parts[0]
  148. name = parts[1]
  149. return
  150. }
  151. //--------------------------------------------------------------------------
  152. // Stream Helpers
  153. //--------------------------------------------------------------------------
  154. // StreamFactoryFunc is an alias for a func that creates a BingenStream implementation.
  155. type StreamFactoryFunc func(io.Reader) BingenStream
  156. // Generated streamable factory map for finding the specific new stream methods
  157. // by T type
  158. var streamFactoryMap map[reflect.Type]StreamFactoryFunc = map[reflect.Type]StreamFactoryFunc{
  159. reflect.TypeFor[UpdateSet](): NewUpdateSetStream,
  160. }
  161. // NewStreamFor accepts an io.Reader, and returns a new BingenStream for the generic T
  162. // type provided _if_ it is a registered bingen type that is annotated as 'streamable'. See
  163. // the streamFactoryMap for generated type listings.
  164. func NewStreamFor[T any](reader io.Reader) (BingenStream, error) {
  165. typeKey := reflect.TypeFor[T]()
  166. factory, ok := streamFactoryMap[typeKey]
  167. if !ok {
  168. return nil, fmt.Errorf("the type: %s is not a registered bingen streamable type", typeKey.Name())
  169. }
  170. return factory(reader), nil
  171. }
  172. // BingenStream is the stream interface for all streamable types
  173. type BingenStream interface {
  174. // Stream returns the iterator which will stream each field of the target type and
  175. // return the field info as well as the value.
  176. Stream() iter.Seq2[BingenFieldInfo, *BingenValue]
  177. // Close will close any dynamic io.Reader used to stream in the fields
  178. Close()
  179. // Error returns an error if one occurred during the process of streaming the type's fields.
  180. // This can be checked after iterating through the Stream().
  181. Error() error
  182. }
  183. // BingenValue contains the value of a field as well as any index/key associated with that value.
  184. type BingenValue struct {
  185. Value any
  186. Index any
  187. }
  188. // IsNil is just a method accessor way to check to see if the value returned was nil
  189. func (bv *BingenValue) IsNil() bool {
  190. return bv == nil
  191. }
  192. // creates a single BingenValue instance without a key or index
  193. func singleV(value any) *BingenValue {
  194. return &BingenValue{
  195. Value: value,
  196. }
  197. }
  198. // creates a pair of key/index and value.
  199. func pairV(index any, value any) *BingenValue {
  200. return &BingenValue{
  201. Value: value,
  202. Index: index,
  203. }
  204. }
  205. // BingenFieldInfo contains the type of the field being streamed as well as the name of the field.
  206. type BingenFieldInfo struct {
  207. Type reflect.Type
  208. Name string
  209. }
  210. //--------------------------------------------------------------------------
  211. // String Table Writer
  212. //--------------------------------------------------------------------------
  213. // StringTableWriter is the interface used to write the string table for encoding.
  214. type StringTableWriter interface {
  215. // AddOrGet adds a string to the string table and returns the new index or
  216. // an existing index.
  217. AddOrGet(s string) int
  218. // WriteTo will write the StringTable data (with the header) to the provided
  219. // Buffer starting a the current write position
  220. WriteTo(b *util.Buffer)
  221. }
  222. // IndexedStringTableWriter maps strings to specific indices for encoding
  223. type IndexedStringTableWriter struct {
  224. indices map[string]int
  225. next int
  226. }
  227. // NewIndexedStringTableWriter Creates a new IndexedStringTableWriter instance.
  228. func NewIndexedStringTableWriter() *IndexedStringTableWriter {
  229. return &IndexedStringTableWriter{
  230. indices: make(map[string]int),
  231. next: 0,
  232. }
  233. }
  234. // AddOrGet retrieves a string entry's index if it exists. Otherwise, it adds the entry and returns the new index.
  235. func (st *IndexedStringTableWriter) AddOrGet(s string) int {
  236. if ind, ok := st.indices[s]; ok {
  237. return ind
  238. }
  239. current := st.next
  240. st.next++
  241. st.indices[s] = current
  242. return current
  243. }
  244. // ToSlice Converts the contents to a string array for encoding.
  245. func (st *IndexedStringTableWriter) ToSlice() []string {
  246. if st.next == 0 {
  247. return []string{}
  248. }
  249. sl := make([]string, st.next)
  250. for s, i := range st.indices {
  251. sl[i] = s
  252. }
  253. return sl
  254. }
  255. // ToBytes Converts the contents to a binary encoded representation
  256. func (st *IndexedStringTableWriter) ToBytes() []byte {
  257. buff := util.NewBuffer()
  258. st.WriteTo(buff)
  259. return buff.Bytes()
  260. }
  261. // WriteTo will write the StringTable data (with the header) to the provided
  262. // Buffer starting a the current write position
  263. func (st *IndexedStringTableWriter) WriteTo(buff *util.Buffer) {
  264. // bingen string table header
  265. buff.WriteBytes([]byte(BinaryTagStringTable))
  266. // get an ordered string slice to encode
  267. strs := st.ToSlice()
  268. buff.WriteInt(len(strs)) // table length
  269. for _, s := range strs {
  270. buff.WriteString(s)
  271. }
  272. }
  273. type indexed struct {
  274. s string
  275. count uint64
  276. index int
  277. }
  278. func newIndexed(s string, index int) *indexed {
  279. return &indexed{
  280. s: s,
  281. count: 1,
  282. index: index,
  283. }
  284. }
  285. // PrepassStringTableWriter maps strings to specific indices for encoding, sorted by the total
  286. // number of times they're accessed
  287. type PrepassStringTableWriter struct {
  288. prepass map[string]*indexed
  289. next int
  290. }
  291. // NewPrepassStringTableWriter creates a new PrepassStringTableWriter instance.
  292. func NewPrepassStringTableWriter() *PrepassStringTableWriter {
  293. return &PrepassStringTableWriter{
  294. prepass: make(map[string]*indexed),
  295. }
  296. }
  297. // AddOrGet retrieves a string entry's index if it exists. Otherwise, it adds the entry and returns the new index.
  298. func (st *PrepassStringTableWriter) AddOrGet(s string) int {
  299. if ind, ok := st.prepass[s]; ok {
  300. ind.count += 1
  301. return ind.index
  302. }
  303. current := st.next
  304. st.next++
  305. st.prepass[s] = newIndexed(s, current)
  306. return current
  307. }
  308. // WriteSortedTo sorts the string table by the number of accesses, writes the table in that
  309. // order, then returns a new StringTableWriter implementation that can be used for the new
  310. // sorted order index lookups.
  311. func (st *PrepassStringTableWriter) WriteSortedTo(buff *util.Buffer) StringTableWriter {
  312. sl := make([]*indexed, st.next)
  313. for _, ind := range st.prepass {
  314. sl[ind.index] = ind
  315. }
  316. slices.SortFunc(sl, func(a *indexed, b *indexed) int {
  317. return -cmp.Compare(a.count, b.count)
  318. })
  319. sti := NewIndexedStringTableWriter()
  320. for _, ind := range sl {
  321. sti.AddOrGet(ind.s)
  322. }
  323. sti.WriteTo(buff)
  324. return sti
  325. }
  326. // WriteTo will write the StringTable data (with the header) to the provided
  327. // Buffer starting a the current write position
  328. func (st *PrepassStringTableWriter) WriteTo(buff *util.Buffer) {
  329. panic("Prepass StringTableWriter cannot write directly")
  330. }
  331. //--------------------------------------------------------------------------
  332. // String Table Reader
  333. //--------------------------------------------------------------------------
  334. // StringTableReader is the interface used to read the string table from the decoding.
  335. type StringTableReader interface {
  336. // At returns the string entry at a specific index, or panics on out of bounds.
  337. At(index int) string
  338. // Len returns the total number of strings loaded in the string table.
  339. Len() int
  340. // Close will clear the loaded table, and drop any external resources used.
  341. Close() error
  342. }
  343. // SliceStringTableReader is a basic pre-loaded []string that provides index-based access.
  344. // The cost of this implementation is holding all strings in memory, which provides faster
  345. // lookup performance at the expense of memory usage.
  346. type SliceStringTableReader struct {
  347. table []string
  348. }
  349. // NewSliceStringTableReaderFrom creates a new SliceStringTableReader instance loading
  350. // data directly from the buffer. The buffer's position should start at the table length.
  351. func NewSliceStringTableReaderFrom(buffer *util.Buffer) StringTableReader {
  352. // table length
  353. tl := buffer.ReadInt()
  354. var table []string
  355. if tl > 0 {
  356. table = make([]string, tl)
  357. for i := range tl {
  358. table[i] = buffer.ReadString()
  359. }
  360. }
  361. return &SliceStringTableReader{
  362. table: table,
  363. }
  364. }
  365. // At returns the string entry at a specific index, or panics on out of bounds.
  366. func (sstr *SliceStringTableReader) At(index int) string {
  367. if index < 0 || index >= len(sstr.table) {
  368. panic(fmt.Errorf("%s: string table index out of bounds: %d", GeneratorPackageName, index))
  369. }
  370. return sstr.table[index]
  371. }
  372. // Len returns the total number of strings loaded in the string table.
  373. func (sstr *SliceStringTableReader) Len() int {
  374. if sstr == nil {
  375. return 0
  376. }
  377. return len(sstr.table)
  378. }
  379. // Close for the slice tables just nils out the slice and returns
  380. func (sstr *SliceStringTableReader) Close() error {
  381. sstr.table = nil
  382. return nil
  383. }
  384. // fileStringRef maps a bingen string-table index to a payload stored in a temp file.
  385. type fileStringRef struct {
  386. off int64
  387. length int
  388. }
  389. // FileStringTableReader leverages a local file to write string table data for lookup. On
  390. // memory focused systems, this allows a slower parse with a significant decrease in memory
  391. // usage. This implementation is often pair with streaming readers for high throughput with
  392. // reduced memory usage.
  393. type FileStringTableReader struct {
  394. f *os.File
  395. refs []fileStringRef
  396. memo []string
  397. }
  398. // NewFileStringTableFromBuffer reads exactly tl length-prefixed (uint16) string payloads from buffer
  399. // and appends each payload to a new temp file. It does not retain full strings in memory.
  400. func NewFileStringTableReaderFrom(buffer *util.Buffer, dir string, memoMaxBytes int64) StringTableReader {
  401. // helper func to cast a string in-place to a byte slice.
  402. // NOTE: Return value is READ-ONLY. DO NOT MODIFY!
  403. byteSliceFor := func(s string) []byte {
  404. return unsafe.Slice(unsafe.StringData(s), len(s))
  405. }
  406. err := os.MkdirAll(dir, 0755)
  407. if err != nil {
  408. panic(fmt.Errorf("%s: failed to create string table directory: %w", GeneratorPackageName, err))
  409. }
  410. f, err := os.CreateTemp(dir, fmt.Sprintf("%s-bgst-*", GeneratorPackageName))
  411. if err != nil {
  412. panic(fmt.Errorf("%s: failed to create string table file: %w", GeneratorPackageName, err))
  413. }
  414. var writeErr error
  415. defer func() {
  416. if writeErr != nil {
  417. _ = f.Close()
  418. }
  419. }()
  420. // table length
  421. tl := buffer.ReadInt()
  422. var refs []fileStringRef
  423. if tl > 0 {
  424. refs = make([]fileStringRef, tl)
  425. for i := range tl {
  426. payload := byteSliceFor(buffer.ReadString())
  427. var off int64
  428. if len(payload) > 0 {
  429. off, err = f.Seek(0, io.SeekEnd)
  430. if err != nil {
  431. writeErr = fmt.Errorf("%s: failed to seek string table file: %w", GeneratorPackageName, err)
  432. panic(writeErr)
  433. }
  434. if _, err := f.Write(payload); err != nil {
  435. writeErr = fmt.Errorf("%s: failed to write string table entry %d: %w", GeneratorPackageName, i, err)
  436. panic(writeErr)
  437. }
  438. }
  439. refs[i] = fileStringRef{
  440. off: off,
  441. length: len(payload),
  442. }
  443. }
  444. }
  445. var memo []string
  446. // Pre-load cache with strings up to memoMaxBytes, respecting string boundaries
  447. if memoMaxBytes > 0 && len(refs) > 0 {
  448. memo = make([]string, len(refs))
  449. var cumulativeSize int64
  450. for i, ref := range refs {
  451. // Check if adding this string would exceed the limit
  452. if cumulativeSize+int64(ref.length)+StringHeaderSize > memoMaxBytes {
  453. // Would exceed limit, stop here
  454. break
  455. }
  456. // Read string from file and cache it
  457. if ref.length > 0 {
  458. b := make([]byte, ref.length)
  459. _, err := f.ReadAt(b, ref.off)
  460. if err != nil {
  461. // If we can't read, skip this entry but continue
  462. continue
  463. }
  464. // Cast the allocated bytes to a string in-place
  465. str := unsafe.String(unsafe.SliceData(b), len(b))
  466. memo[i] = str
  467. cumulativeSize += int64(ref.length) + StringHeaderSize
  468. }
  469. }
  470. }
  471. return &FileStringTableReader{
  472. f: f,
  473. refs: refs,
  474. memo: memo,
  475. }
  476. }
  477. // At returns the string from the internal file using the reference's offset and length.
  478. func (fstr *FileStringTableReader) At(index int) string {
  479. if fstr == nil || fstr.f == nil {
  480. panic(fmt.Errorf("%s: failed to read file string table data", GeneratorPackageName))
  481. }
  482. if index < 0 || index >= len(fstr.refs) {
  483. panic(fmt.Errorf("%s: string table index out of bounds: %d", GeneratorPackageName, index))
  484. }
  485. ref := fstr.refs[index]
  486. if ref.length == 0 {
  487. return ""
  488. }
  489. // Check cache first
  490. if fstr.memo != nil && len(fstr.memo) > index && fstr.memo[index] != "" {
  491. return fstr.memo[index]
  492. }
  493. // Cache miss - read from file
  494. b := make([]byte, ref.length)
  495. _, err := fstr.f.ReadAt(b, ref.off)
  496. if err != nil {
  497. return ""
  498. }
  499. // Cast the allocated bytes to a string in-place, as we were the ones that allocated the bytes
  500. return unsafe.String(unsafe.SliceData(b), len(b))
  501. }
  502. // Len returns the total number of strings loaded in the string table.
  503. func (fstr *FileStringTableReader) Len() int {
  504. if fstr == nil {
  505. return 0
  506. }
  507. return len(fstr.refs)
  508. }
  509. // Close for the file string table reader closes the file and deletes it.
  510. func (fstr *FileStringTableReader) Close() error {
  511. if fstr == nil || fstr.f == nil {
  512. return nil
  513. }
  514. path := fstr.f.Name()
  515. err := fstr.f.Close()
  516. fstr.f = nil
  517. fstr.refs = nil
  518. fstr.memo = nil
  519. if path != "" {
  520. _ = os.Remove(path)
  521. }
  522. return err
  523. }
  524. //--------------------------------------------------------------------------
  525. // Codec Context
  526. //--------------------------------------------------------------------------
  527. // EncodingContext is a context object passed to the encoders to ensure reuse of buffer
  528. // and table data
  529. type EncodingContext struct {
  530. Buffer *util.Buffer
  531. Table StringTableWriter
  532. }
  533. // NewEncodingContext creates a new EncodingContext instance that will create a new []byte buffer
  534. // for writing, and return the context
  535. func NewEncodingContext(tableWriter StringTableWriter) *EncodingContext {
  536. return &EncodingContext{
  537. Buffer: util.NewBuffer(),
  538. Table: tableWriter,
  539. }
  540. }
  541. // NewEncodingContextFromWriter creates a new EncodingContext instance that will create a new Buffer
  542. // from the provided io.Writer and StringTableWriter.
  543. func NewEncodingContextFromWriter(writer io.Writer, tableWriter StringTableWriter) *EncodingContext {
  544. return &EncodingContext{
  545. Buffer: util.NewBufferFromWriter(writer),
  546. Table: tableWriter,
  547. }
  548. }
  549. // NewEncodingContextFromBuffer creates a new EncodingContext instance that will leverage an existing
  550. // Buffer and StringTableWriter.
  551. func NewEncodingContextFromBuffer(buffer *util.Buffer, tableWriter StringTableWriter) *EncodingContext {
  552. return &EncodingContext{
  553. Buffer: buffer,
  554. Table: tableWriter,
  555. }
  556. }
  557. // ToBytes returns the encoded string table bytes (if applicable) combined with the encoded buffer bytes. If
  558. // a string table is being used, the string table bytes will be written first to ensure correct ordering for
  559. // decoding.
  560. func (ec *EncodingContext) ToBytes() []byte {
  561. encBytes := ec.Buffer.Bytes()
  562. if ec.Table != nil {
  563. buff := util.NewBuffer()
  564. ec.Table.WriteTo(buff)
  565. buff.WriteBytes(encBytes)
  566. return buff.Bytes()
  567. }
  568. return encBytes
  569. }
  570. // IsStringTable returns true if the table is available
  571. func (ec *EncodingContext) IsStringTable() bool {
  572. return ec.Table != nil
  573. }
  574. // DecodingContext is a context object passed to the decoders to ensure parent objects
  575. // reuse as much data as possible
  576. type DecodingContext struct {
  577. Buffer *util.Buffer
  578. Table StringTableReader
  579. }
  580. // NewDecodingContextFromBytes creates a new DecodingContext instance using an byte slice
  581. func NewDecodingContextFromBytes(data []byte) *DecodingContext {
  582. var table StringTableReader
  583. buff := util.NewBufferFromBytes(data)
  584. // string table header validation
  585. if isBinaryTag(data, BinaryTagStringTable) {
  586. buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
  587. // always use a slice string table with a byte array since the
  588. // data is already in memory
  589. table = NewSliceStringTableReaderFrom(buff)
  590. }
  591. return &DecodingContext{
  592. Buffer: buff,
  593. Table: table,
  594. }
  595. }
  596. // NewDecodingContextFromReader creates a new DecodingContext instance using an io.Reader
  597. // implementation
  598. func NewDecodingContextFromReader(reader io.Reader) *DecodingContext {
  599. var table StringTableReader
  600. buff := util.NewBufferFromReader(reader)
  601. if isReaderBinaryTag(buff, BinaryTagStringTable) {
  602. buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
  603. // create correct string table implementation
  604. if IsBingenFileBackedStringTableEnabled() {
  605. table = NewFileStringTableReaderFrom(buff, BingenFileBackedStringTableDir(), BingenFileBackedStringTableMemoMaxBytes())
  606. } else {
  607. table = NewSliceStringTableReaderFrom(buff)
  608. }
  609. }
  610. return &DecodingContext{
  611. Buffer: buff,
  612. Table: table,
  613. }
  614. }
  615. // IsStringTable returns true if the table is available
  616. func (dc *DecodingContext) IsStringTable() bool {
  617. return dc.Table != nil && dc.Table.Len() > 0
  618. }
  619. // Close will ensure that any string table resources and buffer resources are
  620. // cleaned up.
  621. func (dc *DecodingContext) Close() {
  622. if dc.Table != nil {
  623. _ = dc.Table.Close()
  624. dc.Table = nil
  625. }
  626. }
  627. //--------------------------------------------------------------------------
  628. // Binary Codec
  629. //--------------------------------------------------------------------------
  630. // BinEncoder is an encoding interface which defines a context based marshal contract.
  631. type BinEncoder interface {
  632. MarshalBinaryWithContext(*EncodingContext) error
  633. }
  634. // BinDecoder is a decoding interface which defines a context based unmarshal contract.
  635. type BinDecoder interface {
  636. UnmarshalBinaryWithContext(*DecodingContext) error
  637. }
  638. //--------------------------------------------------------------------------
  639. // Update
  640. //--------------------------------------------------------------------------
  641. // MarshalBinary serializes the internal properties of this Update instance
  642. // into a byte array
  643. func (target *Update) MarshalBinary() (data []byte, err error) {
  644. ctx := NewEncodingContext(nil)
  645. e := target.MarshalBinaryWithContext(ctx)
  646. if e != nil {
  647. return nil, e
  648. }
  649. return ctx.ToBytes(), nil
  650. }
  651. // MarshalBinary serializes the internal properties of this Update instance
  652. // into an io.Writer.
  653. func (target *Update) MarshalBinaryTo(writer io.Writer) error {
  654. buff := util.NewBufferFromWriter(writer)
  655. defer buff.Flush()
  656. ctx := NewEncodingContextFromBuffer(buff, nil)
  657. return target.MarshalBinaryWithContext(ctx)
  658. }
  659. // MarshalBinaryWithContext serializes the internal properties of this Update instance
  660. // into a byte array leveraging a predefined context.
  661. func (target *Update) MarshalBinaryWithContext(ctx *EncodingContext) (err error) {
  662. // panics are recovered and propagated as errors
  663. defer func() {
  664. if r := recover(); r != nil {
  665. if e, ok := r.(error); ok {
  666. err = e
  667. } else if s, ok := r.(string); ok {
  668. err = fmt.Errorf("unexpected panic: %s", s)
  669. } else {
  670. err = fmt.Errorf("unexpected panic: %+v", r)
  671. }
  672. }
  673. }()
  674. buff := ctx.Buffer
  675. buff.WriteUInt8(DefaultCodecVersion) // version
  676. if ctx.IsStringTable() {
  677. a := ctx.Table.AddOrGet(target.Name)
  678. buff.WriteInt(a) // write table index
  679. } else {
  680. buff.WriteString(target.Name) // write string
  681. }
  682. if target.Labels == nil {
  683. buff.WriteUInt8(uint8(0)) // write nil byte
  684. } else {
  685. buff.WriteUInt8(uint8(1)) // write non-nil byte
  686. // --- [begin][write][map](map[string]string) ---
  687. buff.WriteInt(len(target.Labels)) // map length
  688. for v, z := range target.Labels {
  689. if ctx.IsStringTable() {
  690. b := ctx.Table.AddOrGet(v)
  691. buff.WriteInt(b) // write table index
  692. } else {
  693. buff.WriteString(v) // write string
  694. }
  695. if ctx.IsStringTable() {
  696. c := ctx.Table.AddOrGet(z)
  697. buff.WriteInt(c) // write table index
  698. } else {
  699. buff.WriteString(z) // write string
  700. }
  701. }
  702. // --- [end][write][map](map[string]string) ---
  703. }
  704. buff.WriteFloat64(target.Value) // write float64
  705. if target.AdditionalInfo == nil {
  706. buff.WriteUInt8(uint8(0)) // write nil byte
  707. } else {
  708. buff.WriteUInt8(uint8(1)) // write non-nil byte
  709. // --- [begin][write][map](map[string]string) ---
  710. buff.WriteInt(len(target.AdditionalInfo)) // map length
  711. for vv, zz := range target.AdditionalInfo {
  712. if ctx.IsStringTable() {
  713. d := ctx.Table.AddOrGet(vv)
  714. buff.WriteInt(d) // write table index
  715. } else {
  716. buff.WriteString(vv) // write string
  717. }
  718. if ctx.IsStringTable() {
  719. e := ctx.Table.AddOrGet(zz)
  720. buff.WriteInt(e) // write table index
  721. } else {
  722. buff.WriteString(zz) // write string
  723. }
  724. }
  725. // --- [end][write][map](map[string]string) ---
  726. }
  727. return nil
  728. }
  729. // UnmarshalBinary uses the data passed byte array to set all the internal properties of
  730. // the Update type
  731. func (target *Update) UnmarshalBinary(data []byte) error {
  732. ctx := NewDecodingContextFromBytes(data)
  733. defer ctx.Close()
  734. err := target.UnmarshalBinaryWithContext(ctx)
  735. if err != nil {
  736. return err
  737. }
  738. return nil
  739. }
  740. // UnmarshalBinaryFromReader uses the io.Reader data to set all the internal properties of
  741. // the Update type
  742. func (target *Update) UnmarshalBinaryFromReader(reader io.Reader) error {
  743. ctx := NewDecodingContextFromReader(reader)
  744. defer ctx.Close()
  745. err := target.UnmarshalBinaryWithContext(ctx)
  746. if err != nil {
  747. return err
  748. }
  749. return nil
  750. }
  751. // UnmarshalBinaryWithContext uses the context containing a string table and binary buffer to set all the internal properties of
  752. // the Update type
  753. func (target *Update) UnmarshalBinaryWithContext(ctx *DecodingContext) (err error) {
  754. // panics are recovered and propagated as errors
  755. defer func() {
  756. if r := recover(); r != nil {
  757. if e, ok := r.(error); ok {
  758. err = e
  759. } else if s, ok := r.(string); ok {
  760. err = fmt.Errorf("unexpected panic: %s", s)
  761. } else {
  762. err = fmt.Errorf("unexpected panic: %+v", r)
  763. }
  764. }
  765. }()
  766. buff := ctx.Buffer
  767. version := buff.ReadUInt8()
  768. if version > DefaultCodecVersion {
  769. return fmt.Errorf("Invalid Version Unmarshalling Update. Expected %d or less, got %d", DefaultCodecVersion, version)
  770. }
  771. var b string
  772. if ctx.IsStringTable() {
  773. c := buff.ReadInt() // read string index
  774. b = ctx.Table.At(c)
  775. } else {
  776. b = buff.ReadString() // read string
  777. }
  778. a := b
  779. target.Name = a
  780. if buff.ReadUInt8() == uint8(0) {
  781. target.Labels = nil
  782. } else {
  783. // --- [begin][read][map](map[string]string) ---
  784. e := buff.ReadInt() // map len
  785. d := make(map[string]string, e)
  786. for range e {
  787. var v string
  788. var g string
  789. if ctx.IsStringTable() {
  790. h := buff.ReadInt() // read string index
  791. g = ctx.Table.At(h)
  792. } else {
  793. g = buff.ReadString() // read string
  794. }
  795. f := g
  796. v = f
  797. var z string
  798. var m string
  799. if ctx.IsStringTable() {
  800. n := buff.ReadInt() // read string index
  801. m = ctx.Table.At(n)
  802. } else {
  803. m = buff.ReadString() // read string
  804. }
  805. l := m
  806. z = l
  807. d[v] = z
  808. }
  809. target.Labels = d
  810. // --- [end][read][map](map[string]string) ---
  811. }
  812. o := buff.ReadFloat64() // read float64
  813. target.Value = o
  814. if buff.ReadUInt8() == uint8(0) {
  815. target.AdditionalInfo = nil
  816. } else {
  817. // --- [begin][read][map](map[string]string) ---
  818. q := buff.ReadInt() // map len
  819. p := make(map[string]string, q)
  820. for range q {
  821. var vv string
  822. var s string
  823. if ctx.IsStringTable() {
  824. t := buff.ReadInt() // read string index
  825. s = ctx.Table.At(t)
  826. } else {
  827. s = buff.ReadString() // read string
  828. }
  829. r := s
  830. vv = r
  831. var zz string
  832. var w string
  833. if ctx.IsStringTable() {
  834. x := buff.ReadInt() // read string index
  835. w = ctx.Table.At(x)
  836. } else {
  837. w = buff.ReadString() // read string
  838. }
  839. u := w
  840. zz = u
  841. p[vv] = zz
  842. }
  843. target.AdditionalInfo = p
  844. // --- [end][read][map](map[string]string) ---
  845. }
  846. return nil
  847. }
  848. //--------------------------------------------------------------------------
  849. // UpdateSet
  850. //--------------------------------------------------------------------------
  851. // MarshalBinary serializes the internal properties of this UpdateSet instance
  852. // into a byte array
  853. func (target *UpdateSet) MarshalBinary() (data []byte, err error) {
  854. ctx := NewEncodingContext(NewIndexedStringTableWriter())
  855. e := target.MarshalBinaryWithContext(ctx)
  856. if e != nil {
  857. return nil, e
  858. }
  859. return ctx.ToBytes(), nil
  860. }
  861. // MarshalBinary serializes the internal properties of this UpdateSet instance
  862. // into an io.Writer.
  863. func (target *UpdateSet) MarshalBinaryTo(writer io.Writer) error {
  864. buff := util.NewBufferFromWriter(writer)
  865. defer buff.Flush()
  866. // run a pre-pass to collect all strings into the string table and discard all writes to the main
  867. // buffer. Then, we write the string table, sorted by number of repeated uses (descending), to the
  868. // main buffer, and use the resulting table as part of the context for the main pass.
  869. prepass := NewPrepassStringTableWriter()
  870. prepassCtx := NewEncodingContextFromWriter(io.Discard, prepass)
  871. e := target.MarshalBinaryWithContext(prepassCtx)
  872. if e != nil {
  873. return e
  874. }
  875. tableWriter := prepass.WriteSortedTo(buff)
  876. ctx := NewEncodingContextFromBuffer(buff, tableWriter)
  877. return target.MarshalBinaryWithContext(ctx)
  878. }
  879. // MarshalBinaryWithContext serializes the internal properties of this UpdateSet instance
  880. // into a byte array leveraging a predefined context.
  881. func (target *UpdateSet) MarshalBinaryWithContext(ctx *EncodingContext) (err error) {
  882. // panics are recovered and propagated as errors
  883. defer func() {
  884. if r := recover(); r != nil {
  885. if e, ok := r.(error); ok {
  886. err = e
  887. } else if s, ok := r.(string); ok {
  888. err = fmt.Errorf("unexpected panic: %s", s)
  889. } else {
  890. err = fmt.Errorf("unexpected panic: %+v", r)
  891. }
  892. }
  893. }()
  894. buff := ctx.Buffer
  895. buff.WriteUInt8(DefaultCodecVersion) // version
  896. // --- [begin][write][reference](time.Time) ---
  897. a, errA := target.Timestamp.MarshalBinary()
  898. if errA != nil {
  899. return errA
  900. }
  901. buff.WriteInt(len(a))
  902. buff.WriteBytes(a)
  903. // --- [end][write][reference](time.Time) ---
  904. if target.Updates == nil {
  905. buff.WriteUInt8(uint8(0)) // write nil byte
  906. } else {
  907. buff.WriteUInt8(uint8(1)) // write non-nil byte
  908. // --- [begin][write][slice]([]Update) ---
  909. buff.WriteInt(len(target.Updates)) // slice length
  910. for i := range target.Updates {
  911. // --- [begin][write][struct](Update) ---
  912. buff.WriteInt(0) // [compatibility, unused]
  913. errB := target.Updates[i].MarshalBinaryWithContext(ctx)
  914. if errB != nil {
  915. return errB
  916. }
  917. // --- [end][write][struct](Update) ---
  918. }
  919. // --- [end][write][slice]([]Update) ---
  920. }
  921. return nil
  922. }
  923. // UnmarshalBinary uses the data passed byte array to set all the internal properties of
  924. // the UpdateSet type
  925. func (target *UpdateSet) UnmarshalBinary(data []byte) error {
  926. ctx := NewDecodingContextFromBytes(data)
  927. defer ctx.Close()
  928. err := target.UnmarshalBinaryWithContext(ctx)
  929. if err != nil {
  930. return err
  931. }
  932. return nil
  933. }
  934. // UnmarshalBinaryFromReader uses the io.Reader data to set all the internal properties of
  935. // the UpdateSet type
  936. func (target *UpdateSet) UnmarshalBinaryFromReader(reader io.Reader) error {
  937. ctx := NewDecodingContextFromReader(reader)
  938. defer ctx.Close()
  939. err := target.UnmarshalBinaryWithContext(ctx)
  940. if err != nil {
  941. return err
  942. }
  943. return nil
  944. }
  945. // UnmarshalBinaryWithContext uses the context containing a string table and binary buffer to set all the internal properties of
  946. // the UpdateSet type
  947. func (target *UpdateSet) UnmarshalBinaryWithContext(ctx *DecodingContext) (err error) {
  948. // panics are recovered and propagated as errors
  949. defer func() {
  950. if r := recover(); r != nil {
  951. if e, ok := r.(error); ok {
  952. err = e
  953. } else if s, ok := r.(string); ok {
  954. err = fmt.Errorf("unexpected panic: %s", s)
  955. } else {
  956. err = fmt.Errorf("unexpected panic: %+v", r)
  957. }
  958. }
  959. }()
  960. buff := ctx.Buffer
  961. version := buff.ReadUInt8()
  962. if version > DefaultCodecVersion {
  963. return fmt.Errorf("Invalid Version Unmarshalling UpdateSet. Expected %d or less, got %d", DefaultCodecVersion, version)
  964. }
  965. // --- [begin][read][reference](time.Time) ---
  966. a := new(time.Time)
  967. b := buff.ReadInt() // byte array length
  968. c := buff.ReadBytes(b)
  969. errA := a.UnmarshalBinary(c)
  970. if errA != nil {
  971. return errA
  972. }
  973. target.Timestamp = *a
  974. // --- [end][read][reference](time.Time) ---
  975. if buff.ReadUInt8() == uint8(0) {
  976. target.Updates = nil
  977. } else {
  978. // --- [begin][read][slice]([]Update) ---
  979. e := buff.ReadInt() // slice len
  980. d := make([]Update, e)
  981. for i := range e {
  982. // --- [begin][read][struct](Update) ---
  983. g := new(Update)
  984. buff.ReadInt() // [compatibility, unused]
  985. errB := g.UnmarshalBinaryWithContext(ctx)
  986. if errB != nil {
  987. return errB
  988. }
  989. f := *g
  990. // --- [end][read][struct](Update) ---
  991. d[i] = f
  992. }
  993. target.Updates = d
  994. // --- [end][read][slice]([]Update) ---
  995. }
  996. return nil
  997. }
  998. //--------------------------------------------------------------------------
  999. // UpdateSetStream
  1000. //--------------------------------------------------------------------------
  1001. // UpdateSetStream is a single use field stream for the contents of an UpdateSet instance. Instead of creating an instance and populating
  1002. // the fields on that instance, we provide a streaming iterator which yields (BingenFieldInfo, *BingenValue) tuples for each
  1003. // streamable element. All slices and maps will be flattened one depth and each element streamed individually.
  1004. type UpdateSetStream struct {
  1005. reader io.Reader
  1006. ctx *DecodingContext
  1007. err error
  1008. }
  1009. // Closes closes the internal io.Reader used to read and parse the UpdateSet fields.
  1010. // This should be called once the stream is no longer needed.
  1011. func (stream *UpdateSetStream) Close() {
  1012. if closer, ok := stream.reader.(io.Closer); ok {
  1013. closer.Close()
  1014. }
  1015. stream.ctx.Close()
  1016. }
  1017. // Error returns an error if one occurred during the process of streaming the UpdateSet
  1018. // This can be checked after iterating through the Stream().
  1019. func (stream *UpdateSetStream) Error() error {
  1020. return stream.err
  1021. }
  1022. // NewUpdateSetStream creates a new UpdateSetStream, which uses the io.Reader data to stream all internal fields of an UpdateSet instance
  1023. func NewUpdateSetStream(reader io.Reader) BingenStream {
  1024. ctx := NewDecodingContextFromReader(reader)
  1025. return &UpdateSetStream{
  1026. ctx: ctx,
  1027. reader: reader,
  1028. }
  1029. }
  1030. // Stream returns the iterator which will stream each field of the target type.
  1031. func (stream *UpdateSetStream) Stream() iter.Seq2[BingenFieldInfo, *BingenValue] {
  1032. return func(yield func(BingenFieldInfo, *BingenValue) bool) {
  1033. var fi BingenFieldInfo
  1034. ctx := stream.ctx
  1035. buff := ctx.Buffer
  1036. version := buff.ReadUInt8()
  1037. if version > DefaultCodecVersion {
  1038. stream.err = fmt.Errorf("Invalid Version Unmarshalling UpdateSet. Expected %d or less, got %d", DefaultCodecVersion, version)
  1039. return
  1040. }
  1041. fi = BingenFieldInfo{
  1042. Type: reflect.TypeFor[time.Time](),
  1043. Name: "Timestamp",
  1044. }
  1045. // --- [begin][read][reference](time.Time) ---
  1046. b := new(time.Time)
  1047. c := buff.ReadInt() // byte array length
  1048. d := buff.ReadBytes(c)
  1049. errA := b.UnmarshalBinary(d)
  1050. if errA != nil {
  1051. stream.err = errA
  1052. return
  1053. }
  1054. a := *b
  1055. // --- [end][read][reference](time.Time) ---
  1056. if !yield(fi, singleV(a)) {
  1057. return
  1058. }
  1059. fi = BingenFieldInfo{
  1060. Type: reflect.TypeFor[[]Update](),
  1061. Name: "Updates",
  1062. }
  1063. if buff.ReadUInt8() == uint8(0) {
  1064. if !yield(fi, nil) {
  1065. return
  1066. }
  1067. } else {
  1068. // --- [begin][read][streaming-slice]([]Update) ---
  1069. e := buff.ReadInt() // slice len
  1070. for i := range e {
  1071. // --- [begin][read][struct](Update) ---
  1072. g := new(Update)
  1073. buff.ReadInt() // [compatibility, unused]
  1074. errB := g.UnmarshalBinaryWithContext(ctx)
  1075. if errB != nil {
  1076. stream.err = errB
  1077. return
  1078. }
  1079. f := *g
  1080. // --- [end][read][struct](Update) ---
  1081. if !yield(fi, pairV(i, f)) {
  1082. return
  1083. }
  1084. }
  1085. // --- [end][read][streaming-slice]([]Update) ---
  1086. }
  1087. }
  1088. }