|
|
@@ -12,29 +12,23 @@
|
|
|
package metric
|
|
|
|
|
|
import (
|
|
|
- "cmp"
|
|
|
"fmt"
|
|
|
"io"
|
|
|
"iter"
|
|
|
"os"
|
|
|
"reflect"
|
|
|
- "slices"
|
|
|
"strings"
|
|
|
"sync"
|
|
|
"time"
|
|
|
- "unsafe"
|
|
|
|
|
|
- util "github.com/opencost/opencost/core/pkg/util"
|
|
|
+ bstream "github.com/opencost/bingen/pkg/stream"
|
|
|
+ stringtable "github.com/opencost/bingen/pkg/table"
|
|
|
+ util "github.com/opencost/bingen/pkg/util"
|
|
|
)
|
|
|
|
|
|
const (
|
|
|
// GeneratorPackageName is the package the generator is targetting
|
|
|
GeneratorPackageName string = "metric"
|
|
|
- StringHeaderSize = int64(unsafe.Sizeof(""))
|
|
|
-
|
|
|
- // BinaryTagStringTable is written and/or read prior to the existence of a string
|
|
|
- // table (where each index is encoded as a string entry in the resource
|
|
|
- BinaryTagStringTable string = "BGST"
|
|
|
|
|
|
// DefaultCodecVersion is used for any resources listed in the Default version set
|
|
|
DefaultCodecVersion uint8 = 1
|
|
|
@@ -186,7 +180,7 @@ func resolveType(t string) (pkg string, name string, isPtr bool) {
|
|
|
//--------------------------------------------------------------------------
|
|
|
|
|
|
// StreamFactoryFunc is an alias for a func that creates a BingenStream implementation.
|
|
|
-type StreamFactoryFunc func(io.Reader) BingenStream
|
|
|
+type StreamFactoryFunc func(io.Reader) bstream.BingenStream
|
|
|
|
|
|
// Generated streamable factory map for finding the specific new stream methods
|
|
|
// by T type
|
|
|
@@ -197,7 +191,7 @@ var streamFactoryMap map[reflect.Type]StreamFactoryFunc = map[reflect.Type]Strea
|
|
|
// NewStreamFor accepts an io.Reader, and returns a new BingenStream for the generic T
|
|
|
// type provided _if_ it is a registered bingen type that is annotated as 'streamable'. See
|
|
|
// the streamFactoryMap for generated type listings.
|
|
|
-func NewStreamFor[T any](reader io.Reader) (BingenStream, error) {
|
|
|
+func NewStreamFor[T any](reader io.Reader) (bstream.BingenStream, error) {
|
|
|
typeKey := reflect.TypeFor[T]()
|
|
|
|
|
|
factory, ok := streamFactoryMap[typeKey]
|
|
|
@@ -208,432 +202,6 @@ func NewStreamFor[T any](reader io.Reader) (BingenStream, error) {
|
|
|
return factory(reader), nil
|
|
|
}
|
|
|
|
|
|
-// BingenStream is the stream interface for all streamable types
|
|
|
-type BingenStream interface {
|
|
|
- // Stream returns the iterator which will stream each field of the target type and
|
|
|
- // return the field info as well as the value.
|
|
|
- Stream() iter.Seq2[BingenFieldInfo, *BingenValue]
|
|
|
-
|
|
|
- // Close will close any dynamic io.Reader used to stream in the fields
|
|
|
- Close()
|
|
|
-
|
|
|
- // Error returns an error if one occurred during the process of streaming the type's fields.
|
|
|
- // This can be checked after iterating through the Stream().
|
|
|
- Error() error
|
|
|
-}
|
|
|
-
|
|
|
-// BingenValue contains the value of a field as well as any index/key associated with that value.
|
|
|
-type BingenValue struct {
|
|
|
- Value any
|
|
|
- Index any
|
|
|
-}
|
|
|
-
|
|
|
-// IsNil is just a method accessor way to check to see if the value returned was nil
|
|
|
-func (bv *BingenValue) IsNil() bool {
|
|
|
- return bv == nil
|
|
|
-}
|
|
|
-
|
|
|
-// creates a single BingenValue instance without a key or index
|
|
|
-func singleV(value any) *BingenValue {
|
|
|
- return &BingenValue{
|
|
|
- Value: value,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// creates a pair of key/index and value.
|
|
|
-func pairV(index any, value any) *BingenValue {
|
|
|
- return &BingenValue{
|
|
|
- Value: value,
|
|
|
- Index: index,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// BingenFieldInfo contains the type of the field being streamed as well as the name of the field.
|
|
|
-type BingenFieldInfo struct {
|
|
|
- Type reflect.Type
|
|
|
- Name string
|
|
|
-}
|
|
|
-
|
|
|
-//--------------------------------------------------------------------------
|
|
|
-// String Table Writer
|
|
|
-//--------------------------------------------------------------------------
|
|
|
-
|
|
|
-// StringTableWriter is the interface used to write the string table for encoding.
|
|
|
-type StringTableWriter interface {
|
|
|
- // AddOrGet adds a string to the string table and returns the new index or
|
|
|
- // an existing index.
|
|
|
- AddOrGet(s string) int
|
|
|
-
|
|
|
- // WriteTo will write the StringTable data (with the header) to the provided
|
|
|
- // Buffer starting a the current write position
|
|
|
- WriteTo(b *util.Buffer)
|
|
|
-}
|
|
|
-
|
|
|
-// IndexedStringTableWriter maps strings to specific indices for encoding
|
|
|
-type IndexedStringTableWriter struct {
|
|
|
- indices map[string]int
|
|
|
- next int
|
|
|
-}
|
|
|
-
|
|
|
-// NewIndexedStringTableWriter Creates a new IndexedStringTableWriter instance.
|
|
|
-func NewIndexedStringTableWriter() *IndexedStringTableWriter {
|
|
|
- return &IndexedStringTableWriter{
|
|
|
- indices: make(map[string]int),
|
|
|
- next: 0,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// AddOrGet retrieves a string entry's index if it exists. Otherwise, it adds the entry and returns the new index.
|
|
|
-func (st *IndexedStringTableWriter) AddOrGet(s string) int {
|
|
|
- if ind, ok := st.indices[s]; ok {
|
|
|
- return ind
|
|
|
- }
|
|
|
-
|
|
|
- current := st.next
|
|
|
- st.next++
|
|
|
-
|
|
|
- st.indices[s] = current
|
|
|
- return current
|
|
|
-}
|
|
|
-
|
|
|
-// ToSlice Converts the contents to a string array for encoding.
|
|
|
-func (st *IndexedStringTableWriter) ToSlice() []string {
|
|
|
- if st.next == 0 {
|
|
|
- return []string{}
|
|
|
- }
|
|
|
-
|
|
|
- sl := make([]string, st.next)
|
|
|
- for s, i := range st.indices {
|
|
|
- sl[i] = s
|
|
|
- }
|
|
|
- return sl
|
|
|
-}
|
|
|
-
|
|
|
-// ToBytes Converts the contents to a binary encoded representation
|
|
|
-func (st *IndexedStringTableWriter) ToBytes() []byte {
|
|
|
- buff := util.NewBuffer()
|
|
|
- st.WriteTo(buff)
|
|
|
- return buff.Bytes()
|
|
|
-}
|
|
|
-
|
|
|
-// WriteTo will write the StringTable data (with the header) to the provided
|
|
|
-// Buffer starting a the current write position
|
|
|
-func (st *IndexedStringTableWriter) WriteTo(buff *util.Buffer) {
|
|
|
- // bingen string table header
|
|
|
- buff.WriteBytes([]byte(BinaryTagStringTable))
|
|
|
-
|
|
|
- // get an ordered string slice to encode
|
|
|
- strs := st.ToSlice()
|
|
|
-
|
|
|
- buff.WriteInt(len(strs)) // table length
|
|
|
- for _, s := range strs {
|
|
|
- buff.WriteString(s)
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-type indexed struct {
|
|
|
- s string
|
|
|
- count uint64
|
|
|
- index int
|
|
|
-}
|
|
|
-
|
|
|
-func newIndexed(s string, index int) *indexed {
|
|
|
- return &indexed{
|
|
|
- s: s,
|
|
|
- count: 1,
|
|
|
- index: index,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// PrepassStringTableWriter maps strings to specific indices for encoding, sorted by the total
|
|
|
-// number of times they're accessed
|
|
|
-type PrepassStringTableWriter struct {
|
|
|
- prepass map[string]*indexed
|
|
|
- next int
|
|
|
-}
|
|
|
-
|
|
|
-// NewPrepassStringTableWriter creates a new PrepassStringTableWriter instance.
|
|
|
-func NewPrepassStringTableWriter() *PrepassStringTableWriter {
|
|
|
- return &PrepassStringTableWriter{
|
|
|
- prepass: make(map[string]*indexed),
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// AddOrGet retrieves a string entry's index if it exists. Otherwise, it adds the entry and returns the new index.
|
|
|
-func (st *PrepassStringTableWriter) AddOrGet(s string) int {
|
|
|
- if ind, ok := st.prepass[s]; ok {
|
|
|
- ind.count += 1
|
|
|
- return ind.index
|
|
|
- }
|
|
|
-
|
|
|
- current := st.next
|
|
|
- st.next++
|
|
|
-
|
|
|
- st.prepass[s] = newIndexed(s, current)
|
|
|
- return current
|
|
|
-}
|
|
|
-
|
|
|
-// WriteSortedTo sorts the string table by the number of accesses, writes the table in that
|
|
|
-// order, then returns a new StringTableWriter implementation that can be used for the new
|
|
|
-// sorted order index lookups.
|
|
|
-func (st *PrepassStringTableWriter) WriteSortedTo(buff *util.Buffer) StringTableWriter {
|
|
|
- sl := make([]*indexed, st.next)
|
|
|
- for _, ind := range st.prepass {
|
|
|
- sl[ind.index] = ind
|
|
|
- }
|
|
|
-
|
|
|
- slices.SortFunc(sl, func(a *indexed, b *indexed) int {
|
|
|
- return -cmp.Compare(a.count, b.count)
|
|
|
- })
|
|
|
-
|
|
|
- sti := NewIndexedStringTableWriter()
|
|
|
- for _, ind := range sl {
|
|
|
- sti.AddOrGet(ind.s)
|
|
|
- }
|
|
|
-
|
|
|
- sti.WriteTo(buff)
|
|
|
- return sti
|
|
|
-}
|
|
|
-
|
|
|
-// WriteTo will write the StringTable data (with the header) to the provided
|
|
|
-// Buffer starting a the current write position
|
|
|
-func (st *PrepassStringTableWriter) WriteTo(buff *util.Buffer) {
|
|
|
- panic("Prepass StringTableWriter cannot write directly")
|
|
|
-}
|
|
|
-
|
|
|
-//--------------------------------------------------------------------------
|
|
|
-// String Table Reader
|
|
|
-//--------------------------------------------------------------------------
|
|
|
-
|
|
|
-// StringTableReader is the interface used to read the string table from the decoding.
|
|
|
-type StringTableReader interface {
|
|
|
- // At returns the string entry at a specific index, or panics on out of bounds.
|
|
|
- At(index int) string
|
|
|
-
|
|
|
- // Len returns the total number of strings loaded in the string table.
|
|
|
- Len() int
|
|
|
-
|
|
|
- // Close will clear the loaded table, and drop any external resources used.
|
|
|
- Close() error
|
|
|
-}
|
|
|
-
|
|
|
-// SliceStringTableReader is a basic pre-loaded []string that provides index-based access.
|
|
|
-// The cost of this implementation is holding all strings in memory, which provides faster
|
|
|
-// lookup performance at the expense of memory usage.
|
|
|
-type SliceStringTableReader struct {
|
|
|
- table []string
|
|
|
-}
|
|
|
-
|
|
|
-// NewSliceStringTableReaderFrom creates a new SliceStringTableReader instance loading
|
|
|
-// data directly from the buffer. The buffer's position should start at the table length.
|
|
|
-func NewSliceStringTableReaderFrom(buffer *util.Buffer) StringTableReader {
|
|
|
- // table length
|
|
|
- tl := buffer.ReadInt()
|
|
|
-
|
|
|
- var table []string
|
|
|
- if tl > 0 {
|
|
|
- table = make([]string, tl)
|
|
|
- for i := range tl {
|
|
|
- table[i] = buffer.ReadString()
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return &SliceStringTableReader{
|
|
|
- table: table,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// At returns the string entry at a specific index, or panics on out of bounds.
|
|
|
-func (sstr *SliceStringTableReader) At(index int) string {
|
|
|
- if index < 0 || index >= len(sstr.table) {
|
|
|
- panic(fmt.Errorf("%s: string table index out of bounds: %d", GeneratorPackageName, index))
|
|
|
- }
|
|
|
-
|
|
|
- return sstr.table[index]
|
|
|
-}
|
|
|
-
|
|
|
-// Len returns the total number of strings loaded in the string table.
|
|
|
-func (sstr *SliceStringTableReader) Len() int {
|
|
|
- if sstr == nil {
|
|
|
- return 0
|
|
|
- }
|
|
|
-
|
|
|
- return len(sstr.table)
|
|
|
-}
|
|
|
-
|
|
|
-// Close for the slice tables just nils out the slice and returns
|
|
|
-func (sstr *SliceStringTableReader) Close() error {
|
|
|
- sstr.table = nil
|
|
|
- return nil
|
|
|
-}
|
|
|
-
|
|
|
-// fileStringRef maps a bingen string-table index to a payload stored in a temp file.
|
|
|
-type fileStringRef struct {
|
|
|
- off int64
|
|
|
- length int
|
|
|
-}
|
|
|
-
|
|
|
-// FileStringTableReader leverages a local file to write string table data for lookup. On
|
|
|
-// memory focused systems, this allows a slower parse with a significant decrease in memory
|
|
|
-// usage. This implementation is often pair with streaming readers for high throughput with
|
|
|
-// reduced memory usage.
|
|
|
-type FileStringTableReader struct {
|
|
|
- f *os.File
|
|
|
- refs []fileStringRef
|
|
|
- memo []string
|
|
|
-}
|
|
|
-
|
|
|
-// NewFileStringTableFromBuffer reads exactly tl length-prefixed (uint16) string payloads from buffer
|
|
|
-// and appends each payload to a new temp file. It does not retain full strings in memory.
|
|
|
-func NewFileStringTableReaderFrom(buffer *util.Buffer, dir string, memoMaxBytes int64) StringTableReader {
|
|
|
- // helper func to cast a string in-place to a byte slice.
|
|
|
- // NOTE: Return value is READ-ONLY. DO NOT MODIFY!
|
|
|
- byteSliceFor := func(s string) []byte {
|
|
|
- return unsafe.Slice(unsafe.StringData(s), len(s))
|
|
|
- }
|
|
|
-
|
|
|
- err := os.MkdirAll(dir, 0755)
|
|
|
- if err != nil {
|
|
|
- panic(fmt.Errorf("%s: failed to create string table directory: %w", GeneratorPackageName, err))
|
|
|
- }
|
|
|
-
|
|
|
- f, err := os.CreateTemp(dir, fmt.Sprintf("%s-bgst-*", GeneratorPackageName))
|
|
|
- if err != nil {
|
|
|
- panic(fmt.Errorf("%s: failed to create string table file: %w", GeneratorPackageName, err))
|
|
|
- }
|
|
|
-
|
|
|
- var writeErr error
|
|
|
- defer func() {
|
|
|
- if writeErr != nil {
|
|
|
- _ = f.Close()
|
|
|
- }
|
|
|
- }()
|
|
|
-
|
|
|
- // table length
|
|
|
- tl := buffer.ReadInt()
|
|
|
-
|
|
|
- var refs []fileStringRef
|
|
|
- if tl > 0 {
|
|
|
- refs = make([]fileStringRef, tl)
|
|
|
-
|
|
|
- for i := range tl {
|
|
|
- payload := byteSliceFor(buffer.ReadString())
|
|
|
-
|
|
|
- var off int64
|
|
|
- if len(payload) > 0 {
|
|
|
- off, err = f.Seek(0, io.SeekEnd)
|
|
|
- if err != nil {
|
|
|
- writeErr = fmt.Errorf("%s: failed to seek string table file: %w", GeneratorPackageName, err)
|
|
|
- panic(writeErr)
|
|
|
- }
|
|
|
- if _, err := f.Write(payload); err != nil {
|
|
|
- writeErr = fmt.Errorf("%s: failed to write string table entry %d: %w", GeneratorPackageName, i, err)
|
|
|
- panic(writeErr)
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- refs[i] = fileStringRef{
|
|
|
- off: off,
|
|
|
- length: len(payload),
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- var memo []string
|
|
|
-
|
|
|
- // Pre-load cache with strings up to memoMaxBytes, respecting string boundaries
|
|
|
- if memoMaxBytes > 0 && len(refs) > 0 {
|
|
|
- memo = make([]string, len(refs))
|
|
|
- var cumulativeSize int64
|
|
|
- for i, ref := range refs {
|
|
|
- // Check if adding this string would exceed the limit
|
|
|
- if cumulativeSize+int64(ref.length)+StringHeaderSize > memoMaxBytes {
|
|
|
- // Would exceed limit, stop here
|
|
|
- break
|
|
|
- }
|
|
|
-
|
|
|
- // Read string from file and cache it
|
|
|
- if ref.length > 0 {
|
|
|
- b := make([]byte, ref.length)
|
|
|
- _, err := f.ReadAt(b, ref.off)
|
|
|
- if err != nil {
|
|
|
- // If we can't read, skip this entry but continue
|
|
|
- continue
|
|
|
- }
|
|
|
-
|
|
|
- // Cast the allocated bytes to a string in-place
|
|
|
- str := unsafe.String(unsafe.SliceData(b), len(b))
|
|
|
- memo[i] = str
|
|
|
- cumulativeSize += int64(ref.length) + StringHeaderSize
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return &FileStringTableReader{
|
|
|
- f: f,
|
|
|
- refs: refs,
|
|
|
- memo: memo,
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// At returns the string from the internal file using the reference's offset and length.
|
|
|
-func (fstr *FileStringTableReader) At(index int) string {
|
|
|
- if fstr == nil || fstr.f == nil {
|
|
|
- panic(fmt.Errorf("%s: failed to read file string table data", GeneratorPackageName))
|
|
|
- }
|
|
|
- if index < 0 || index >= len(fstr.refs) {
|
|
|
- panic(fmt.Errorf("%s: string table index out of bounds: %d", GeneratorPackageName, index))
|
|
|
- }
|
|
|
-
|
|
|
- ref := fstr.refs[index]
|
|
|
- if ref.length == 0 {
|
|
|
- return ""
|
|
|
- }
|
|
|
-
|
|
|
- // Check cache first
|
|
|
- if fstr.memo != nil && len(fstr.memo) > index && fstr.memo[index] != "" {
|
|
|
- return fstr.memo[index]
|
|
|
- }
|
|
|
-
|
|
|
- // Cache miss - read from file
|
|
|
- b := make([]byte, ref.length)
|
|
|
- _, err := fstr.f.ReadAt(b, ref.off)
|
|
|
- if err != nil {
|
|
|
- return ""
|
|
|
- }
|
|
|
-
|
|
|
- // Cast the allocated bytes to a string in-place, as we were the ones that allocated the bytes
|
|
|
- return unsafe.String(unsafe.SliceData(b), len(b))
|
|
|
-}
|
|
|
-
|
|
|
-// Len returns the total number of strings loaded in the string table.
|
|
|
-func (fstr *FileStringTableReader) Len() int {
|
|
|
- if fstr == nil {
|
|
|
- return 0
|
|
|
- }
|
|
|
-
|
|
|
- return len(fstr.refs)
|
|
|
-}
|
|
|
-
|
|
|
-// Close for the file string table reader closes the file and deletes it.
|
|
|
-func (fstr *FileStringTableReader) Close() error {
|
|
|
- if fstr == nil || fstr.f == nil {
|
|
|
- return nil
|
|
|
- }
|
|
|
-
|
|
|
- path := fstr.f.Name()
|
|
|
- err := fstr.f.Close()
|
|
|
- fstr.f = nil
|
|
|
- fstr.refs = nil
|
|
|
- fstr.memo = nil
|
|
|
-
|
|
|
- if path != "" {
|
|
|
- _ = os.Remove(path)
|
|
|
- }
|
|
|
-
|
|
|
- return err
|
|
|
-}
|
|
|
-
|
|
|
//--------------------------------------------------------------------------
|
|
|
// Codec Context
|
|
|
//--------------------------------------------------------------------------
|
|
|
@@ -642,12 +210,12 @@ func (fstr *FileStringTableReader) Close() error {
|
|
|
// and table data
|
|
|
type EncodingContext struct {
|
|
|
Buffer *util.Buffer
|
|
|
- Table StringTableWriter
|
|
|
+ Table stringtable.StringTableWriter
|
|
|
}
|
|
|
|
|
|
// NewEncodingContext creates a new EncodingContext instance that will create a new []byte buffer
|
|
|
// for writing, and return the context
|
|
|
-func NewEncodingContext(tableWriter StringTableWriter) *EncodingContext {
|
|
|
+func NewEncodingContext(tableWriter stringtable.StringTableWriter) *EncodingContext {
|
|
|
return &EncodingContext{
|
|
|
Buffer: util.NewBuffer(),
|
|
|
Table: tableWriter,
|
|
|
@@ -656,7 +224,7 @@ func NewEncodingContext(tableWriter StringTableWriter) *EncodingContext {
|
|
|
|
|
|
// NewEncodingContextFromWriter creates a new EncodingContext instance that will create a new Buffer
|
|
|
// from the provided io.Writer and StringTableWriter.
|
|
|
-func NewEncodingContextFromWriter(writer io.Writer, tableWriter StringTableWriter) *EncodingContext {
|
|
|
+func NewEncodingContextFromWriter(writer io.Writer, tableWriter stringtable.StringTableWriter) *EncodingContext {
|
|
|
return &EncodingContext{
|
|
|
Buffer: util.NewBufferFromWriter(writer),
|
|
|
Table: tableWriter,
|
|
|
@@ -665,7 +233,7 @@ func NewEncodingContextFromWriter(writer io.Writer, tableWriter StringTableWrite
|
|
|
|
|
|
// NewEncodingContextFromBuffer creates a new EncodingContext instance that will leverage an existing
|
|
|
// Buffer and StringTableWriter.
|
|
|
-func NewEncodingContextFromBuffer(buffer *util.Buffer, tableWriter StringTableWriter) *EncodingContext {
|
|
|
+func NewEncodingContextFromBuffer(buffer *util.Buffer, tableWriter stringtable.StringTableWriter) *EncodingContext {
|
|
|
return &EncodingContext{
|
|
|
Buffer: buffer,
|
|
|
Table: tableWriter,
|
|
|
@@ -696,22 +264,22 @@ func (ec *EncodingContext) IsStringTable() bool {
|
|
|
// reuse as much data as possible
|
|
|
type DecodingContext struct {
|
|
|
Buffer *util.Buffer
|
|
|
- Table StringTableReader
|
|
|
+ Table stringtable.StringTableReader
|
|
|
}
|
|
|
|
|
|
// NewDecodingContextFromBytes creates a new DecodingContext instance using an byte slice
|
|
|
func NewDecodingContextFromBytes(data []byte) *DecodingContext {
|
|
|
- var table StringTableReader
|
|
|
+ var table stringtable.StringTableReader
|
|
|
|
|
|
buff := util.NewBufferFromBytes(data)
|
|
|
|
|
|
// string table header validation
|
|
|
- if isBinaryTag(data, BinaryTagStringTable) {
|
|
|
- buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
|
|
|
+ if isBinaryTag(data, stringtable.BinaryTagStringTable) {
|
|
|
+ buff.ReadBytes(len(stringtable.BinaryTagStringTable)) // strip tag length
|
|
|
|
|
|
// always use a slice string table with a byte array since the
|
|
|
// data is already in memory
|
|
|
- table = NewSliceStringTableReaderFrom(buff)
|
|
|
+ table = stringtable.NewSliceStringTableReaderFrom(buff)
|
|
|
}
|
|
|
|
|
|
return &DecodingContext{
|
|
|
@@ -723,18 +291,18 @@ func NewDecodingContextFromBytes(data []byte) *DecodingContext {
|
|
|
// NewDecodingContextFromReader creates a new DecodingContext instance using an io.Reader
|
|
|
// implementation
|
|
|
func NewDecodingContextFromReader(reader io.Reader) *DecodingContext {
|
|
|
- var table StringTableReader
|
|
|
+ var table stringtable.StringTableReader
|
|
|
|
|
|
buff := util.NewBufferFromReader(reader)
|
|
|
|
|
|
- if isReaderBinaryTag(buff, BinaryTagStringTable) {
|
|
|
- buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
|
|
|
+ if isReaderBinaryTag(buff, stringtable.BinaryTagStringTable) {
|
|
|
+ buff.ReadBytes(len(stringtable.BinaryTagStringTable)) // strip tag length
|
|
|
|
|
|
// create correct string table implementation
|
|
|
if IsBingenFileBackedStringTableEnabled() {
|
|
|
- table = NewFileStringTableReaderFrom(buff, BingenFileBackedStringTableDir(), BingenFileBackedStringTableMemoMaxBytes())
|
|
|
+ table = stringtable.NewFileStringTableReaderFrom(buff, BingenFileBackedStringTableDir(), GeneratorPackageName, BingenFileBackedStringTableMemoMaxBytes())
|
|
|
} else {
|
|
|
- table = NewSliceStringTableReaderFrom(buff)
|
|
|
+ table = stringtable.NewSliceStringTableReaderFrom(buff)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -1031,7 +599,7 @@ func (target *Update) UnmarshalBinaryWithContext(ctx *DecodingContext) (err erro
|
|
|
// MarshalBinary serializes the internal properties of this UpdateSet instance
|
|
|
// into a byte array
|
|
|
func (target *UpdateSet) MarshalBinary() (data []byte, err error) {
|
|
|
- ctx := NewEncodingContext(NewIndexedStringTableWriter())
|
|
|
+ ctx := NewEncodingContext(stringtable.NewIndexedStringTableWriter())
|
|
|
|
|
|
e := target.MarshalBinaryWithContext(ctx)
|
|
|
if e != nil {
|
|
|
@@ -1050,7 +618,7 @@ func (target *UpdateSet) MarshalBinaryTo(writer io.Writer) error {
|
|
|
// run a pre-pass to collect all strings into the string table and discard all writes to the main
|
|
|
// buffer. Then, we write the string table, sorted by number of repeated uses (descending), to the
|
|
|
// main buffer, and use the resulting table as part of the context for the main pass.
|
|
|
- prepass := NewPrepassStringTableWriter()
|
|
|
+ prepass := stringtable.NewPrepassStringTableWriter()
|
|
|
prepassCtx := NewEncodingContextFromWriter(io.Discard, prepass)
|
|
|
|
|
|
e := target.MarshalBinaryWithContext(prepassCtx)
|
|
|
@@ -1236,7 +804,7 @@ func (stream *UpdateSetStream) Error() error {
|
|
|
}
|
|
|
|
|
|
// NewUpdateSetStream creates a new UpdateSetStream, which uses the io.Reader data to stream all internal fields of an UpdateSet instance
|
|
|
-func NewUpdateSetStream(reader io.Reader) BingenStream {
|
|
|
+func NewUpdateSetStream(reader io.Reader) bstream.BingenStream {
|
|
|
ctx := NewDecodingContextFromReader(reader)
|
|
|
|
|
|
return &UpdateSetStream{
|
|
|
@@ -1246,9 +814,9 @@ func NewUpdateSetStream(reader io.Reader) BingenStream {
|
|
|
}
|
|
|
|
|
|
// Stream returns the iterator which will stream each field of the target type.
|
|
|
-func (stream *UpdateSetStream) Stream() iter.Seq2[BingenFieldInfo, *BingenValue] {
|
|
|
- return func(yield func(BingenFieldInfo, *BingenValue) bool) {
|
|
|
- var fi BingenFieldInfo
|
|
|
+func (stream *UpdateSetStream) Stream() iter.Seq2[bstream.BingenFieldInfo, *bstream.BingenValue] {
|
|
|
+ return func(yield func(bstream.BingenFieldInfo, *bstream.BingenValue) bool) {
|
|
|
+ var fi bstream.BingenFieldInfo
|
|
|
|
|
|
ctx := stream.ctx
|
|
|
buff := ctx.Buffer
|
|
|
@@ -1259,7 +827,7 @@ func (stream *UpdateSetStream) Stream() iter.Seq2[BingenFieldInfo, *BingenValue]
|
|
|
return
|
|
|
}
|
|
|
|
|
|
- fi = BingenFieldInfo{
|
|
|
+ fi = bstream.BingenFieldInfo{
|
|
|
Type: reflect.TypeFor[time.Time](),
|
|
|
Name: "Timestamp",
|
|
|
}
|
|
|
@@ -1276,11 +844,11 @@ func (stream *UpdateSetStream) Stream() iter.Seq2[BingenFieldInfo, *BingenValue]
|
|
|
}
|
|
|
a := *b
|
|
|
// --- [end][read][reference](time.Time) ---
|
|
|
- if !yield(fi, singleV(a)) {
|
|
|
+ if !yield(fi, bstream.SingleV(a)) {
|
|
|
return
|
|
|
}
|
|
|
|
|
|
- fi = BingenFieldInfo{
|
|
|
+ fi = bstream.BingenFieldInfo{
|
|
|
Type: reflect.TypeFor[[]Update](),
|
|
|
Name: "Updates",
|
|
|
}
|
|
|
@@ -1305,7 +873,7 @@ func (stream *UpdateSetStream) Stream() iter.Seq2[BingenFieldInfo, *BingenValue]
|
|
|
f := *g
|
|
|
// --- [end][read][struct](Update) ---
|
|
|
|
|
|
- if !yield(fi, pairV(i, f)) {
|
|
|
+ if !yield(fi, bstream.PairV(i, f)) {
|
|
|
return
|
|
|
}
|
|
|
}
|
Matt Bolt