Remove buffer and stringbank from opencost, replace uses with bingen utils.

core/pkg/exporter/decoder_test.go

@@ -12,7 +12,8 @@ import (
 	"github.com/opencost/opencost/core/pkg/model/pb"
 	"github.com/opencost/opencost/core/pkg/opencost"
 	"github.com/opencost/opencost/core/pkg/storage"
-	"github.com/opencost/opencost/core/pkg/util"
+
+	"github.com/opencost/bingen/pkg/util"
 	"github.com/opencost/opencost/core/pkg/util/json"
 	"google.golang.org/protobuf/proto"
 )

core/pkg/util/buffer.go

@@ -1,624 +0,0 @@
-package util
-
-import (
-	"bufio"
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"os"
-	"unsafe"
-
-	"github.com/opencost/opencost/core/pkg/util/stringutil"
-)
-
-var bytePool *bufferPool = newBufferPool()
-
-// NonPrimitiveTypeError represents an error where the user provided a non-primitive data type for reading/writing
-var NonPrimitiveTypeError error = errors.New("Type provided to read/write does not fit inside 8 bytes.")
-
-// Mode is used to represent the 3 possible states of the buffer. note there is
-// no overlapping between states, as each Mode is handled exclusively.
-type Mode uint8
-
-const (
-	ReadWrite Mode = iota
-	ReadOnly
-	WriteOnly
-)
-
-// Buffer is a utility type which implements a very basic binary protocol for
-// writing core go types. It can run as read-only, write-only, or read-write.
-type Buffer struct {
-	r  *bufio.Reader
-	w  *bufio.Writer
-	rw *bytes.Buffer
-	m  Mode
-}
-
-// NewBuffer creates a new Buffer instance using LittleEndian ByteOrder.
-func NewBuffer() *Buffer {
-	return &Buffer{
-		r:  nil,
-		w:  nil,
-		rw: new(bytes.Buffer),
-		m:  ReadWrite,
-	}
-}
-
-// NewBufferFromBytes creates a new read/write Buffer instance using the provided byte slice.
-// The new buffer assumes ownership of the byte slice.
-func NewBufferFromBytes(b []byte) *Buffer {
-	return &Buffer{
-		r:  nil,
-		w:  nil,
-		rw: bytes.NewBuffer(b),
-		m:  ReadWrite,
-	}
-}
-
-// NewBufferFrom creates a new Buffer instance using the remaining unread data from the
-// provided Buffer instance. The new buffer assumes ownership of the underlying data.
-func NewBufferFrom(b *Buffer) *Buffer {
-	bb := b.Bytes()
-	return &Buffer{
-		r:  nil,
-		w:  nil,
-		rw: bytes.NewBuffer(bb),
-		m:  ReadWrite,
-	}
-}
-
-// NewBufferFromReader creates a new Buffer instance using the provided io.Reader. This
-// buffer is set to read-only.
-func NewBufferFromReader(reader io.Reader) *Buffer {
-	return &Buffer{
-		r:  bufio.NewReader(reader),
-		w:  nil,
-		rw: nil,
-		m:  ReadOnly,
-	}
-}
-
-// NewBufferFromWriter creates a new Buffer instance using the provided io.Writer. This
-// buffer is set to write-only.
-func NewBufferFromWriter(writer io.Writer) *Buffer {
-	return &Buffer{
-		r:  nil,
-		w:  bufio.NewWriter(writer),
-		rw: nil,
-		m:  WriteOnly,
-	}
-}
-
-// WriteBool writes a bool value to the buffer
-func (b *Buffer) WriteBool(i bool) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeBool(b.rw, i)
-		return
-	}
-
-	writeBuffBool(b.w, i)
-}
-
-// WriteInt writes an int value to the buffer.
-func (b *Buffer) WriteInt(i int) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeInt(b.rw, i)
-		return
-	}
-
-	writeBuffInt(b.w, i)
-}
-
-// WriteInt8 writes an int8 value to the buffer.
-func (b *Buffer) WriteInt8(i int8) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeInt8(b.rw, i)
-		return
-	}
-
-	writeBuffInt8(b.w, i)
-}
-
-// WriteInt16 writes an int16 value to the buffer.
-func (b *Buffer) WriteInt16(i int16) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeInt16(b.rw, i)
-		return
-	}
-
-	writeBuffInt16(b.w, i)
-}
-
-// WriteInt32 writes an int32 value to the buffer.
-func (b *Buffer) WriteInt32(i int32) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeInt32(b.rw, i)
-		return
-	}
-
-	writeBuffInt32(b.w, i)
-}
-
-// WriteInt64 writes an int64 value to the buffer.
-func (b *Buffer) WriteInt64(i int64) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeInt64(b.rw, i)
-		return
-	}
-
-	writeBuffInt64(b.w, i)
-}
-
-// WriteUInt writes a uint value to the buffer.
-func (b *Buffer) WriteUInt(i uint) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeUint(b.rw, i)
-		return
-	}
-
-	writeBuffUint(b.w, i)
-}
-
-// WriteUInt8 writes a uint8 value to the buffer.
-func (b *Buffer) WriteUInt8(i uint8) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeUint8(b.rw, i)
-		return
-	}
-
-	writeBuffUint8(b.w, i)
-}
-
-// WriteUInt16 writes a uint16 value to the buffer.
-func (b *Buffer) WriteUInt16(i uint16) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeUint16(b.rw, i)
-		return
-	}
-
-	writeBuffUint16(b.w, i)
-}
-
-// WriteUInt32 writes a uint32 value to the buffer.
-func (b *Buffer) WriteUInt32(i uint32) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeUint32(b.rw, i)
-		return
-	}
-
-	writeBuffUint32(b.w, i)
-}
-
-// WriteUInt64 writes a uint64 value to the buffer.
-func (b *Buffer) WriteUInt64(i uint64) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeUint64(b.rw, i)
-		return
-	}
-
-	writeBuffUint64(b.w, i)
-}
-
-// WriteFloat32 writes a float32 value to the buffer.
-func (b *Buffer) WriteFloat32(i float32) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeFloat32(b.rw, i)
-		return
-	}
-
-	writeBuffFloat32(b.w, i)
-}
-
-// WriteFloat64 writes a float64 value to the buffer.
-func (b *Buffer) WriteFloat64(i float64) {
-	b.checkRO()
-
-	if b.rw != nil {
-		writeFloat64(b.rw, i)
-		return
-	}
-
-	writeBuffFloat64(b.w, i)
-}
-
-// WriteString writes the string's length as a uint16 followed by the string contents.
-func (b *Buffer) WriteString(i string) {
-	b.checkRO()
-	s := stringToBytes(i)
-
-	// string lengths are limited to uint16 - See ReadString()
-	if len(s) > math.MaxUint16 {
-		s = s[:math.MaxUint16]
-	}
-
-	l := uint16(len(s))
-
-	if b.rw != nil {
-		writeUint16(b.rw, l)
-		b.rw.Write(s)
-		return
-	}
-
-	writeBuffUint16(b.w, l)
-	b.w.Write(s)
-}
-
-// WriteBytes writes the contents of the byte slice to the buffer.
-func (b *Buffer) WriteBytes(bytes []byte) {
-	b.checkRO()
-
-	if b.rw != nil {
-		b.rw.Write(bytes)
-		return
-	}
-
-	b.w.Write(bytes)
-}
-
-// Bytes returns the unread portion of the underlying buffer storage. If the buffer was
-// created with an `io.Reader`, then the remaining unread bytes are drained into a byte
-// slice and returned.
-func (b *Buffer) Bytes() []byte {
-	b.checkWO()
-
-	if b.rw != nil {
-		return b.rw.Bytes()
-	}
-
-	bytes, err := io.ReadAll(b.r)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "failed to read remaining bytes from Buffer: %s\n", err)
-	}
-	return bytes
-}
-
-// Peek will attempt to peek ahead if the buffer is in read-only mode.
-func (b *Buffer) Peek(length int) ([]byte, error) {
-	b.checkWO()
-
-	if b.rw != nil {
-		return nil, fmt.Errorf("unsupported Peek() operation on read/write buffer.")
-	}
-
-	return b.r.Peek(length)
-}
-
-// Flush will attempt to flush any pending writes if the buffer is in write-only mode.
-func (b *Buffer) Flush() {
-	if b.IsWriteOnly() {
-		if err := b.w.Flush(); err != nil {
-			fmt.Fprintf(os.Stderr, "Flushing io.Writer failed: %s\n", err)
-		}
-	}
-}
-
-// this should be inlined
-func (b *Buffer) checkRO() {
-	if b.IsReadOnly() {
-		panic("Tried to write to a Buffer that is set to read-only")
-	}
-}
-
-func (b *Buffer) checkWO() {
-	if b.IsWriteOnly() {
-		panic("Tried to read from a Buffer that is set to write-only")
-	}
-}
-
-// IsReadOnly returns true if the buffer is set to only read mode.
-func (b *Buffer) IsReadOnly() bool {
-	return b.m == ReadOnly
-}
-
-// IsWriteOnly returns true if the buffer is set to only write mode.
-func (b *Buffer) IsWriteOnly() bool {
-	return b.m == WriteOnly
-}
-
-// IsReadWrite returns true if the buffer can be written to and read from.
-func (b *Buffer) IsReadWrite() bool {
-	return b.m == ReadWrite
-}
-
-// ReadBool reads a bool value from the buffer.
-func (b *Buffer) ReadBool() bool {
-	b.checkWO()
-
-	var i bool
-	if b.rw != nil {
-		readBool(b.rw, &i)
-		return i
-	}
-
-	readBuffBool(b.r, &i)
-	return i
-}
-
-// ReadInt reads an int value from the buffer.
-func (b *Buffer) ReadInt() int {
-	b.checkWO()
-
-	var i int
-	if b.rw != nil {
-		readInt(b.rw, &i)
-		return i
-	}
-
-	readBuffInt(b.r, &i)
-	return i
-}
-
-// ReadInt8 reads an int8 value from the buffer.
-func (b *Buffer) ReadInt8() int8 {
-	b.checkWO()
-
-	var i int8
-	if b.rw != nil {
-		readInt8(b.rw, &i)
-		return i
-	}
-
-	readBuffInt8(b.r, &i)
-	return i
-}
-
-// ReadInt16 reads an int16 value from the buffer.
-func (b *Buffer) ReadInt16() int16 {
-	b.checkWO()
-
-	var i int16
-	if b.rw != nil {
-		readInt16(b.rw, &i)
-		return i
-	}
-
-	readBuffInt16(b.r, &i)
-	return i
-}
-
-// ReadInt32 reads an int32 value from the buffer.
-func (b *Buffer) ReadInt32() int32 {
-	b.checkWO()
-
-	var i int32
-	if b.rw != nil {
-		readInt32(b.rw, &i)
-		return i
-	}
-
-	readBuffInt32(b.r, &i)
-	return i
-}
-
-// ReadInt64 reads an int64 value from the buffer.
-func (b *Buffer) ReadInt64() int64 {
-	b.checkWO()
-
-	var i int64
-	if b.rw != nil {
-		readInt64(b.rw, &i)
-		return i
-	}
-
-	readBuffInt64(b.r, &i)
-	return i
-}
-
-// ReadUInt reads a uint value from the buffer.
-func (b *Buffer) ReadUInt() uint {
-	b.checkWO()
-
-	var i uint
-	if b.rw != nil {
-		readUint(b.rw, &i)
-		return i
-	}
-
-	readBuffUint(b.r, &i)
-	return i
-}
-
-// ReadUInt8 reads a uint8 value from the buffer.
-func (b *Buffer) ReadUInt8() uint8 {
-	b.checkWO()
-
-	var i uint8
-	if b.rw != nil {
-		readUint8(b.rw, &i)
-		return i
-	}
-
-	readBuffUint8(b.r, &i)
-	return i
-}
-
-// ReadUInt16 reads a uint16 value from the buffer.
-func (b *Buffer) ReadUInt16() uint16 {
-	b.checkWO()
-
-	var i uint16
-	if b.rw != nil {
-		readUint16(b.rw, &i)
-		return i
-	}
-
-	readBuffUint16(b.r, &i)
-	return i
-}
-
-// ReadUInt32 reads a uint32 value from the buffer.
-func (b *Buffer) ReadUInt32() uint32 {
-	b.checkWO()
-
-	var i uint32
-	if b.rw != nil {
-		readUint32(b.rw, &i)
-		return i
-	}
-
-	readBuffUint32(b.r, &i)
-	return i
-}
-
-// ReadUInt64 reads a uint64 value from the buffer.
-func (b *Buffer) ReadUInt64() uint64 {
-	b.checkWO()
-
-	var i uint64
-	if b.rw != nil {
-		readUint64(b.rw, &i)
-		return i
-	}
-
-	readBuffUint64(b.r, &i)
-	return i
-}
-
-// ReadFloat32 reads a float32 value from the buffer.
-func (b *Buffer) ReadFloat32() float32 {
-	b.checkWO()
-
-	var i float32
-	if b.rw != nil {
-		readFloat32(b.rw, &i)
-		return i
-	}
-
-	readBuffFloat32(b.r, &i)
-	return i
-}
-
-// ReadFloat64 reads a float64 value from the buffer.
-func (b *Buffer) ReadFloat64() float64 {
-	b.checkWO()
-
-	var i float64
-	if b.rw != nil {
-		readFloat64(b.rw, &i)
-		return i
-	}
-
-	readBuffFloat64(b.r, &i)
-	return i
-}
-
-// ReadString reads a uint16 value from the buffer representing the string's length,
-// then uses the length to extract the exact length []byte representing the string.
-func (b *Buffer) ReadString() string {
-	b.checkWO()
-
-	var l uint16
-	if b.rw != nil {
-		readUint16(b.rw, &l)
-		return bytesToString(b.rw.Next(int(l)))
-	}
-
-	readBuffUint16(b.r, &l)
-
-	bytes := bytePool.Get(int(l))
-	defer bytePool.Put(bytes)
-
-	_, err := readBuffFull(b.r, bytes)
-	if err != nil {
-		return ""
-	}
-
-	return bytesToString(bytes)
-}
-
-// ReadBytes reads the specified length from the buffer and returns the byte slice.
-func (b *Buffer) ReadBytes(length int) []byte {
-	b.checkWO()
-
-	if b.rw != nil {
-		return b.rw.Next(length)
-	}
-
-	bytes := make([]byte, length)
-	_, err := readBuffFull(b.r, bytes)
-	if err != nil {
-		return bytes
-	}
-
-	return bytes
-}
-
-// bytesAsString converts a []byte into a string in place. Note that you should use this helper
-// when the []byte slice contains _only_ the string data and isn't part of a larger underlying array.
-// For example, a case where you should *not* use this helper:
-//
-//	func parseString(buffer *bytes.Buffer, length int) string {
-//	  bytes := buffer.Next(length)   // this extracts a sub-slice of the underlying byte array from pos->pos+length
-//
-//	  return bytesAsString(bytes)
-//	}
-//
-// Now both the []byte AND the value string are linked and neither can be GC'd until the other one is GC'd.
-// This is especially problematic if you drop the references to the byte array, as you're effectively requiring
-// 1024 bytes for an 11-byte string.
-//
-// An example where it _is_ ok, and recommended to drop the underlying []byte reference is the following:
-//
-//	func parseString(reader io.Reader, length int) string {
-//	  bytes := make([]byte, length)
-//	  io.ReadFull(reader, bytes)
-//
-//	  return bytesAsString(bytes)
-//	}
-//
-// In this case, we've created a byte array just big enough for the string, we extract the string data from the reader
-// and then cast the byte array in place to the string, and finally drop the byte array reference. This omits an additional
-// allocation if you were to use string(bytes)
-func bytesAsString(b []byte) string {
-	return unsafe.String(unsafe.SliceData(b), len(b))
-}
-
-// Conversion from byte slice to string
-func bytesToString(b []byte) string {
-	// This code will take the passed byte slice and cast it in-place into a string. By doing
-	// this, we are pinning the byte slice's underlying array in memory, preventing it from
-	// being garbage collected while the string is still in use. If we are using the Bank()
-	// functionality to cache new strings, we risk keeping the pinned array alive. To avoid this,
-	// we will use the BankFunc() call which uses the casted string to check for existence of a
-	// cached string. If it exists, then we drop the pinned reference immediately and use the
-	// cached string. If it does _not_ exist, then we use the passed func() string to allocate a new
-	// string and cache it. This will prevent us from allocating throw-away strings just to
-	// check our cache.
-	pinned := bytesAsString(b)
-
-	return stringutil.BankFunc(pinned, func() string {
-		return string(b)
-	})
-}
-
-// Direct string to byte conversion that doesn't allocate.
-func stringToBytes(s string) []byte {
-	return unsafe.Slice(unsafe.StringData(s), len(s))
-}

core/pkg/util/buffer_test.go

@@ -1,458 +0,0 @@
-package util
-
-import (
-	"bytes"
-	"io"
-	"math"
-	"math/rand/v2"
-	"runtime"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestBufferReadWrite(t *testing.T) {
-	buf := NewBuffer()
-
-	buf.WriteBool(true)
-	buf.WriteInt(42)
-	buf.WriteFloat64(3.14)
-	buf.WriteString("Testing, 1, 2, 3!")
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-
-	boolVal := readBuf.ReadBool()
-	intVal := readBuf.ReadInt()
-	floatVal := readBuf.ReadFloat64()
-	stringVal := readBuf.ReadString()
-
-	if boolVal != true {
-		t.Errorf("Expected bool value to be true, got %v", boolVal)
-	}
-	if intVal != 42 {
-		t.Errorf("Expected int value to be 42, got %v", intVal)
-	}
-	if floatVal != 3.14 {
-		t.Errorf("Expected float value to be 3.14, got %v", floatVal)
-	}
-	if stringVal != "Testing, 1, 2, 3!" {
-		t.Errorf("Expected string value to be 'Hello, World!', got %v", stringVal)
-	}
-}
-
-func TestBufferWriteReadBytes(t *testing.T) {
-	buf := NewBuffer()
-
-	bytesToWrite := []byte{0x01, 0x02, 0x03, 0x04}
-	buf.WriteBytes(bytesToWrite)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readBytes := readBuf.ReadBytes(len(bytesToWrite))
-
-	if !bytes.Equal(readBytes, bytesToWrite) {
-		t.Errorf("Expected bytes to be %v, got %v", bytesToWrite, readBytes)
-	}
-}
-
-func TestBufferWriteReadUInt64(t *testing.T) {
-	buf := NewBuffer()
-
-	uint64Val := uint64(1234567890)
-	buf.WriteUInt64(uint64Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readUInt64 := readBuf.ReadUInt64()
-
-	if readUInt64 != uint64Val {
-		t.Errorf("Expected uint64 value to be %v, got %v", uint64Val, readUInt64)
-	}
-}
-
-func TestBufferWriteReadFloat32(t *testing.T) {
-	buf := NewBuffer()
-
-	float32Val := float32(3.14159)
-	buf.WriteFloat32(float32Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readFloat32 := readBuf.ReadFloat32()
-
-	if readFloat32 != float32Val {
-		t.Errorf("Expected float32 value to be %v, got %v", float32Val, readFloat32)
-	}
-}
-
-func TestBufferWriteReadInt8(t *testing.T) {
-	buf := NewBuffer()
-
-	int8Val := int8(-42)
-	buf.WriteInt8(int8Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readInt8 := readBuf.ReadInt8()
-
-	if readInt8 != int8Val {
-		t.Errorf("Expected int8 value to be %v, got %v", int8Val, readInt8)
-	}
-}
-
-func TestBufferWriteReadUInt16(t *testing.T) {
-	buf := NewBuffer()
-
-	uint16Val := uint16(65535)
-	buf.WriteUInt16(uint16Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readUInt16 := readBuf.ReadUInt16()
-
-	if readUInt16 != uint16Val {
-		t.Errorf("Expected uint16 value to be %v, got %v", uint16Val, readUInt16)
-	}
-}
-
-func TestBufferWriteReadInt32(t *testing.T) {
-	buf := NewBuffer()
-
-	int32Val := int32(-1234567890)
-	buf.WriteInt32(int32Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readInt32 := readBuf.ReadInt32()
-
-	if readInt32 != int32Val {
-		t.Errorf("Expected int32 value to be %v, got %v", int32Val, readInt32)
-	}
-}
-
-func TestBufferWriteReadUInt8(t *testing.T) {
-	buf := NewBuffer()
-
-	uint8Val := uint8(255)
-	buf.WriteUInt8(uint8Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readUInt8 := readBuf.ReadUInt8()
-
-	if readUInt8 != uint8Val {
-		t.Errorf("Expected uint8 value to be %v, got %v", uint8Val, readUInt8)
-	}
-}
-
-func TestBufferWriteReadInt16(t *testing.T) {
-	buf := NewBuffer()
-
-	int16Val := int16(-32768)
-	buf.WriteInt16(int16Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readInt16 := readBuf.ReadInt16()
-
-	if readInt16 != int16Val {
-		t.Errorf("Expected int16 value to be %v, got %v", int16Val, readInt16)
-	}
-}
-
-func TestBufferWriteReadUInt32(t *testing.T) {
-	buf := NewBuffer()
-
-	uint32Val := uint32(4294967295)
-	buf.WriteUInt32(uint32Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readUInt32 := readBuf.ReadUInt32()
-
-	if readUInt32 != uint32Val {
-		t.Errorf("Expected uint32 value to be %v, got %v", uint32Val, readUInt32)
-	}
-}
-
-func TestBufferWriteReadInt64(t *testing.T) {
-	buf := NewBuffer()
-
-	int64Val := int64(-9223372036854775808)
-	buf.WriteInt64(int64Val)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-	readInt64 := readBuf.ReadInt64()
-
-	if readInt64 != int64Val {
-		t.Errorf("Expected int64 value to be %v, got %v", int64Val, readInt64)
-	}
-}
-
-func TestBufferBytes(t *testing.T) {
-	buf := NewBuffer()
-
-	buf.WriteInt(-42)
-	buf.WriteFloat64(-3.14)
-
-	unreadBytes := buf.Bytes()
-
-	newBuf := NewBufferFromBytes(unreadBytes)
-
-	intVal := newBuf.ReadInt()
-	floatVal := newBuf.ReadFloat64()
-
-	if intVal != -42 {
-		t.Errorf("Expected int value to be -42, got %v", intVal)
-	}
-	if floatVal != -3.14 {
-		t.Errorf("Expected float value to be -3.14, got %v", floatVal)
-	}
-}
-
-func TestBufferNewBufferFrom(t *testing.T) {
-	buf := NewBuffer()
-
-	buf.WriteInt(42)
-	buf.WriteFloat64(3.14)
-
-	newBuf := NewBufferFrom(buf)
-
-	intVal := newBuf.ReadInt()
-	floatVal := newBuf.ReadFloat64()
-
-	if intVal != 42 {
-		t.Errorf("Expected int value to be 42, got %v", intVal)
-	}
-	if floatVal != 3.14 {
-		t.Errorf("Expected float value to be 3.14, got %v", floatVal)
-	}
-}
-
-const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-
-func generateRandomString(ln int) string {
-	b := make([]byte, ln)
-	for i := range b {
-		b[i] = letters[rand.IntN(len(letters))]
-	}
-	return string(b)
-}
-
-func memMib() float64 {
-	var m runtime.MemStats
-	runtime.ReadMemStats(&m)
-	return float64(m.Alloc) / 1024.0 / 1024.0
-}
-
-func TestStringBytes(t *testing.T) {
-	baselineMem := memMib()
-
-	b := make([]byte, 10<<20)
-
-	afterMem := memMib()
-	delta := afterMem - baselineMem
-	t.Logf("Allocated %v MiB, Delta: %v MiB", afterMem, delta)
-
-	s := "Hello World!"
-	sl := b[512 : 512+len(s)]
-	copy(sl, stringToBytes(s))
-
-	afterMem = memMib()
-	delta = afterMem - baselineMem
-	t.Logf("Allocated %v MiB, Delta: %v MiB", afterMem, delta)
-
-	// this should pin the large backing array in memory, preventing it from being GC'd
-	newS := bytesToString(sl)
-
-	runtime.GC()
-	time.Sleep(time.Second)
-
-	afterMem = memMib()
-	delta = afterMem - baselineMem
-	t.Logf("S: %s, Allocated %v MiB, Delta: %v MiB", newS, afterMem, delta)
-
-	// copy the string into a new string and clear out pinned string
-	sCopy := strings.Clone(newS)
-	newS = ""
-
-	// Now that we've dropped the reference to the pinned backing array, it should be GC'd
-	runtime.GC()
-	time.Sleep(time.Second)
-
-	afterMem = memMib()
-	delta = afterMem - baselineMem
-	t.Logf("S: %s, Allocated %v MiB, Delta: %v MiB", sCopy, afterMem, delta)
-
-	if sCopy != s {
-		t.Errorf("Expected string to be %v, got %v", s, sCopy)
-	}
-
-	if delta > 0.5 {
-		t.Errorf("Expected memory delta to be less than 0.5 MiB, got %v MiB", delta)
-	}
-}
-
-type randomByteReader struct {
-	bytes []byte
-	pos   int
-}
-
-func newRandomByteReader(bytes []byte) *randomByteReader {
-	return &randomByteReader{
-		bytes: bytes,
-		pos:   0,
-	}
-}
-
-// reads a random number of bytes from 1-4 each time Read is called.
-// simulates partial buffered reads
-func (sbr *randomByteReader) Read(b []byte) (int, error) {
-	if sbr.pos >= len(sbr.bytes) {
-		return 0, io.EOF
-	}
-
-	toCopy := rand.IntN(4) + 1
-	if toCopy > len(b) {
-		toCopy = len(b)
-	}
-
-	var err error
-	remaining := len(sbr.bytes) - sbr.pos
-	if toCopy > remaining {
-		err = io.EOF
-		toCopy = remaining
-	}
-
-	bytesCopied := copy(b, sbr.bytes[sbr.pos:sbr.pos+toCopy])
-	sbr.pos += bytesCopied
-
-	return bytesCopied, err
-}
-
-func TestBufferWriterSupport(t *testing.T) {
-	byteBuff := new(bytes.Buffer)
-
-	buf := NewBufferFromWriter(byteBuff)
-	buf.WriteBool(true)
-	buf.WriteInt(42)
-	buf.WriteFloat64(3.14)
-	buf.WriteString("Testing, 1, 2, 3!")
-	buf.WriteUInt64(uint64(123456))
-	buf.WriteInt16(44)
-	buf.WriteFloat32(float32(5.0))
-	buf.Flush()
-
-	readerBuff := NewBufferFromBytes(byteBuff.Bytes())
-	b := readerBuff.ReadBool()
-	i := readerBuff.ReadInt()
-	f := readerBuff.ReadFloat64()
-	s := readerBuff.ReadString()
-	ui64 := readerBuff.ReadUInt64()
-	i16 := readerBuff.ReadInt16()
-	f32 := readerBuff.ReadFloat32()
-
-	if !b {
-		t.Errorf("expected true, got: false")
-	}
-	if i != 42 {
-		t.Errorf("expected 42, got: %d", i)
-	}
-	if f != 3.14 {
-		t.Errorf("expected 3.14, got: %f", f)
-	}
-	if s != "Testing, 1, 2, 3!" {
-		t.Errorf("expected 'Testing, 1, 2, 3!', got: '%s'", s)
-	}
-	if ui64 != uint64(123456) {
-		t.Errorf("expected 123456, got: %d", ui64)
-	}
-	if i16 != int16(44) {
-		t.Errorf("expected 44, got: %d", i16)
-	}
-	if f32 != float32(5.0) {
-		t.Errorf("expected 5.0, got: %f", f32)
-	}
-}
-
-func TestBufferReaderSupport(t *testing.T) {
-	buf := NewBuffer()
-	buf.WriteBool(true)
-	buf.WriteInt(42)
-	buf.WriteFloat64(3.14)
-	buf.WriteString("Testing, 1, 2, 3!")
-	buf.WriteUInt64(uint64(123456))
-	buf.WriteInt16(44)
-	buf.WriteFloat32(float32(5.0))
-
-	reader := newRandomByteReader(buf.Bytes())
-	readerBuff := NewBufferFromReader(reader)
-
-	b := readerBuff.ReadBool()
-	i := readerBuff.ReadInt()
-	f := readerBuff.ReadFloat64()
-	s := readerBuff.ReadString()
-	ui64 := readerBuff.ReadUInt64()
-	i16 := readerBuff.ReadInt16()
-	f32 := readerBuff.ReadFloat32()
-
-	if !b {
-		t.Errorf("expected true, got: false")
-	}
-	if i != 42 {
-		t.Errorf("expected 42, got: %d", i)
-	}
-	if f != 3.14 {
-		t.Errorf("expected 3.14, got: %f", f)
-	}
-	if s != "Testing, 1, 2, 3!" {
-		t.Errorf("expected 'Testing, 1, 2, 3!', got: '%s'", s)
-	}
-	if ui64 != uint64(123456) {
-		t.Errorf("expected 123456, got: %d", ui64)
-	}
-	if i16 != int16(44) {
-		t.Errorf("expected 44, got: %d", i16)
-	}
-	if f32 != float32(5.0) {
-		t.Errorf("expected 5.0, got: %f", f32)
-	}
-}
-
-func TestTooLargeStringTruncate(t *testing.T) {
-	normalStr := generateRandomString(100)
-	bigStr := generateRandomString(math.MaxUint16 + (math.MaxUint16 / 2))
-	expectedBigStrRead := bigStr[:math.MaxUint16]
-
-	otherBigStr := generateRandomString(math.MaxUint16)
-	plusOne := generateRandomString(math.MaxUint16 + 1)
-	expectedPlusOne := plusOne[:math.MaxUint16]
-
-	buf := NewBuffer()
-
-	buf.WriteInt(42)
-	buf.WriteFloat64(3.14)
-	buf.WriteString(normalStr)
-	buf.WriteString(bigStr)
-	buf.WriteString(otherBigStr)
-	buf.WriteString(plusOne)
-
-	readBuf := NewBufferFromBytes(buf.Bytes())
-
-	intVal := readBuf.ReadInt()
-	floatVal := readBuf.ReadFloat64()
-	normalStrRead := readBuf.ReadString()
-	bigStrRead := readBuf.ReadString()
-	otherBigStrRead := readBuf.ReadString()
-	plusOneRead := readBuf.ReadString()
-
-	if intVal != 42 {
-		t.Errorf("Expected int value to be 42, got %v", intVal)
-	}
-	if floatVal != 3.14 {
-		t.Errorf("Expected float value to be 3.14, got %v", floatVal)
-	}
-	if normalStrRead != normalStr {
-		t.Errorf("Expected string value to be %v, got %v", normalStr, normalStrRead)
-	}
-	if bigStrRead != expectedBigStrRead {
-		t.Errorf("Expected large string values to be equivalent!")
-	}
-	if otherBigStrRead != otherBigStr {
-		t.Errorf("Expected large string values to be equivalent!")
-	}
-	if plusOneRead != expectedPlusOne {
-		t.Errorf("Expected large string values to be equivalent!")
-	}
-}

core/pkg/util/bufferhelper.go

@@ -1,595 +0,0 @@
-package util
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/binary"
-	"io"
-	"math"
-)
-
-func readBool(r *bytes.Buffer, data *bool) error {
-	b, err := r.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	*data = b != 0
-	return nil
-}
-
-func readInt8(r *bytes.Buffer, data *int8) error {
-	b, err := r.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	*data = int8(b)
-	return nil
-}
-
-func readUint8(r *bytes.Buffer, data *uint8) error {
-	b, err := r.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	*data = uint8(b)
-	return nil
-}
-
-func readInt16(r *bytes.Buffer, data *int16) error {
-	order := binary.LittleEndian
-	var b [2]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int16(order.Uint16(bs))
-	return nil
-}
-
-func readUint16(r *bytes.Buffer, data *uint16) error {
-	order := binary.LittleEndian
-	var b [2]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = order.Uint16(bs)
-	return nil
-}
-
-func readInt(r *bytes.Buffer, data *int) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int(int32(order.Uint32(bs)))
-	return nil
-}
-
-func readInt32(r *bytes.Buffer, data *int32) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int32(order.Uint32(bs))
-	return nil
-}
-
-func readUint(r *bytes.Buffer, data *uint) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = uint(order.Uint32(bs))
-	return nil
-}
-
-func readUint32(r *bytes.Buffer, data *uint32) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = order.Uint32(bs)
-	return nil
-}
-
-func readInt64(r *bytes.Buffer, data *int64) error {
-	order := binary.LittleEndian
-	var b [8]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int64(order.Uint64(bs))
-	return nil
-}
-
-func readUint64(r *bytes.Buffer, data *uint64) error {
-	order := binary.LittleEndian
-	var b [8]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = order.Uint64(bs)
-	return nil
-}
-
-func readFloat32(r *bytes.Buffer, data *float32) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = math.Float32frombits(order.Uint32(bs))
-	return nil
-}
-
-func readFloat64(r *bytes.Buffer, data *float64) error {
-	order := binary.LittleEndian
-	var b [8]byte
-
-	bs := b[:]
-	_, err := readFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = math.Float64frombits(order.Uint64(bs))
-	return nil
-}
-
-func readBuffBool(r *bufio.Reader, data *bool) error {
-	b, err := r.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	*data = b != 0
-	return nil
-}
-
-func readBuffInt8(r *bufio.Reader, data *int8) error {
-	b, err := r.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	*data = int8(b)
-	return nil
-}
-
-func readBuffUint8(r *bufio.Reader, data *uint8) error {
-	b, err := r.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	*data = uint8(b)
-	return nil
-}
-
-func readBuffInt16(r *bufio.Reader, data *int16) error {
-	order := binary.LittleEndian
-	var b [2]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int16(order.Uint16(bs))
-	return nil
-}
-
-func readBuffUint16(r *bufio.Reader, data *uint16) error {
-	order := binary.LittleEndian
-	var b [2]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = order.Uint16(bs)
-	return nil
-}
-
-func readBuffInt(r *bufio.Reader, data *int) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int(int32(order.Uint32(bs)))
-	return nil
-}
-
-func readBuffInt32(r *bufio.Reader, data *int32) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int32(order.Uint32(bs))
-	return nil
-}
-
-func readBuffUint(r *bufio.Reader, data *uint) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = uint(order.Uint32(bs))
-	return nil
-}
-
-func readBuffUint32(r *bufio.Reader, data *uint32) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = order.Uint32(bs)
-	return nil
-}
-
-func readBuffInt64(r *bufio.Reader, data *int64) error {
-	order := binary.LittleEndian
-	var b [8]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = int64(order.Uint64(bs))
-	return nil
-}
-
-func readBuffUint64(r *bufio.Reader, data *uint64) error {
-	order := binary.LittleEndian
-	var b [8]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = order.Uint64(bs)
-	return nil
-}
-
-func readBuffFloat32(r *bufio.Reader, data *float32) error {
-	order := binary.LittleEndian
-	var b [4]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = math.Float32frombits(order.Uint32(bs))
-	return nil
-}
-
-func readBuffFloat64(r *bufio.Reader, data *float64) error {
-	order := binary.LittleEndian
-	var b [8]byte
-
-	bs := b[:]
-	_, err := readBuffFull(r, bs)
-	if err != nil {
-		return err
-	}
-
-	*data = math.Float64frombits(order.Uint64(bs))
-	return nil
-}
-
-// read full is a bufio.Reader specific implementation of io.ReadFull() which
-// avoids escaping our stack allocated scratch bytes
-func readBuffFull(r *bufio.Reader, buf []byte) (n int, err error) {
-	min := len(buf)
-	for n < min && err == nil {
-		var nn int
-		nn, err = r.Read(buf[n:])
-		n += nn
-	}
-	if n >= min {
-		err = nil
-	} else if n > 0 && err == io.EOF {
-		err = io.ErrUnexpectedEOF
-	}
-	return
-}
-
-// read full is a bytes.Buffer specific implementation of io.ReadFull() which
-// avoids escaping our stack allocated scratch bytes
-func readFull(r *bytes.Buffer, buf []byte) (n int, err error) {
-	min := len(buf)
-	for n < min && err == nil {
-		var nn int
-		nn, err = r.Read(buf[n:])
-		n += nn
-	}
-	if n >= min {
-		err = nil
-	} else if n > 0 && err == io.EOF {
-		err = io.ErrUnexpectedEOF
-	}
-	return
-}
-
-func writeBool(w *bytes.Buffer, data bool) error {
-	if data {
-		return w.WriteByte(1)
-	}
-
-	return w.WriteByte(0)
-}
-
-func writeInt8(w *bytes.Buffer, data int8) error {
-	return w.WriteByte(byte(data))
-}
-
-func writeUint8(w *bytes.Buffer, data uint8) error {
-	return w.WriteByte(byte(data))
-}
-
-func writeInt16(w *bytes.Buffer, data int16) error {
-	var b [2]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint16(bs, uint16(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeUint16(w *bytes.Buffer, data uint16) error {
-	var b [2]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint16(bs, data)
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeInt32(w *bytes.Buffer, data int32) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, uint32(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeUint32(w *bytes.Buffer, data uint32) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, data)
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeInt(w *bytes.Buffer, data int) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, uint32(int32(data)))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeUint(w *bytes.Buffer, data uint) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, uint32(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeInt64(w *bytes.Buffer, data int64) error {
-	var b [8]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint64(bs, uint64(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeUint64(w *bytes.Buffer, data uint64) error {
-	var b [8]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint64(bs, data)
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeFloat32(w *bytes.Buffer, data float32) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, math.Float32bits(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeFloat64(w *bytes.Buffer, data float64) error {
-	var b [8]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint64(bs, math.Float64bits(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffBool(w *bufio.Writer, data bool) error {
-	if data {
-		return w.WriteByte(1)
-	}
-
-	return w.WriteByte(0)
-}
-
-func writeBuffInt8(w *bufio.Writer, data int8) error {
-	return w.WriteByte(byte(data))
-}
-
-func writeBuffUint8(w *bufio.Writer, data uint8) error {
-	return w.WriteByte(byte(data))
-}
-
-func writeBuffInt16(w *bufio.Writer, data int16) error {
-	var b [2]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint16(bs, uint16(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffUint16(w *bufio.Writer, data uint16) error {
-	var b [2]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint16(bs, data)
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffInt32(w *bufio.Writer, data int32) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, uint32(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffUint32(w *bufio.Writer, data uint32) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, data)
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffInt(w *bufio.Writer, data int) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, uint32(int32(data)))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffUint(w *bufio.Writer, data uint) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, uint32(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffInt64(w *bufio.Writer, data int64) error {
-	var b [8]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint64(bs, uint64(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffUint64(w *bufio.Writer, data uint64) error {
-	var b [8]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint64(bs, data)
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffFloat32(w *bufio.Writer, data float32) error {
-	var b [4]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint32(bs, math.Float32bits(data))
-	_, err := w.Write(bs)
-	return err
-}
-
-func writeBuffFloat64(w *bufio.Writer, data float64) error {
-	var b [8]byte
-	bs := b[:]
-
-	binary.LittleEndian.PutUint64(bs, math.Float64bits(data))
-	_, err := w.Write(bs)
-	return err
-}

core/pkg/util/bufferpool.go

@@ -1,68 +0,0 @@
-package util
-
-import (
-	"math"
-	"math/bits"
-	"sync"
-)
-
-// bufferPool holds "tiered" []byte `sync.Pool` instances by capacity up to math.MaxUint16
-type bufferPool struct {
-	pools [17]sync.Pool
-}
-
-func newBufferPool() *bufferPool {
-	bp := new(bufferPool)
-
-	for i := 0; i < 17; i++ {
-		length := 1 << i
-		bp.pools[i].New = func() any {
-			return make([]byte, length)
-		}
-	}
-	return bp
-}
-
-// poolIndex returns the pool index for a buffer of the given size.
-func poolIndex(length int) int {
-	return bits.Len32(uint32(length - 1))
-}
-
-// putIndex returns the pool index for returning a buffer with the given capacity.
-// It is the inverse of poolIndex: given a capacity that was originally handed out
-// by Get, it finds the pool that owns it.
-//
-// Because Get always returns buffers with capacity 1<<i, the capacity here will
-// always be a power of two. bits.Len32(1<<i) = i+1, so we subtract 1 to recover i.
-func putIndex(capacity int) int {
-	return bits.Len32(uint32(capacity)) - 1
-}
-
-func isPowerOfTwo(capacity int) bool {
-	return capacity&(capacity-1) == 0
-}
-
-func (bp *bufferPool) Get(length int) []byte {
-	if length <= 0 {
-		return nil
-	}
-
-	// Beyond our pool range: allocate directly
-	if length > math.MaxUint16 {
-		return make([]byte, length)
-	}
-
-	i := poolIndex(length)
-	buf := bp.pools[i].Get().([]byte)
-	return buf[:length]
-}
-
-func (bp *bufferPool) Put(buf []byte) {
-	capacity := cap(buf)
-	if capacity == 0 || capacity > math.MaxUint16 || !isPowerOfTwo(capacity) {
-		return
-	}
-
-	i := putIndex(capacity)
-	bp.pools[i].Put(buf[:cap(buf)])
-}

core/pkg/util/bufferpool_test.go

@@ -1,311 +0,0 @@
-package util
-
-import (
-	"math"
-	"math/bits"
-	"sync"
-	"testing"
-)
-
-// --- poolIndex / putIndex unit tests ---
-
-func TestPoolIndex(t *testing.T) {
-	cases := []struct {
-		length int
-		want   int
-	}{
-		{1, 0},
-		{2, 1},
-		{3, 2},
-		{4, 2},
-		{5, 3},
-		{7, 3},
-		{8, 3},
-		{255, 8},
-		{256, 8},
-		{1023, 10},
-		{1024, 10},
-		{math.MaxUint16 - 50, 16},
-	}
-	for _, c := range cases {
-		got := poolIndex(c.length)
-		if got != c.want {
-			t.Errorf("poolIndex(%d) = %d, want %d", c.length, got, c.want)
-		}
-	}
-}
-
-func TestAllocMinusOne(t *testing.T) {
-	bp := newBufferPool()
-	for i := 1; i <= 16; i++ {
-		capacity := 1 << i
-		length := capacity - 1
-		if length <= 0 {
-			continue
-		}
-
-		b := bp.Get(length)
-		c := cap(b)
-
-		pIndex := poolIndex(length)
-		rIndex := putIndex(c)
-
-		if pIndex != rIndex {
-			t.Errorf("pIndex: %d != rIndex: %d\n", pIndex, rIndex)
-		}
-
-	}
-}
-
-func TestPutIndex(t *testing.T) {
-	// putIndex must be the inverse of poolIndex for all power-of-two capacities
-	// that Get hands out.
-	for i := 1; i <= 16; i++ {
-		cap := 1 << i
-		got := putIndex(cap)
-		if got != i {
-			t.Errorf("putIndex(1<<%d = %d) = %d, want %d", i, cap, got, i)
-		}
-	}
-}
-
-func TestPoolIndexPutIndexRoundTrip(t *testing.T) {
-	// For any requested length, the buffer Get returns has capacity 1<<poolIndex(length).
-	// Confirm that putIndex maps that capacity back to the same pool slot.
-	for length := 1; length <= math.MaxUint16; length++ {
-		i := poolIndex(length)
-		capacity := 1 << i
-		j := putIndex(capacity)
-		if i != j {
-			t.Errorf("length=%d: poolIndex=%d, capacity=1<<%d=%d, putIndex=%d — round-trip broken",
-				length, i, i, capacity, j)
-		}
-	}
-}
-
-// --- Get ---
-
-func TestGetNilOnZeroOrNegative(t *testing.T) {
-	bp := newBufferPool()
-	for _, n := range []int{0, -1, -100} {
-		if got := bp.Get(n); got != nil {
-			t.Errorf("Get(%d) = %v, want nil", n, got)
-		}
-	}
-}
-
-func TestGetLengthIsExact(t *testing.T) {
-	bp := newBufferPool()
-	for _, n := range []int{1, 2, 3, 7, 8, 100, 1000, 65535, 65536} {
-		buf := bp.Get(n)
-		if len(buf) != n {
-			t.Errorf("Get(%d): len = %d, want %d", n, len(buf), n)
-		}
-	}
-}
-
-func TestGetCapacityIsPowerOfTwo(t *testing.T) {
-	bp := newBufferPool()
-	for _, n := range []int{1, 2, 3, 4, 5, 100, 1000, 550, math.MaxUint16 - 100, math.MaxUint16} {
-		buf := bp.Get(n)
-		c := cap(buf)
-		if c == 0 || !isPowerOfTwo(c) {
-			t.Errorf("Get(%d): cap = %d, not a power of two", n, c)
-		}
-	}
-}
-
-func TestGetCapacityIsSmallestFittingPowerOfTwo(t *testing.T) {
-	bp := newBufferPool()
-	cases := []struct {
-		n       int
-		wantCap int
-	}{
-		{1, 1},
-		{2, 2},
-		{3, 4},
-		{4, 4},
-		{5, 8},
-		{8, 8},
-		{9, 16},
-		{255, 256},
-		{256, 256},
-		{1024, 1024},
-	}
-	for _, c := range cases {
-		buf := bp.Get(c.n)
-		if cap(buf) != c.wantCap {
-			t.Errorf("Get(%d): cap = %d, want %d", c.n, cap(buf), c.wantCap)
-		}
-	}
-}
-
-func TestGetOversizeFallback(t *testing.T) {
-	bp := newBufferPool()
-	n := math.MaxUint16 + 1
-	buf := bp.Get(n)
-	if len(buf) != n {
-		t.Errorf("Get(MaxUint16+1): len = %d, want %d", len(buf), n)
-	}
-}
-
-// --- Put ---
-
-func TestPutDropsZeroCapBuffer(t *testing.T) {
-	// Put on a nil or zero-cap slice must not panic.
-	bp := newBufferPool()
-	bp.Put(nil)
-	bp.Put([]byte{})
-}
-
-// --- Get / Put round-trip ---
-
-func TestGetPutSamePool(t *testing.T) {
-	// A buffer returned via Put must land in the same pool that Get draws from,
-	// so the very next Get (with the same length) should reuse it.
-	bp := newBufferPool()
-
-	buf := bp.Get(100)
-	ptr := &buf[0]
-	bp.Put(buf)
-
-	buf2 := bp.Get(100)
-	if &buf2[0] != ptr {
-		// sync.Pool may have GC'd the entry; this is not a hard failure but
-		// we at minimum require length and capacity to be correct.
-		if len(buf2) != 100 {
-			t.Errorf("Get(100) after Put: len = %d, want 100", len(buf2))
-		}
-	}
-}
-
-func TestPutRestoresFullCapacity(t *testing.T) {
-	// After Put, the pooled slice should have full capacity, not the resliced length.
-	// We verify this by inspecting what comes out of the pool on the next Get.
-	bp := newBufferPool()
-
-	buf := bp.Get(10)  // len=10, cap=16
-	bp.Put(buf)        // must put back with cap=16
-	buf2 := bp.Get(15) // asks for 15 — still fits in cap=16 pool
-	if cap(buf2) < 15 {
-		t.Errorf("After Put(cap=16), Get(15): cap = %d, too small", cap(buf2))
-	}
-}
-
-func TestIsPowerOfTwo(t *testing.T) {
-	for i := 0; i < 16; i++ {
-		cap := 1 << i
-
-		if !isPowerOfTwo(cap) {
-			t.Fatalf("Failed at: i=%d, cap=%d\n", i, cap)
-		}
-	}
-
-	for _, v := range []int{5, 17, 19, 31, 55} {
-		if isPowerOfTwo(v) {
-			t.Fatalf("Unexpected isPowerOfTwo: %d", v)
-		}
-	}
-}
-
-func TestPutNonPowerOfTwoCapIsDiscarded(t *testing.T) {
-	// Buffers with non-power-of-two capacities (e.g. from outside the pool)
-	// get silently dropped. Confirm no panic and pool still works after.
-	bp := newBufferPool()
-	value := make([]byte, 0, 17)
-	bp.Put(value)
-
-	buf := bp.Get(24)
-	if len(buf) != 24 {
-		t.Errorf("Get(24) after spurious Put: len = %d, want 24", len(buf))
-	}
-	if cap(buf) != 32 {
-		t.Errorf("Get(24) after spurious Put: cap = %d, want 32", cap(buf))
-	}
-}
-
-// --- Concurrency ---
-
-func TestConcurrentGetPut(t *testing.T) {
-	bp := newBufferPool()
-	var wg sync.WaitGroup
-	const goroutines = 64
-	const iters = 1000
-
-	for g := 0; g < goroutines; g++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			for i := 0; i < iters; i++ {
-				n := (id*iters + i) % 4096
-				if n == 0 {
-					n = 1
-				}
-				buf := bp.Get(n)
-				if len(buf) != n {
-					t.Errorf("concurrent Get(%d): len = %d", n, len(buf))
-				}
-				// Write to every byte to catch races under -race.
-				for j := range buf {
-					buf[j] = byte(j)
-				}
-				bp.Put(buf)
-			}
-		}(g)
-	}
-	wg.Wait()
-}
-
-// --- Edge cases at pool boundaries ---
-
-func TestGetExactPowerOfTwo(t *testing.T) {
-	// Exact powers of two are the boundary between two pools; confirm correct
-	// bucket selection and full round-trip for each.
-	bp := newBufferPool()
-	for i := 0; i < 17; i++ {
-		n := 1 << i
-		buf := bp.Get(n)
-		if len(buf) != n {
-			t.Errorf("Get(1<<%d=%d): len = %d", i, n, len(buf))
-		}
-		expectedCap := 1 << (bits.Len16(uint16(n - 1)))
-		if cap(buf) != expectedCap {
-			t.Errorf("Get(1<<%d=%d): cap = %d, want %d", i, n, cap(buf), expectedCap)
-		}
-		bp.Put(buf)
-	}
-}
-
-func TestGetMaxInt16(t *testing.T) {
-	i := poolIndex(math.MaxUint16)
-	if i >= 17 {
-		t.Errorf("poolIndex(MaxUint16) = %d, overflows pool array", i)
-	}
-}
-
-// --- Benchmarks ---
-
-func BenchmarkGetPut(b *testing.B) {
-	sizes := []int{64, 512, 4096, 65535}
-	for _, size := range sizes {
-		bp := newBufferPool()
-		b.Run("", func(b *testing.B) {
-			b.ReportAllocs()
-			for i := 0; i < b.N; i++ {
-				buf := bp.Get(size)
-				bp.Put(buf)
-			}
-		})
-	}
-}
-
-func BenchmarkGetPutParallel(b *testing.B) {
-	bp := newBufferPool()
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			buf := bp.Get(4096)
-			bp.Put(buf)
-		}
-	})
-}

core/pkg/util/stringutil/lrubank.go

@@ -1,160 +0,0 @@
-package stringutil
-
-import (
-	"container/heap"
-	"sync"
-	"time"
-)
-
-type lruEntry struct {
-	value string
-	used  int64
-}
-type maxHeap []*lruEntry
-
-func (h maxHeap) Len() int           { return len(h) }
-func (h maxHeap) Less(i, j int) bool { return h[i].used > h[j].used } // newer = "larger"
-func (h maxHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
-
-func (h *maxHeap) Push(x any) {
-	*h = append(*h, x.(*lruEntry))
-}
-
-func (h *maxHeap) Pop() any {
-	old := *h
-	n := len(old)
-	x := old[n-1]
-	*h = old[:n-1]
-	return x
-}
-
-func nOldest(arr []*lruEntry, n int) []*lruEntry {
-	if n <= 0 {
-		return []*lruEntry{}
-	}
-
-	if n >= len(arr) {
-		return arr
-	}
-
-	h := maxHeap(arr[:n])
-	heap.Init(&h)
-
-	for _, entry := range arr[n:] {
-		// swap in oldest, re-heapify
-		if entry.used < h[0].used {
-			h[0] = entry
-			heap.Fix(&h, 0)
-		}
-	}
-
-	return []*lruEntry(h)
-}
-
-type lruStringBank struct {
-	lock     sync.Mutex
-	stop     chan struct{}
-	m        map[string]*lruEntry
-	capacity int
-}
-
-func NewLruStringBank(capacity int, evictionInterval time.Duration) StringBank {
-	stop := make(chan struct{})
-	bank := &lruStringBank{
-		stop:     stop,
-		m:        make(map[string]*lruEntry),
-		capacity: capacity,
-	}
-
-	go func() {
-		for {
-			select {
-			case <-stop:
-				return
-			case <-time.After(evictionInterval):
-			}
-
-			// need to take the lock during eviction
-			bank.lock.Lock()
-			evict(bank, capacity)
-			bank.lock.Unlock()
-		}
-	}()
-
-	return bank
-}
-
-func evict(bank *lruStringBank, capacity int) {
-	if len(bank.m) <= capacity {
-		return
-	}
-
-	// we collect a list of all lru entries so we can max heap the first n elements
-	arr := make([]*lruEntry, 0, len(bank.m))
-	for _, v := range bank.m {
-		arr = append(arr, v)
-	}
-
-	oldest := nOldest(arr, len(bank.m)-capacity)
-	for _, old := range oldest {
-		delete(bank.m, old.value)
-	}
-}
-
-func (sb *lruStringBank) Stop() {
-	sb.lock.Lock()
-	defer sb.lock.Unlock()
-
-	if sb.stop != nil {
-		close(sb.stop)
-		sb.stop = nil
-	}
-}
-
-func (sb *lruStringBank) LoadOrStore(key, value string) (string, bool) {
-	sb.lock.Lock()
-
-	if v, ok := sb.m[key]; ok {
-		v.used = time.Now().UnixMilli()
-		sb.lock.Unlock()
-		return v.value, ok
-	}
-
-	sb.m[key] = &lruEntry{
-		value: value,
-		used:  time.Now().UnixMilli(),
-	}
-	if len(sb.m) > (sb.capacity + (sb.capacity / 2)) {
-		evict(sb, sb.capacity)
-	}
-	sb.lock.Unlock()
-	return value, false
-}
-
-func (sb *lruStringBank) LoadOrStoreFunc(key string, f func() string) (string, bool) {
-	sb.lock.Lock()
-
-	if v, ok := sb.m[key]; ok {
-		v.used = time.Now().UnixMilli()
-		sb.lock.Unlock()
-		return v.value, ok
-	}
-
-	// create the key and value using the func (the key could be deallocated later)
-	value := f()
-	sb.m[value] = &lruEntry{
-		value: value,
-		used:  time.Now().UnixMilli(),
-	}
-	if len(sb.m) > (sb.capacity + (sb.capacity / 2)) {
-		evict(sb, sb.capacity)
-	}
-	sb.lock.Unlock()
-	return value, false
-}
-
-func (sb *lruStringBank) Clear() {
-	sb.lock.Lock()
-	sb.m = make(map[string]*lruEntry)
-	sb.lock.Unlock()
-}

core/pkg/util/stringutil/lrubank_test.go

@@ -1,408 +0,0 @@
-package stringutil
-
-import (
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestBasicLruEvict(t *testing.T) {
-	lruBank := NewLruStringBank(3, 2*time.Second).(*lruStringBank)
-	defer lruBank.Stop()
-
-	lruBank.LoadOrStore("foo", "foo")
-	time.Sleep(500 * time.Millisecond)
-	lruBank.LoadOrStore("bar", "bar")
-	time.Sleep(500 * time.Millisecond)
-	lruBank.LoadOrStore("whaz", "whaz")
-	time.Sleep(500 * time.Millisecond)
-	// access foo, updating recency
-	lruBank.LoadOrStore("foo", "foo")
-	// should push bar out after eviction runs
-	lruBank.LoadOrStore("test", "test")
-	time.Sleep(time.Second)
-
-	lruBank.lock.Lock()
-	for _, v := range lruBank.m {
-		t.Logf("Value: %s\n", v.value)
-		if v.value == "bar" {
-			t.Errorf("The 'bar' entry should've been replaced by 'test'")
-		}
-	}
-	lruBank.lock.Unlock()
-}
-
-// ---------------------------------------------------------------------------
-// LoadOrStore
-// ---------------------------------------------------------------------------
-
-// A stored value must be retrievable and LoadOrStore must signal the hit/miss
-// correctly via the boolean return.
-func TestLoadOrStore_MissAndHit(t *testing.T) {
-	bank := NewLruStringBank(10, time.Minute).(*lruStringBank)
-	defer bank.Stop()
-
-	v, loaded := bank.LoadOrStore("hello", "hello")
-	if loaded {
-		t.Errorf("first LoadOrStore: expected loaded=false, got true")
-	}
-	if v != "hello" {
-		t.Errorf("first LoadOrStore: expected value %q, got %q", "hello", v)
-	}
-
-	v, loaded = bank.LoadOrStore("hello", "world")
-	if !loaded {
-		t.Errorf("second LoadOrStore: expected loaded=true, got false")
-	}
-	// The original value must be returned on a hit, not the new candidate.
-	if v != "hello" {
-		t.Errorf("second LoadOrStore: expected cached value %q, got %q", "hello", v)
-	}
-}
-
-// Hitting an existing entry must update its recency so it is not evicted ahead
-// of entries that were never touched again.
-func TestLoadOrStore_HitUpdateRecency(t *testing.T) {
-	bank := NewLruStringBank(2, 500*time.Millisecond).(*lruStringBank)
-	defer bank.Stop()
-
-	bank.LoadOrStore("old", "old")
-	time.Sleep(100 * time.Millisecond)
-	bank.LoadOrStore("keep", "keep")
-	time.Sleep(100 * time.Millisecond)
-
-	// Re-touch "old" so it becomes the most-recently-used.
-	bank.LoadOrStore("old", "old")
-	time.Sleep(100 * time.Millisecond)
-
-	// Adding a third entry exceeds capacity; "keep" should be the oldest now.
-	bank.LoadOrStore("new", "new")
-
-	// Wait for the eviction goroutine.
-	time.Sleep(600 * time.Millisecond)
-
-	bank.lock.Lock()
-	defer bank.lock.Unlock()
-
-	if _, ok := bank.m["keep"]; ok {
-		t.Error("expected 'keep' to be evicted but it is still present")
-	}
-	if _, ok := bank.m["old"]; !ok {
-		t.Error("expected 'old' to survive eviction after its recency was refreshed")
-	}
-}
-
-// ---------------------------------------------------------------------------
-// LoadOrStoreFunc
-// ---------------------------------------------------------------------------
-
-// The factory function must only be called on a cache miss, not on a hit.
-func TestLoadOrStoreFunc_FactoryCalledOnMissOnly(t *testing.T) {
-	bank := NewLruStringBank(10, time.Minute).(*lruStringBank)
-	defer bank.Stop()
-	calls := 0
-
-	factory := func() string {
-		calls++
-		return "k"
-	}
-
-	bank.LoadOrStoreFunc("k", factory)
-	bank.LoadOrStoreFunc("k", factory)
-
-	if calls != 1 {
-		t.Errorf("factory should be called exactly once, got %d calls", calls)
-	}
-}
-
-// ---------------------------------------------------------------------------
-// Capacity / eviction
-// ---------------------------------------------------------------------------
-
-// If the bank never exceeds capacity, nothing should be evicted.
-func TestEviction_BelowCapacityNoEviction(t *testing.T) {
-	const capacity = 5
-	bank := NewLruStringBank(capacity, 200*time.Millisecond).(*lruStringBank)
-	defer bank.Stop()
-
-	for i := 0; i < capacity; i++ {
-		bank.LoadOrStore(fmt.Sprintf("v%d", i), fmt.Sprintf("v%d", i))
-	}
-
-	// Wait several eviction cycles.
-	time.Sleep(600 * time.Millisecond)
-
-	bank.lock.Lock()
-	defer bank.lock.Unlock()
-
-	if got := len(bank.m); got != capacity {
-		t.Errorf("expected %d entries, got %d", capacity, got)
-	}
-}
-
-// After eviction the map must be trimmed down to exactly capacity.
-func TestEviction_ExceedCapacityTrimsToCapacity(t *testing.T) {
-	const capacity = 3
-	bank := NewLruStringBank(capacity, 350*time.Millisecond).(*lruStringBank)
-	defer bank.Stop()
-
-	for i := 0; i < capacity+3; i++ {
-		bank.LoadOrStore(fmt.Sprintf("v%d", i), fmt.Sprintf("v%d", i))
-		time.Sleep(20 * time.Millisecond) // ensure distinct timestamps
-	}
-
-	// Wait for eviction.
-	time.Sleep(500 * time.Millisecond)
-
-	bank.lock.Lock()
-	defer bank.lock.Unlock()
-
-	if got := len(bank.m); got > capacity {
-		t.Errorf("expected at most %d entries after eviction, got %d", capacity, got)
-	}
-}
-
-// The most-recently-used entries must survive eviction.
-func TestEviction_MRUSurvives(t *testing.T) {
-	const capacity = 2
-	bank := NewLruStringBank(capacity, 300*time.Millisecond).(*lruStringBank)
-	defer bank.Stop()
-
-	bank.LoadOrStore("evict1", "evict1")
-	time.Sleep(50 * time.Millisecond)
-	bank.LoadOrStore("evict2", "evict2")
-	time.Sleep(50 * time.Millisecond)
-
-	// These two are the most recent; they must survive.
-	bank.LoadOrStore("keep1", "keep1")
-	time.Sleep(50 * time.Millisecond)
-	bank.LoadOrStore("keep2", "keep2")
-
-	time.Sleep(500 * time.Millisecond)
-
-	bank.lock.Lock()
-	defer bank.lock.Unlock()
-
-	for _, must := range []string{"keep1", "keep2"} {
-		if _, ok := bank.m[must]; !ok {
-			t.Errorf("expected %q to survive eviction", must)
-		}
-	}
-}
-
-// ---------------------------------------------------------------------------
-// Clear
-// ---------------------------------------------------------------------------
-
-func TestClear_EmptiesMap(t *testing.T) {
-	bank := NewLruStringBank(10, time.Minute).(*lruStringBank)
-	defer bank.Stop()
-
-	for i := 0; i < 5; i++ {
-		bank.LoadOrStore(fmt.Sprintf("v%d", i), fmt.Sprintf("v%d", i))
-	}
-
-	bank.Clear()
-
-	bank.lock.Lock()
-	defer bank.lock.Unlock()
-
-	if len(bank.m) != 0 {
-		t.Errorf("expected empty map after Clear, got %d entries", len(bank.m))
-	}
-}
-
-// After a Clear, previously stored keys must not be found.
-func TestClear_PreviousKeysGone(t *testing.T) {
-	bank := NewLruStringBank(10, time.Minute).(*lruStringBank)
-	defer bank.Stop()
-
-	bank.LoadOrStore("hello", "world")
-	bank.Clear()
-
-	_, loaded := bank.LoadOrStore("hello", "new")
-	if loaded {
-		t.Error("expected key to be absent after Clear, but it was found")
-	}
-}
-
-// ---------------------------------------------------------------------------
-// nOldest helper
-// ---------------------------------------------------------------------------
-
-func TestNOldest_ReturnsCorrectCount(t *testing.T) {
-	now := time.Now()
-	entries := []*lruEntry{
-		{value: "a", used: now.Add(-4 * time.Second).UnixMilli()},
-		{value: "b", used: now.Add(-3 * time.Second).UnixMilli()},
-		{value: "c", used: now.Add(-2 * time.Second).UnixMilli()},
-		{value: "d", used: now.Add(-1 * time.Second).UnixMilli()},
-		{value: "e", used: now.UnixMilli()},
-	}
-
-	oldest := nOldest(entries, 2)
-	if len(oldest) != 2 {
-		t.Fatalf("expected 2 oldest entries, got %d", len(oldest))
-	}
-
-	values := map[string]bool{}
-	for _, e := range oldest {
-		values[e.value] = true
-	}
-	for _, must := range []string{"a", "b"} {
-		if !values[must] {
-			t.Errorf("expected %q in oldest set, got %v", must, values)
-		}
-	}
-}
-
-func TestNOldest_NGreaterThanLen(t *testing.T) {
-	now := time.Now()
-	entries := []*lruEntry{
-		{value: "x", used: now.UnixMilli()},
-		{value: "y", used: now.Add(-time.Second).UnixMilli()},
-	}
-
-	result := nOldest(entries, 10)
-	if len(result) != 2 {
-		t.Errorf("expected all %d entries when n >= len, got %d", 2, len(result))
-	}
-}
-
-func TestNOldest_NEqualsLen(t *testing.T) {
-	now := time.Now()
-	entries := []*lruEntry{
-		{value: "x", used: now.UnixMilli()},
-		{value: "y", used: now.Add(-time.Second).UnixMilli()},
-	}
-
-	result := nOldest(entries, 2)
-	if len(result) != 2 {
-		t.Errorf("expected 2 entries when n == len, got %d", len(result))
-	}
-}
-
-func TestNOldest_NIsZero(t *testing.T) {
-	now := time.Now()
-	entries := []*lruEntry{
-		{value: "x", used: now.UnixMilli()},
-	}
-
-	result := nOldest(entries, 0)
-	if len(result) != 0 {
-		t.Errorf("expected 0 entries when n=0, got %d", len(result))
-	}
-}
-
-// ---------------------------------------------------------------------------
-// Concurrency
-// ---------------------------------------------------------------------------
-
-// Concurrent LoadOrStore calls must not race or panic.
-func TestConcurrentLoadOrStore(t *testing.T) {
-	bank := NewLruStringBank(50, 100*time.Millisecond).(*lruStringBank)
-	defer bank.Stop()
-
-	const goroutines = 20
-	const opsEach = 100
-
-	var wg sync.WaitGroup
-	for i := 0; i < goroutines; i++ {
-		g := i
-		wg.Go(func() {
-			for i := 0; i < opsEach; i++ {
-				key := fmt.Sprintf("k%d", (g*opsEach+i)%30)
-				bank.LoadOrStore(key, key)
-			}
-		})
-	}
-
-	waiter := func() chan struct{} {
-		st := make(chan struct{})
-
-		go func() {
-			wg.Wait()
-			close(st)
-		}()
-
-		return st
-	}
-
-	select {
-	case <-waiter():
-		t.Logf("Completed Successfully\n")
-	case <-time.After(10 * time.Second):
-		t.Logf("Timed out\n")
-	}
-}
-
-// Concurrent calls interleaved with eviction cycles must not deadlock or race.
-func TestConcurrentLoadOrStoreWithEviction(t *testing.T) {
-	bank := NewLruStringBank(5, 50*time.Millisecond).(*lruStringBank)
-	defer bank.Stop()
-
-	const goroutines = 10
-	const duration = 300 * time.Millisecond
-
-	var wg sync.WaitGroup
-
-	for i := 0; i < goroutines; i++ {
-		g := i
-		stop := time.After(duration)
-
-		wg.Go(func() {
-			for {
-				select {
-				case <-stop:
-					return
-				default:
-					key := fmt.Sprintf("g%d", g)
-					bank.LoadOrStore(key, key)
-				}
-			}
-		})
-	}
-
-	waiter := func() chan struct{} {
-		st := make(chan struct{})
-
-		go func() {
-			wg.Wait()
-			close(st)
-		}()
-
-		return st
-	}
-
-	select {
-	case <-waiter():
-		t.Logf("Completed Successfully\n")
-	case <-time.After(10 * time.Second):
-		t.Logf("Timed out\n")
-	}
-}
-
-// Concurrent Clear calls alongside reads/writes must not panic.
-func TestConcurrentClear(t *testing.T) {
-	bank := NewLruStringBank(10, time.Minute).(*lruStringBank)
-	defer bank.Stop()
-
-	var wg sync.WaitGroup
-	for i := 0; i < 5; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			bank.LoadOrStore(fmt.Sprintf("k%d", i), "v")
-		}(i)
-	}
-	for i := 0; i < 3; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			bank.Clear()
-		}()
-	}
-
-	wg.Wait()
-}

core/pkg/util/stringutil/mapbank.go

@@ -1,48 +0,0 @@
-package stringutil
-
-import "sync"
-
-type stringBank struct {
-	lock sync.Mutex
-	m    map[string]string
-}
-
-func NewStringBank() StringBank {
-	return &stringBank{
-		m: make(map[string]string),
-	}
-}
-
-func (sb *stringBank) LoadOrStore(key, value string) (string, bool) {
-	sb.lock.Lock()
-
-	if v, ok := sb.m[key]; ok {
-		sb.lock.Unlock()
-		return v, ok
-	}
-
-	sb.m[value] = value
-	sb.lock.Unlock()
-	return value, false
-}
-
-func (sb *stringBank) LoadOrStoreFunc(key string, f func() string) (string, bool) {
-	sb.lock.Lock()
-
-	if v, ok := sb.m[key]; ok {
-		sb.lock.Unlock()
-		return v, ok
-	}
-
-	// create the key and value using the func (the key could be deallocated later)
-	value := f()
-	sb.m[value] = value
-	sb.lock.Unlock()
-	return value, false
-}
-
-func (sb *stringBank) Clear() {
-	sb.lock.Lock()
-	sb.m = make(map[string]string)
-	sb.lock.Unlock()
-}

core/pkg/util/stringutil/noopbank.go

@@ -1,17 +0,0 @@
-package stringutil
-
-type noOpStringBank struct{}
-
-func NewNoOpStringBank() StringBank {
-	return new(noOpStringBank)
-}
-
-func (nsb *noOpStringBank) LoadOrStore(key, value string) (string, bool) {
-	return value, true
-}
-
-func (nsb *noOpStringBank) LoadOrStoreFunc(key string, f func() string) (string, bool) {
-	return f(), true
-}
-
-func (nsb *noOpStringBank) Clear() {}

core/pkg/util/stringutil/stringutil.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
-	"sync"
 	"time"
 )
 
@@ -23,60 +22,10 @@ const (
 var alpha = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 var alphanumeric = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
 
-type StringBank interface {
-	LoadOrStore(key, value string) (string, bool)
-	LoadOrStoreFunc(key string, f func() string) (string, bool)
-	Clear()
-}
-
-var (
-	lock sync.RWMutex
-
-	// stringBank is an unbounded string cache that is thread-safe. It is especially useful if
-	// storing a large frequency of dynamically allocated duplicate strings.
-	strings StringBank = NewStringBank()
-)
-
 func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-func UpdateStringBank(sb StringBank) {
-	lock.Lock()
-	defer lock.Unlock()
-
-	strings.Clear()
-	strings = sb
-}
-
-// GetStringBank returns the _current_ StringBank implementation. Note that the read-lock is
-// not held for the duration of usage, so the returned string bank could be swapped out
-// after being retrieved.
-func GetStringBank() StringBank {
-	lock.RLock()
-	defer lock.RUnlock()
-
-	return strings
-}
-
-// Bank will return a non-copy of a string if it has been used before. Otherwise, it will store
-// the string as the unique instance.
-func Bank(s string) string {
-	ss, _ := GetStringBank().LoadOrStore(s, s)
-	return ss
-}
-
-// BankFunc will use the provided s string to check for an existing allocation of the string. However,
-// if no allocation exists, the f parameter will be used to create the string and store in the bank.
-func BankFunc(s string, f func() string) string {
-	ss, _ := GetStringBank().LoadOrStoreFunc(s, f)
-	return ss
-}
-
-func ClearBank() {
-	GetStringBank().Clear()
-}
-
 // RandSeq generates a pseudo-random alphabetic string of the given length
 func RandSeq(n int) string {
 	b := make([]rune, n)

core/pkg/util/stringutil/stringutil_test.go

@@ -1,256 +0,0 @@
-package stringutil_test
-
-import (
-	"fmt"
-	"math/rand"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-	"unsafe"
-
-	"github.com/opencost/opencost/core/pkg/util/stringutil"
-)
-
-var oldBank sync.Map
-
-type bankTest struct {
-	Bank     func(string) string
-	BankFunc func(string, func() string) string
-	Clear    func()
-}
-
-var (
-	legacyTest = bankTest{
-		Bank:     BankLegacy,
-		BankFunc: func(s string, f func() string) string { return s },
-		Clear:    ClearBankLegacy,
-	}
-
-	standardBankTest = bankTest{
-		Bank:     stringutil.Bank,
-		BankFunc: stringutil.BankFunc,
-		Clear:    stringutil.ClearBank,
-	}
-)
-
-// This is the old implementation of the string bank to use for comparison benchmarks
-func BankLegacy(s string) string {
-	ss, _ := oldBank.LoadOrStore(s, s)
-	return ss.(string)
-}
-
-func ClearBankLegacy() {
-	oldBank = sync.Map{}
-}
-
-func copyString(s string) string {
-	return string([]byte(s))
-}
-
-func generateBenchData(totalStrings, totalUnique int) [][]byte {
-	randStrings := make([]string, 0, totalStrings)
-	r := rand.New(rand.NewSource(27644437))
-
-	// create totalUnique unique strings
-	for range totalUnique {
-		randStrings = append(
-			randStrings,
-			fmt.Sprintf("%s/%s/%s", stringutil.RandSeqWith(r, 10), stringutil.RandSeqWith(r, 10), stringutil.RandSeqWith(r, 10)),
-		)
-	}
-
-	// set the seed such that the resulting "remainder" strings are deterministic for each bench
-	r = rand.New(rand.NewSource(1523942))
-
-	// append a random selection from 0-totalUnique to the list.
-	for range totalStrings - totalUnique {
-		randStrings = append(randStrings, strings.Clone(randStrings[r.Intn(totalUnique)]))
-	}
-
-	// shuffle the list of strings
-	r.Shuffle(totalStrings, func(i, j int) { randStrings[i], randStrings[j] = randStrings[j], randStrings[i] })
-
-	stringBytes := make([][]byte, 0, totalStrings)
-	for _, str := range randStrings {
-		stringBytes = append(stringBytes, []byte(str))
-	}
-	return stringBytes
-}
-
-func benchmarkStringBank(b *testing.B, bt bankTest, totalStrings, totalUnique int, useBankFunc bool) {
-	b.StopTimer()
-	randStrings := generateBenchData(totalStrings, totalUnique)
-
-	b.Run(b.Name(), func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			b.StartTimer()
-			for bb := 0; bb < totalStrings; bb++ {
-				bytes := randStrings[bb]
-
-				if useBankFunc {
-					str := unsafe.String(unsafe.SliceData(bytes), len(bytes))
-
-					bt.BankFunc(str, func() string {
-						return string(bytes)
-					})
-				} else {
-					bt.Bank(string(bytes))
-				}
-			}
-			b.StopTimer()
-			bt.Clear()
-			//runtime.GC()
-			//debug.FreeOSMemory()
-		}
-	})
-}
-
-func BenchmarkLegacyStringBank90PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, legacyTest, 1_000_000, 100_000, false)
-}
-
-func BenchmarkLegacyStringBank75PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, legacyTest, 1_000_000, 250_000, false)
-}
-
-func BenchmarkLegacyStringBank50PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, legacyTest, 1_000_000, 100_000, false)
-}
-
-func BenchmarkLegacyStringBank25PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, legacyTest, 1_000_000, 750_000, false)
-}
-
-func BenchmarkLegacyStringBankNoDuplicate(b *testing.B) {
-	benchmarkStringBank(b, legacyTest, 1_000_000, 1_000_000, false)
-}
-
-func BenchmarkStringBank90PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 100_000, false)
-}
-
-func BenchmarkStringBank75PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 250_000, false)
-}
-
-func BenchmarkStringBank50PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 100_000, false)
-}
-
-func BenchmarkStringBank25PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 750_000, false)
-}
-
-func BenchmarkStringBankNoDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 1_000_000, false)
-}
-
-func BenchmarkStringBankFunc90PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 100_000, true)
-}
-
-func BenchmarkStringBankFunc75PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 250_000, true)
-}
-
-func BenchmarkStringBankFunc50PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 100_000, true)
-}
-
-func BenchmarkStringBankFunc25PercentDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 750_000, true)
-}
-
-func BenchmarkStringBankFuncNoDuplicate(b *testing.B) {
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 1_000_000, true)
-}
-
-const LruCapacity = 500_000
-const LruEvictInterval = 5 * time.Second
-
-func BenchmarkLruStringBankFunc90PercentDuplicate(b *testing.B) {
-	prevBank := stringutil.GetStringBank()
-	defer func() {
-		stringutil.UpdateStringBank(prevBank)
-	}()
-
-	sb := stringutil.NewLruStringBank(LruCapacity, LruEvictInterval)
-	defer func() {
-		if lruBank, ok := sb.(interface{ Stop() }); ok {
-			lruBank.Stop()
-		}
-
-	}()
-
-	stringutil.UpdateStringBank(sb)
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 100_000, true)
-}
-
-func BenchmarkLruStringBankFunc75PercentDuplicate(b *testing.B) {
-	prevBank := stringutil.GetStringBank()
-	defer func() {
-		stringutil.UpdateStringBank(prevBank)
-	}()
-
-	sb := stringutil.NewLruStringBank(LruCapacity, LruEvictInterval)
-	defer func() {
-		if lruBank, ok := sb.(interface{ Stop() }); ok {
-			lruBank.Stop()
-		}
-	}()
-
-	stringutil.UpdateStringBank(sb)
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 250_000, true)
-}
-
-func BenchmarkLruStringBankFunc50PercentDuplicate(b *testing.B) {
-	prevBank := stringutil.GetStringBank()
-	defer func() {
-		stringutil.UpdateStringBank(prevBank)
-	}()
-
-	sb := stringutil.NewLruStringBank(LruCapacity, LruEvictInterval)
-	defer func() {
-		if lruBank, ok := sb.(interface{ Stop() }); ok {
-			lruBank.Stop()
-		}
-	}()
-
-	stringutil.UpdateStringBank(sb)
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 100_000, true)
-}
-
-func BenchmarkLruStringBankFunc25PercentDuplicate(b *testing.B) {
-	prevBank := stringutil.GetStringBank()
-	defer func() {
-		stringutil.UpdateStringBank(prevBank)
-	}()
-
-	sb := stringutil.NewLruStringBank(LruCapacity, LruEvictInterval)
-	defer func() {
-		if lruBank, ok := sb.(interface{ Stop() }); ok {
-			lruBank.Stop()
-		}
-	}()
-
-	stringutil.UpdateStringBank(sb)
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 750_000, true)
-}
-
-func BenchmarkLruStringBankFuncNoDuplicate(b *testing.B) {
-	prevBank := stringutil.GetStringBank()
-	defer func() {
-		stringutil.UpdateStringBank(prevBank)
-	}()
-
-	sb := stringutil.NewLruStringBank(LruCapacity, LruEvictInterval)
-	defer func() {
-		if lruBank, ok := sb.(interface{ Stop() }); ok {
-			lruBank.Stop()
-		}
-	}()
-
-	stringutil.UpdateStringBank(sb)
-	benchmarkStringBank(b, standardBankTest, 1_000_000, 1_000_000, true)
-}

pkg/cloud/aws/athenaquerier.go

@@ -11,9 +11,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/athena"
 	"github.com/aws/aws-sdk-go-v2/service/athena/types"
+	"github.com/opencost/bingen/pkg/util/stringutil"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/opencost"
-	"github.com/opencost/opencost/core/pkg/util/stringutil"
 	"github.com/opencost/opencost/pkg/cloud"
 )
 

pkg/cloud/aws/s3selectquerier.go

@@ -12,7 +12,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/opencost/opencost/core/pkg/util/stringutil"
+	"github.com/opencost/bingen/pkg/util/stringutil"
 	"github.com/opencost/opencost/pkg/cloud"
 )