Sth/kcm 4452 (#3376)

Signed-off-by: Sean Holcomb <seanholcomb@gmail.com>

core/pkg/exporter/decoder.go

@@ -0,0 +1,81 @@
+package exporter
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding"
+	"fmt"
+	"io"
+
+	"github.com/opencost/opencost/core/pkg/util/json"
+	"google.golang.org/protobuf/proto"
+)
+
+type Decoder[T any] func([]byte) (*T, error)
+
+// BinaryMarshalerPtr[T] is a generic constraint to ensure types passed to the encoder implement
+// encoding.BinaryMarshaler and are pointers to T.
+type BinaryUnmarshalerPtr[T any] interface {
+	encoding.BinaryUnmarshaler
+	*T
+}
+
+func BingenDecoder[T any, U BinaryUnmarshalerPtr[T]](data []byte) (*T, error) {
+	var set U = new(T)
+
+	err := set.UnmarshalBinary(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode bingen: %w", err)
+	}
+
+	return set, nil
+}
+
+func JSONDecoder[T any](data []byte) (*T, error) {
+	var instance = new(T)
+	err := json.Unmarshal(data, instance)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode json: %w", err)
+	}
+
+	return instance, nil
+}
+
+func ProtobufDecoder[T any, U ProtoMessagePtr[T]](data []byte) (*T, error) {
+	var message U = new(T)
+
+	err := proto.Unmarshal(data, message)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode protobuf message: %w", err)
+	}
+
+	return message, nil
+}
+
+func GetGzipDecoder[T any](decoder Decoder[T]) Decoder[T] {
+	return func(data []byte) (*T, error) {
+		// Check for gzip compression. Ref: https://www.ietf.org/rfc/rfc1952.txt Page 5
+		if len(data) > 2 && data[0] == 0x1F && data[1] == 0x8B {
+			buf := bytes.NewBuffer(data)
+			reader, err := gzip.NewReader(buf)
+			if err != nil {
+				return nil, fmt.Errorf("failed to decompress gzip: %w", err)
+
+			}
+			defer reader.Close()
+			decompressed, err := io.ReadAll(reader)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read decompressed gzip: %w", err)
+			}
+
+			data = decompressed
+		}
+
+		instance, err := decoder(data)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode decompress gzip data: %w", err)
+		}
+
+		return instance, nil
+	}
+}

core/pkg/exporter/decoder_test.go

@@ -0,0 +1,326 @@
+package exporter
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/diagnostics"
+	"github.com/opencost/opencost/core/pkg/heartbeat"
+	"github.com/opencost/opencost/core/pkg/model"
+	"github.com/opencost/opencost/core/pkg/model/pb"
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/core/pkg/util"
+	"github.com/opencost/opencost/core/pkg/util/json"
+	"google.golang.org/protobuf/proto"
+)
+
+type decoderTestCase[T any] struct {
+	name    string
+	data    []byte
+	want    *T
+	wantErr bool
+}
+
+func generateBadBytes() []byte {
+	buff := util.NewBuffer()
+	for i := 0; i < 10; i++ {
+		buff.WriteUInt64(9999999)
+	}
+
+	return buff.Bytes()
+}
+
+func TestBingenDecoder(t *testing.T) {
+	badBytes := generateBadBytes()
+
+	now := time.Now().UTC().Truncate(24 * time.Hour)
+	start := now.Add(-(24 * 5) * time.Hour)
+
+	// Define and Run Allocation Tests
+	allocSet := opencost.GenerateMockAllocationSet(start)
+	allocSetRaw, err := allocSet.MarshalBinary()
+	if err != nil {
+		t.Errorf("failed to marshal allocation set: %s", err.Error())
+	}
+
+	allocTests := []decoderTestCase[opencost.AllocationSet]{
+		{
+			name:    "allocation valid",
+			data:    allocSetRaw,
+			want:    allocSet,
+			wantErr: false,
+		},
+		{
+			name:    "allocation invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testDecoder(t, BingenDecoder, allocTests)
+
+	// Define and Run Asset Tests
+	assetSet := opencost.GenerateMockAssetSet(start, 24*time.Hour)
+	assetSetRaw, err := assetSet.MarshalBinary()
+	if err != nil {
+		t.Errorf("failed to marshal asset set: %s", err.Error())
+	}
+
+	assetTests := []decoderTestCase[opencost.AssetSet]{
+		{
+			name:    "asset valid",
+			data:    assetSetRaw,
+			want:    assetSet,
+			wantErr: false,
+		},
+		{
+			name:    "asset invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testDecoder(t, BingenDecoder, assetTests)
+
+	// Define and Run Cloud Cost Tests
+	CloudCostSet := opencost.GenerateMockCloudCostSet(start, start.Add(24*time.Hour), "gcp", "gke")
+	CloudCostSetRaw, err := CloudCostSet.MarshalBinary()
+	if err != nil {
+		t.Errorf("failed to marshal cloud cost set: %s", err.Error())
+	}
+
+	cloudCostTests := []decoderTestCase[opencost.CloudCostSet]{
+		{
+			name:    "cloud cost valid",
+			data:    CloudCostSetRaw,
+			want:    CloudCostSet,
+			wantErr: false,
+		},
+		{
+			name:    "cloud cost invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testDecoder(t, BingenDecoder, cloudCostTests)
+
+	// Define and Run Network Insight Tests
+	networkInsightSet := opencost.GenerateMockNetworkInsightSet(start, start.Add(24*time.Hour))
+	networkInsightSetRaw, err := networkInsightSet.MarshalBinary()
+	if err != nil {
+		t.Errorf("failed to marshal network insight set: %s", err.Error())
+	}
+
+	networkInsightTests := []decoderTestCase[opencost.NetworkInsightSet]{
+		{
+			name:    "network insight valid",
+			data:    networkInsightSetRaw,
+			want:    networkInsightSet,
+			wantErr: false,
+		},
+		{
+			name:    "network insight invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testDecoder(t, BingenDecoder, networkInsightTests)
+}
+
+func TestJsonDecoder(t *testing.T) {
+	badBytes := generateBadBytes()
+
+	now := time.Now().UTC().Truncate(24 * time.Hour)
+	start := now.Add(-(24 * 5) * time.Hour)
+
+	hb := heartbeat.Heartbeat{
+		Id:          "heartBeatID",
+		Timestamp:   start,
+		Uptime:      123,
+		Application: "mock",
+		Version:     "test",
+		Metadata: map[string]any{
+			"str": "test",
+			"num": 1.0,
+		},
+	}
+	hbraw, err := json.Marshal(hb)
+	if err != nil {
+		t.Errorf("failed to marshal heartbeat: %s", err.Error())
+	}
+
+	heartbeatTests := []decoderTestCase[heartbeat.Heartbeat]{
+		{
+			name:    "heartbeat valid",
+			data:    hbraw,
+			want:    &hb,
+			wantErr: false,
+		},
+		{
+			name:    "heartbeat invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testDecoder(t, JSONDecoder, heartbeatTests)
+}
+
+func TestGzipDecoder(t *testing.T) {
+	badBytes := generateBadBytes()
+
+	now := time.Now().UTC().Truncate(24 * time.Hour)
+	start := now.Add(-(24 * 5) * time.Hour)
+
+	diag := diagnostics.DiagnosticResult{
+		ID:          "diagnosticID",
+		Name:        "diagnisticName",
+		Description: "Test Diagnostic",
+		Category:    "test",
+		Timestamp:   start,
+		Error:       "test",
+		Details: map[string]any{
+			"str": "test",
+			"num": 1.0,
+		},
+	}
+	diagRaw, err := json.Marshal(diag)
+	if err != nil {
+		t.Errorf("failed to marshal diagnostic: %s", err.Error())
+	}
+	diagCompressed, err := gZipEncode(diagRaw)
+	if err != nil {
+		t.Errorf("failed to compress diagnostic: %s", err.Error())
+	}
+
+	badCompressed, err := gZipEncode(badBytes)
+	if err != nil {
+		t.Errorf("failed to compress bad bytes: %s", err.Error())
+	}
+
+	diagnosticTests := []decoderTestCase[diagnostics.DiagnosticResult]{
+		{
+			name:    "diagnostic valid",
+			data:    diagCompressed,
+			want:    &diag,
+			wantErr: false,
+		},
+		{
+			name:    "diagnostic invalid",
+			data:    badCompressed,
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "diagnostic bypass valid",
+			data:    diagRaw,
+			want:    &diag,
+			wantErr: false,
+		},
+		{
+			name:    "diagnostic bypass invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testDecoder(t, GetGzipDecoder[diagnostics.DiagnosticResult](JSONDecoder), diagnosticTests)
+}
+
+func TestProtobufDecoder(t *testing.T) {
+	badBytes := generateBadBytes()
+
+	now := time.Now().UTC().Truncate(24 * time.Hour)
+	start := now.Add(-(24 * 5) * time.Hour)
+
+	customCostSet := model.GenerateMockCustomCostSet(start, start.Add(24*time.Hour))
+	customCostSetRaw, err := proto.Marshal(customCostSet)
+	if err != nil {
+		t.Errorf("failed to marshal custom cost set: %s", err.Error())
+	}
+
+	customCostTests := []decoderTestCase[pb.CustomCostResponse]{
+		{
+			name:    "custom cost valid",
+			data:    customCostSetRaw,
+			want:    customCostSet,
+			wantErr: false,
+		},
+		{
+			name:    "custom cost invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testProtoBufDecoder(t, ProtobufDecoder, customCostTests)
+
+	labelsResponse := model.GenerateMockLabelResponse(start, "1d")
+	labelsResponseRaw, err := proto.Marshal(labelsResponse)
+	if err != nil {
+		t.Errorf("failed to marshal custom cost set: %s", err.Error())
+	}
+
+	labelsResponseTests := []decoderTestCase[pb.LabelsResponse]{
+		{
+			name:    "labels response valid",
+			data:    labelsResponseRaw,
+			want:    labelsResponse,
+			wantErr: false,
+		},
+		{
+			name:    "labels response invalid",
+			data:    badBytes,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	testProtoBufDecoder(t, ProtobufDecoder, labelsResponseTests)
+}
+
+func testProtoBufDecoder[T any, U ProtoMessagePtr[T]](t *testing.T, decoder Decoder[T], testCases []decoderTestCase[T]) {
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := decoder(tt.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Decoder() error = %v, wantErr %v", err, tt.wantErr)
+				if err != nil {
+					t.Errorf("Error: %s", err.Error())
+				}
+				return
+			}
+			if !proto.Equal(U(got), U(tt.want)) {
+				t.Errorf("Decoder() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func testDecoder[T any](t *testing.T, decoder Decoder[T], testCases []decoderTestCase[T]) {
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := decoder(tt.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Decoder() error = %v, wantErr %v", err, tt.wantErr)
+				if err != nil {
+					t.Errorf("Error: %s", err.Error())
+				}
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Decoder() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

core/pkg/exporter/encoder.go

@@ -4,8 +4,11 @@ import (
 	"bytes"
 	"compress/gzip"
 	"encoding"
+	"fmt"
 
 	"github.com/opencost/opencost/core/pkg/util/json"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 )
 
 // Encoder[T] is a generic interface for encoding an instance of a T type into a byte slice.
@@ -83,9 +86,17 @@ func NewGZipEncoder[T any](encoder Encoder[T]) Encoder[T] {
 func (gz *GZipEncoder[T]) Encode(data *T) ([]byte, error) {
 	encoded, err := gz.encoder.Encode(data)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("GZipEncoder: nested encode failure: %w", err)
 	}
 
+	compressed, err := gZipEncode(encoded)
+	if err != nil {
+		return nil, fmt.Errorf("GZipEncoder: failed to compress encoded data: %w", err)
+	}
+	return compressed, nil
+}
+
+func gZipEncode(data []byte) ([]byte, error) {
 	var buf bytes.Buffer
 
 	gzWriter, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
@@ -93,7 +104,7 @@ func (gz *GZipEncoder[T]) Encode(data *T) ([]byte, error) {
 		return nil, err
 	}
 
-	gzWriter.Write(encoded)
+	gzWriter.Write(data)
 	gzWriter.Close()
 
 	return buf.Bytes(), nil
@@ -104,3 +115,62 @@ func (gz *GZipEncoder[T]) Encode(data *T) ([]byte, error) {
 func (gz *GZipEncoder[T]) FileExt() string {
 	return gz.encoder.FileExt() + ".gz"
 }
+
+// ProtoMessagePtr [T] is a generic constraint to ensure types passed to the encoder implement
+// proto.Message and are pointers to T.
+type ProtoMessagePtr[T any] interface {
+	proto.Message
+	*T
+}
+
+// ProtobufEncoder [T, U] is a generic encoder that uses the proto.Message interface to encode data.
+// It supports any type T that implements the proto.Message interface.
+type ProtobufEncoder[T any, U ProtoMessagePtr[T]] struct{}
+
+// NewProtobufEncoder creates an `Encoder[T]` implementation which supports binary encoding for the `T`
+// type.
+func NewProtobufEncoder[T any, U ProtoMessagePtr[T]]() Encoder[T] {
+	return new(ProtobufEncoder[T, U])
+}
+
+// Encode encodes the provided data of type T into a byte slice using the proto.Message interface.
+func (p *ProtobufEncoder[T, U]) Encode(data *T) ([]byte, error) {
+	var message U = data
+	raw, err := proto.Marshal(message)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode protobuf message: %w", err)
+	}
+	return raw, nil
+}
+
+// FileExt returns the file extension for the encoded data. In this case, it returns an empty string
+// to indicate that there is no specific file extension for the binary encoded data.
+func (p *ProtobufEncoder[T, U]) FileExt() string {
+	return "binpb"
+}
+
+// ProtoJsonEncoder [T, U] is a generic encoder that uses the proto.Message interface to encode data in json format.
+// It supports any type T that implements the proto.Message interface.
+type ProtoJsonEncoder[T any, U ProtoMessagePtr[T]] struct{}
+
+// NewProtoJsonEncoder creates an `Encoder[T]` implementation which supports binary encoding for the `T`
+// type.
+func NewProtoJsonEncoder[T any, U ProtoMessagePtr[T]]() Encoder[T] {
+	return new(ProtoJsonEncoder[T, U])
+}
+
+// Encode encodes the provided data of type T into a byte slice using the proto.Message interface.
+func (p *ProtoJsonEncoder[T, U]) Encode(data *T) ([]byte, error) {
+	var message U = data
+	raw, err := protojson.Marshal(message)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode protobuf message to json: %w", err)
+	}
+	return raw, nil
+}
+
+// FileExt returns the file extension for the encoded data. In this case, it returns an empty string
+// to indicate that there is no specific file extension for the binary encoded data.
+func (p *ProtoJsonEncoder[T, U]) FileExt() string {
+	return "json"
+}

core/pkg/model/helper.go

@@ -0,0 +1,35 @@
+package model
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/model/pb"
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/core/pkg/util/timeutil"
+)
+
+// ConvertWindow validates and converts a protobuf window to a closed opencost.Window or returns an error
+func ConvertWindow(window *pb.Window) (opencost.Window, error) {
+	if window == nil {
+		return opencost.Window{}, fmt.Errorf("cannot convert nil window")
+	}
+	var res time.Duration
+	switch window.Resolution {
+	case "1d":
+		res = timeutil.Day
+	case "1h":
+		res = time.Hour
+	case "10m":
+		res = time.Minute * 10
+	default:
+		return opencost.Window{}, fmt.Errorf("invalid window resolution %s", window.Resolution)
+	}
+
+	start := window.Start.AsTime().UTC()
+	if !start.Equal(start.Truncate(res)) {
+		return opencost.Window{}, fmt.Errorf("invalid start time for resolution '%s': %s", window.Resolution, start.Format(time.RFC3339))
+	}
+	win := opencost.NewClosedWindow(start, start.Add(res))
+	return win, nil
+}

core/pkg/model/helper_test.go

@@ -0,0 +1,89 @@
+package model
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/model/pb"
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/core/pkg/util/timeutil"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestConvertWindow(t *testing.T) {
+	timeDay := time.Date(2025, 9, 1, 0, 0, 0, 0, time.UTC)
+	timeHour := timeDay.Add(time.Hour)
+	timeTenMinute := timeHour.Add(time.Minute * 10)
+	invalidTime := timeTenMinute.Add(time.Second)
+	tests := []struct {
+		name    string
+		window  *pb.Window
+		want    opencost.Window
+		wantErr bool
+	}{
+		{
+			name:    "nil window",
+			window:  nil,
+			want:    opencost.Window{},
+			wantErr: true,
+		},
+		{
+			name: "invalid resolution",
+			window: &pb.Window{
+				Resolution: "invalid",
+				Start:      timestamppb.New(timeDay),
+			},
+			want:    opencost.Window{},
+			wantErr: true,
+		},
+		{
+			name: "invalid time",
+			window: &pb.Window{
+				Resolution: "1d",
+				Start:      timestamppb.New(invalidTime),
+			},
+			want:    opencost.Window{},
+			wantErr: true,
+		},
+		{
+			name: "valid 1d",
+			window: &pb.Window{
+				Resolution: "1d",
+				Start:      timestamppb.New(timeDay),
+			},
+			want:    opencost.NewClosedWindow(timeDay, timeDay.Add(timeutil.Day)),
+			wantErr: false,
+		},
+		{
+			name: "valid 1h",
+			window: &pb.Window{
+				Resolution: "1h",
+				Start:      timestamppb.New(timeHour),
+			},
+			want:    opencost.NewClosedWindow(timeHour, timeHour.Add(time.Hour)),
+			wantErr: false,
+		},
+		{
+			name: "valid 10m",
+			window: &pb.Window{
+				Resolution: "10m",
+				Start:      timestamppb.New(timeTenMinute),
+			},
+			want:    opencost.NewClosedWindow(timeTenMinute, timeTenMinute.Add(10*time.Minute)),
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ConvertWindow(tt.window)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ConvertWindow() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ConvertWindow() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

core/pkg/model/mock.go

@@ -0,0 +1,83 @@
+package model
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/model/pb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func createCustomCost(postfix string) *pb.CustomCost {
+	n := func(a string) string {
+		return fmt.Sprintf("%s_%s", a, postfix)
+	}
+
+	cost := rand.Float32() * 250.0
+
+	return &pb.CustomCost{
+		Metadata: map[string]string{
+			n("custom_cost"): n("metadata"),
+		},
+		Zone:           "zone-a",
+		AccountName:    n("account"),
+		ChargeCategory: n("charge"),
+		Description:    "this is a test cost description(" + postfix + ")",
+		ResourceName:   "test-custom-cost-" + postfix,
+		ResourceType:   "custom",
+		Id:             n("id"),
+		ProviderId:     "gke",
+		BilledCost:     cost,
+		ListCost:       cost,
+		ListUnitPrice:  cost,
+		UsageQuantity:  1.0,
+		UsageUnit:      n("unit"),
+		Labels: map[string]string{
+			n("label"): n("value"),
+		},
+	}
+}
+
+func GenerateMockCustomCostSet(start, end time.Time) *pb.CustomCostResponse {
+	costs := []*pb.CustomCost{}
+
+	for i := 0; i < 50; i++ {
+		costs = append(costs, createCustomCost(fmt.Sprintf("%d", i)))
+	}
+
+	return &pb.CustomCostResponse{
+		Metadata: map[string]string{
+			"key1": "value1",
+			"test": "1, 2, 3",
+		},
+		CostSource: "none",
+		Domain:     "testing",
+		Version:    "v1",
+		Currency:   "USD",
+		Start:      timestamppb.New(start),
+		End:        timestamppb.New(end),
+		Costs:      costs,
+	}
+}
+
+func GenerateMockLabelResponse(start time.Time, res string) *pb.LabelsResponse {
+	return &pb.LabelsResponse{
+		Type:    "account-labels",
+		GroupId: "billing_account_xzy",
+		Window: &pb.Window{
+			Resolution: res,
+			Start:      timestamppb.New(start),
+		},
+		LabelSets: map[string]*pb.LabelSet{
+			"account1": {Labels: map[string]string{
+				"account": "account1",
+				"test":    "test1",
+			}},
+			"account2": {Labels: map[string]string{
+				"account": "account2",
+				"test":    "test2",
+			}},
+		},
+	}
+}

core/pkg/model/pb/labels.pb.go

@@ -0,0 +1,218 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.9
+// 	protoc        v5.29.3
+// source: protos/model/labels.proto
+
+package pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// LabelsResponse is a grouping of LabelSets for a specific window for a type of label grouped by their source.
+type LabelsResponse struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// The type of labels in the set (e.g. account_labels, namespace_labels)
+	Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"`
+	// type dependent identifier for the grouping of labels most likely the cluster id or the configuration key, used
+	// as part of the export path
+	GroupId string `protobuf:"bytes,2,opt,name=group_id,json=groupId,proto3" json:"group_id,omitempty"`
+	// The window for the label sets
+	Window *Window `protobuf:"bytes,3,opt,name=window,proto3" json:"window,omitempty"`
+	// Mapping of LabelSets for individual items by a unique identifier
+	LabelSets     map[string]*LabelSet `protobuf:"bytes,4,rep,name=label_sets,json=labelSets,proto3" json:"label_sets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *LabelsResponse) Reset() {
+	*x = LabelsResponse{}
+	mi := &file_protos_model_labels_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *LabelsResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LabelsResponse) ProtoMessage() {}
+
+func (x *LabelsResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_protos_model_labels_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LabelsResponse.ProtoReflect.Descriptor instead.
+func (*LabelsResponse) Descriptor() ([]byte, []int) {
+	return file_protos_model_labels_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *LabelsResponse) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *LabelsResponse) GetGroupId() string {
+	if x != nil {
+		return x.GroupId
+	}
+	return ""
+}
+
+func (x *LabelsResponse) GetWindow() *Window {
+	if x != nil {
+		return x.Window
+	}
+	return nil
+}
+
+func (x *LabelsResponse) GetLabelSets() map[string]*LabelSet {
+	if x != nil {
+		return x.LabelSets
+	}
+	return nil
+}
+
+// LabelSet is an internal message meant to enable nesting maps
+type LabelSet struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Labels        map[string]string      `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *LabelSet) Reset() {
+	*x = LabelSet{}
+	mi := &file_protos_model_labels_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *LabelSet) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*LabelSet) ProtoMessage() {}
+
+func (x *LabelSet) ProtoReflect() protoreflect.Message {
+	mi := &file_protos_model_labels_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use LabelSet.ProtoReflect.Descriptor instead.
+func (*LabelSet) Descriptor() ([]byte, []int) {
+	return file_protos_model_labels_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *LabelSet) GetLabels() map[string]string {
+	if x != nil {
+		return x.Labels
+	}
+	return nil
+}
+
+var File_protos_model_labels_proto protoreflect.FileDescriptor
+
+const file_protos_model_labels_proto_rawDesc = "" +
+	"\n" +
+	"\x19protos/model/labels.proto\x12\x05model\x1a\x19protos/model/window.proto\"\xfa\x01\n" +
+	"\x0eLabelsResponse\x12\x12\n" +
+	"\x04type\x18\x01 \x01(\tR\x04type\x12\x19\n" +
+	"\bgroup_id\x18\x02 \x01(\tR\agroupId\x12%\n" +
+	"\x06window\x18\x03 \x01(\v2\r.model.WindowR\x06window\x12C\n" +
+	"\n" +
+	"label_sets\x18\x04 \x03(\v2$.model.LabelsResponse.LabelSetsEntryR\tlabelSets\x1aM\n" +
+	"\x0eLabelSetsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12%\n" +
+	"\x05value\x18\x02 \x01(\v2\x0f.model.LabelSetR\x05value:\x028\x01\"z\n" +
+	"\bLabelSet\x123\n" +
+	"\x06labels\x18\x01 \x03(\v2\x1b.model.LabelSet.LabelsEntryR\x06labels\x1a9\n" +
+	"\vLabelsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B0Z.github.com/opencost/opencost/core/pkg/model/pbb\x06proto3"
+
+var (
+	file_protos_model_labels_proto_rawDescOnce sync.Once
+	file_protos_model_labels_proto_rawDescData []byte
+)
+
+func file_protos_model_labels_proto_rawDescGZIP() []byte {
+	file_protos_model_labels_proto_rawDescOnce.Do(func() {
+		file_protos_model_labels_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_protos_model_labels_proto_rawDesc), len(file_protos_model_labels_proto_rawDesc)))
+	})
+	return file_protos_model_labels_proto_rawDescData
+}
+
+var file_protos_model_labels_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_protos_model_labels_proto_goTypes = []any{
+	(*LabelsResponse)(nil), // 0: model.LabelsResponse
+	(*LabelSet)(nil),       // 1: model.LabelSet
+	nil,                    // 2: model.LabelsResponse.LabelSetsEntry
+	nil,                    // 3: model.LabelSet.LabelsEntry
+	(*Window)(nil),         // 4: model.Window
+}
+var file_protos_model_labels_proto_depIdxs = []int32{
+	4, // 0: model.LabelsResponse.window:type_name -> model.Window
+	2, // 1: model.LabelsResponse.label_sets:type_name -> model.LabelsResponse.LabelSetsEntry
+	3, // 2: model.LabelSet.labels:type_name -> model.LabelSet.LabelsEntry
+	1, // 3: model.LabelsResponse.LabelSetsEntry.value:type_name -> model.LabelSet
+	4, // [4:4] is the sub-list for method output_type
+	4, // [4:4] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
+}
+
+func init() { file_protos_model_labels_proto_init() }
+func file_protos_model_labels_proto_init() {
+	if File_protos_model_labels_proto != nil {
+		return
+	}
+	file_protos_model_window_proto_init()
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_protos_model_labels_proto_rawDesc), len(file_protos_model_labels_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_protos_model_labels_proto_goTypes,
+		DependencyIndexes: file_protos_model_labels_proto_depIdxs,
+		MessageInfos:      file_protos_model_labels_proto_msgTypes,
+	}.Build()
+	File_protos_model_labels_proto = out.File
+	file_protos_model_labels_proto_goTypes = nil
+	file_protos_model_labels_proto_depIdxs = nil
+}

core/pkg/model/pb/window.pb.go

@@ -0,0 +1,139 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.9
+// 	protoc        v5.29.3
+// source: protos/model/window.proto
+
+package pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// Window defines a unit of time by a resolution and a start time
+type Window struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// resolution in "1h" or "1d" format
+	Resolution string `protobuf:"bytes,1,opt,name=resolution,proto3" json:"resolution,omitempty"`
+	// the start time of the window described
+	Start         *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=start,proto3" json:"start,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Window) Reset() {
+	*x = Window{}
+	mi := &file_protos_model_window_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Window) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Window) ProtoMessage() {}
+
+func (x *Window) ProtoReflect() protoreflect.Message {
+	mi := &file_protos_model_window_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Window.ProtoReflect.Descriptor instead.
+func (*Window) Descriptor() ([]byte, []int) {
+	return file_protos_model_window_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Window) GetResolution() string {
+	if x != nil {
+		return x.Resolution
+	}
+	return ""
+}
+
+func (x *Window) GetStart() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Start
+	}
+	return nil
+}
+
+var File_protos_model_window_proto protoreflect.FileDescriptor
+
+const file_protos_model_window_proto_rawDesc = "" +
+	"\n" +
+	"\x19protos/model/window.proto\x12\x05model\x1a\x1fgoogle/protobuf/timestamp.proto\"Z\n" +
+	"\x06Window\x12\x1e\n" +
+	"\n" +
+	"resolution\x18\x01 \x01(\tR\n" +
+	"resolution\x120\n" +
+	"\x05start\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x05startB0Z.github.com/opencost/opencost/core/pkg/model/pbb\x06proto3"
+
+var (
+	file_protos_model_window_proto_rawDescOnce sync.Once
+	file_protos_model_window_proto_rawDescData []byte
+)
+
+func file_protos_model_window_proto_rawDescGZIP() []byte {
+	file_protos_model_window_proto_rawDescOnce.Do(func() {
+		file_protos_model_window_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_protos_model_window_proto_rawDesc), len(file_protos_model_window_proto_rawDesc)))
+	})
+	return file_protos_model_window_proto_rawDescData
+}
+
+var file_protos_model_window_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_protos_model_window_proto_goTypes = []any{
+	(*Window)(nil),                // 0: model.Window
+	(*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp
+}
+var file_protos_model_window_proto_depIdxs = []int32{
+	1, // 0: model.Window.start:type_name -> google.protobuf.Timestamp
+	1, // [1:1] is the sub-list for method output_type
+	1, // [1:1] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
+}
+
+func init() { file_protos_model_window_proto_init() }
+func file_protos_model_window_proto_init() {
+	if File_protos_model_window_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_protos_model_window_proto_rawDesc), len(file_protos_model_window_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_protos_model_window_proto_goTypes,
+		DependencyIndexes: file_protos_model_window_proto_depIdxs,
+		MessageInfos:      file_protos_model_window_proto_msgTypes,
+	}.Build()
+	File_protos_model_window_proto = out.File
+	file_protos_model_window_proto_goTypes = nil
+	file_protos_model_window_proto_depIdxs = nil
+}

core/pkg/opencost/mock.go

@@ -930,3 +930,86 @@ func GenerateMockNetworkInsightSet(start time.Time, end time.Time) *NetworkInsig
 
 	return nis
 }
+
+func GenerateMockCloudCostSet(start, end time.Time, provider, integration string) *CloudCostSet {
+	ccs := NewCloudCostSet(start, end)
+
+	ccs.Integration = integration
+
+	ccs.Insert(&CloudCost{
+		Window: ccs.Window,
+		Properties: &CloudCostProperties{
+			Provider:        provider,
+			AccountID:       "account1",
+			InvoiceEntityID: "invoiceEntity1",
+			Service:         provider + "-storage",
+			Category:        StorageCategory,
+			Labels: CloudCostLabels{
+				"label1": "value1",
+				"label2": "value2",
+				"label3": "value3",
+			},
+			ProviderID: "id1",
+		},
+		ListCost: CostMetric{
+			Cost:              100,
+			KubernetesPercent: 0,
+		},
+		NetCost: CostMetric{
+			Cost:              100,
+			KubernetesPercent: 0,
+		},
+	})
+
+	ccs.Insert(&CloudCost{
+		Window: ccs.Window,
+		Properties: &CloudCostProperties{
+			Provider:        provider,
+			AccountID:       "account1",
+			InvoiceEntityID: "invoiceEntity1",
+			Service:         provider + "-compute",
+			Category:        ComputeCategory,
+			Labels: CloudCostLabels{
+				"label1": "value1",
+				"label2": "value2",
+				"label3": "value3",
+			},
+			ProviderID: "id2",
+		},
+		ListCost: CostMetric{
+			Cost:              2000,
+			KubernetesPercent: 1,
+		},
+		NetCost: CostMetric{
+			Cost:              1800,
+			KubernetesPercent: 1,
+		},
+	})
+
+	ccs.Insert(&CloudCost{
+		Window: ccs.Window,
+		Properties: &CloudCostProperties{
+			Provider:        provider,
+			AccountID:       "account2",
+			InvoiceEntityID: "invoiceEntity2",
+			Service:         provider + "-compute",
+			Category:        ComputeCategory,
+			Labels: CloudCostLabels{
+				"label1": "value1",
+				"label2": "value2",
+				"label3": "value3",
+			},
+			ProviderID: "id3",
+		},
+		ListCost: CostMetric{
+			Cost:              8000,
+			KubernetesPercent: 1,
+		},
+		NetCost: CostMetric{
+			Cost:              8000,
+			KubernetesPercent: 1,
+		},
+	})
+
+	return ccs
+}

modules/collector-source/pkg/metric/walinator.go

@@ -19,7 +19,7 @@ import (
 	"github.com/opencost/opencost/modules/collector-source/pkg/util"
 )
 
-const ControllerEventName = "controller"
+const CollectorEventName = "collector"
 
 type fileInfo struct {
 	name      string
@@ -48,7 +48,7 @@ func NewWalinator(
 			limitResolution = resolution
 		}
 	}
-	pathFormatter, err := pathing.NewEventStoragePathFormatter(applicationName, clusterID, ControllerEventName)
+	pathFormatter, err := pathing.NewEventStoragePathFormatter(applicationName, clusterID, CollectorEventName)
 	if err != nil {
 		return nil, fmt.Errorf("filed to create path formatter for scrape controller: %s", err.Error())
 	}

pkg/cloud/gcp/bigqueryconfiguration.go

@@ -12,10 +12,11 @@ import (
 )
 
 type BigQueryConfiguration struct {
-	ProjectID  string     `json:"projectID"`
-	Dataset    string     `json:"dataset"`
-	Table      string     `json:"table"`
-	Authorizer Authorizer `json:"authorizer"`
+	ProjectID            string     `json:"projectID"`
+	Dataset              string     `json:"dataset"`
+	Table                string     `json:"table"`
+	ExcludePartitionTime bool       `json:"excludePartitionTime"`
+	Authorizer           Authorizer `json:"authorizer"`
 }
 
 func (bqc *BigQueryConfiguration) Validate() error {

pkg/cloud/gcp/bigqueryintegration.go

@@ -26,6 +26,7 @@ const (
 	ServiceDescriptionColumnName = "service"
 	SKUDescriptionColumnName     = "description"
 	LabelsColumnName             = "labels"
+	ProjectLabelsColumnName      = "project_labels"
 	ResourceNameColumnName       = "resource"
 	ResourceGlobalNameColumnName = "global_resource"
 	CostColumnName               = "cost"
@@ -74,7 +75,7 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		ResourceGlobalNameColumnName,
 	}
 
-	whereConjuncts := GetWhereConjuncts(start, end)
+	whereConjuncts := GetWhereConjuncts(start, end, bqi.ExcludePartitionTime)
 
 	columnStr := strings.Join(selectColumns, ", ")
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
@@ -127,12 +128,17 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 
 // GetWhereConjuncts creates a list of Where filter statements that filter for usage start date and partition time
 // additional filters can be added before combining into the final where clause
-func GetWhereConjuncts(start time.Time, end time.Time) []string {
-	partitionStart := start
-	partitionEnd := end.AddDate(0, 0, 2)
-	wherePartition := fmt.Sprintf(BiqQueryWherePartitionFmt, partitionStart.Format("2006-01-02"), partitionEnd.Format("2006-01-02"))
+func GetWhereConjuncts(start time.Time, end time.Time, excludePartitions bool) []string {
+	var conjuncts []string
+	if !excludePartitions {
+		partitionStart := start
+		partitionEnd := end.AddDate(0, 0, 2)
+		wherePartition := fmt.Sprintf(BiqQueryWherePartitionFmt, partitionStart.Format("2006-01-02"), partitionEnd.Format("2006-01-02"))
+		conjuncts = append(conjuncts, wherePartition)
+	}
 	whereDate := fmt.Sprintf(BiqQueryWhereDateFmt, start.Format("2006-01-02"), end.Format("2006-01-02"))
-	return []string{wherePartition, whereDate}
+	conjuncts = append(conjuncts, whereDate)
+	return conjuncts
 }
 
 // FlexibleCUDRates are the total amount paid / total amount credited per day for all Flexible CUDs. Since credited will be a negative value
@@ -194,7 +200,7 @@ func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCosts(start time.Time, end
 	`
 
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
-	whereConjuncts := GetWhereConjuncts(start, end)
+	whereConjuncts := GetWhereConjuncts(start, end, bqi.ExcludePartitionTime)
 	whereConjuncts = append(whereConjuncts, "sku.description like 'Commitment - dollar based v1:%'")
 	whereClause := strings.Join(whereConjuncts, " AND ")
 	query := fmt.Sprintf(queryFmt, table, whereClause)
@@ -227,7 +233,7 @@ func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCredits(start time.Time, en
 	`
 
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
-	whereConjuncts := GetWhereConjuncts(start, end)
+	whereConjuncts := GetWhereConjuncts(start, end, bqi.ExcludePartitionTime)
 	whereConjuncts = append(whereConjuncts, "credits.type = 'COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE'")
 	whereClause := strings.Join(whereConjuncts, " AND ")
 	query := fmt.Sprintf(queryFmt, table, whereClause)

pkg/env/cloudcost.go

@@ -44,28 +44,28 @@ func GetCloudCostMonthToDateInterval() int {
 	return env.GetInt(CloudCostMonthToDateIntervalVar, 6)
 }
 
-func GetCloudCostRefreshRateHours() int64 {
-	return env.GetInt64(CloudCostRefreshRateHoursEnvVar, 6)
+func GetCloudCostRefreshRateHours() int {
+	return env.GetInt(CloudCostRefreshRateHoursEnvVar, 6)
 }
 
-func GetCloudCostQueryWindowDays() int64 {
-	return env.GetInt64(CloudCostQueryWindowDaysEnvVar, 7)
+func GetCloudCostQueryWindowDays() int {
+	return env.GetInt(CloudCostQueryWindowDaysEnvVar, 7)
 }
 
-func GetCloudCostRunWindowDays() int64 {
-	return env.GetInt64(CloudCostRunWindowDaysEnvVar, 3)
+func GetCloudCostRunWindowDays() int {
+	return env.GetInt(CloudCostRunWindowDaysEnvVar, 3)
 }
 
 func GetCloudCost1dRetention() int {
 	return env.GetPrefixInt(CloudCostEnvVarPrefix, env.Resolution1dRetentionEnvVar, 30)
 }
 
-func GetCustomCostQueryWindowHours() int64 {
-	return env.GetInt64(CustomCostQueryWindowDaysEnvVar, 1)
+func GetCustomCostQueryWindowHours() int {
+	return env.GetInt(CustomCostQueryWindowDaysEnvVar, 1)
 }
 
-func GetCustomCostQueryWindowDays() int64 {
-	return env.GetInt64(CustomCostQueryWindowDaysEnvVar, 7)
+func GetCustomCostQueryWindowDays() int {
+	return env.GetInt(CustomCostQueryWindowDaysEnvVar, 7)
 }
 
 func GetCustomCost1dRetention() int {

protos/model/labels.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+
+package model;
+
+import "protos/model/window.proto";
+
+// Sets the golang package for the protobuf generated code
+option go_package = "github.com/opencost/opencost/core/pkg/model/pb";
+
+// LabelsResponse is a grouping of LabelSets for a specific window for a type of label grouped by their source.
+message LabelsResponse {
+  // The type of labels in the set (e.g. account_labels, namespace_labels)
+  string type = 1;
+  // type dependent identifier for the grouping of labels most likely the cluster id or the configuration key, used
+  // as part of the export path
+  string group_id = 2;
+
+  // The window for the label sets
+  model.Window window = 3;
+
+  // Mapping of LabelSets for individual items by a unique identifier
+  map<string, LabelSet> label_sets = 4;
+}
+
+// LabelSet is an internal message meant to enable nesting maps
+message LabelSet {
+  map<string, string> labels = 1;
+}

protos/model/window.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package model;
+
+import "google/protobuf/timestamp.proto";
+
+// Sets the golang package for the protobuf generated code
+option go_package = "github.com/opencost/opencost/core/pkg/model/pb";
+
+// Window defines a unit of time by a resolution and a start time
+message Window {
+  // resolution in "1h" or "1d" format
+  string resolution = 1;
+  // the start time of the window described
+  google.protobuf.Timestamp start = 2;
+}