Merge pull request #2625 from ameijer/atm/working-branch-fixes

.github/workflows/build-test.yaml

@@ -10,6 +10,36 @@ on:
       - develop
 
 jobs:
+  validate-protobuf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: ./
+      -
+        name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+          
+      -
+        name: Install protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: '25.3'
+      -
+        name: Install just
+        uses: extractions/setup-just@v1
+
+      - name: install protobuf-go
+        run: |
+          go install github.com/golang/protobuf/protoc-gen-go@latest
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+          which protoc-gen-go-grpc
+      -
+        name: Validate
+        run: |
+          just validate-protobuf
   backend:
     runs-on: ubuntu-latest
     steps:

core/pkg/model/pb/messages.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.30.0
-// 	protoc        v3.21.12
+// 	protoc-gen-go v1.33.0
+// 	protoc        v4.25.3
 // source: protos/customcost/messages.proto
 
 package pb

core/pkg/model/pb/messages_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.3.0
-// - protoc             v3.21.12
+// - protoc             v4.25.3
 // source: protos/customcost/messages.proto
 
 package pb

justfile

@@ -26,7 +26,6 @@ build-binary VERSION=version:
         {{commonenv}} GOOS=linux GOARCH=amd64 go build \
         -ldflags \
           "-X github.com/opencost/opencost/pkg/version.Version={{VERSION}} \
-           -X github.com/opencost/opencost/pkg/version.Architecture=amd64 \
            -X github.com/opencost/opencost/pkg/version.GitCommit={{commit}}" \
         -o ./costmodel-amd64
 
@@ -34,7 +33,6 @@ build-binary VERSION=version:
         {{commonenv}} GOOS=linux GOARCH=arm64 go build \
         -ldflags \
           "-X github.com/opencost/opencost/pkg/version.Version={{VERSION}} \
-           -X github.com/opencost/opencost/pkg/version.Architecture=arm64 \
            -X github.com/opencost/opencost/pkg/version.GitCommit={{commit}}" \
         -o ./costmodel-arm64
 
@@ -68,3 +66,7 @@ build IMAGE_TAG RELEASE_VERSION: test (build-binary RELEASE_VERSION)
         --platforms "linux/amd64,linux/arm64" \
         --template {{IMAGE_TAG}}-ARCH \
         --target {{IMAGE_TAG}}
+
+validate-protobuf:
+    ./generate.sh
+    git diff --exit-code

pkg/cloud/provider/csvprovider.go

@@ -320,11 +320,11 @@ func NodeValueFromMapField(m string, n *v1.Node, useRegion bool) string {
 			akey := strings.Join(mf[2:len(mf)], ".")
 			return toReturn + n.Annotations[akey]
 		} else {
-			log.Errorf("Unsupported InstanceIDField %s in CSV For Node", m)
+			log.DedupedInfof(10, "Unsupported InstanceIDField %s in CSV For Node", m)
 			return ""
 		}
 	} else {
-		log.Errorf("Unsupported InstanceIDField %s in CSV For Node", m)
+		log.DedupedInfof(10, "Unsupported InstanceIDField %s in CSV For Node", m)
 		return ""
 	}
 }

pkg/costmodel/router.go

@@ -1846,8 +1846,12 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 	a.Router.GET("/cloud/config/disable", a.CloudConfigController.GetDisableConfigHandler())
 	a.Router.GET("/cloud/config/delete", a.CloudConfigController.GetDeleteConfigHandler())
 
-	a.Router.GET("/customCost/total", a.CustomCostQueryService.GetCustomCostTotalHandler())
-	a.Router.GET("/customCost/timeseries", a.CustomCostQueryService.GetCustomCostTimeseriesHandler())
+	if env.IsCustomCostEnabled() {
+		a.Router.GET("/customCost/total", a.CustomCostQueryService.GetCustomCostTotalHandler())
+		a.Router.GET("/customCost/timeseries", a.CustomCostQueryService.GetCustomCostTimeseriesHandler())
+	}
+
+	a.Router.GET("/customCost/status", a.CustomCostPipelineService.GetCustomCostStatusHandler())
 
 	a.httpServices.RegisterAll(a.Router)
 

pkg/customcost/ingestor.go

@@ -42,13 +42,12 @@ type CustomCostIngestorConfig struct {
 // DefaultIngestorConfiguration retrieves an CustomCostIngestorConfig from env variables
 func DefaultIngestorConfiguration() CustomCostIngestorConfig {
 	return CustomCostIngestorConfig{
-		DailyDuration:          timeutil.Day * time.Duration(env.GetDataRetentionDailyResolutionDays()),
-		HourlyDuration:         time.Hour * time.Duration(env.GetDataRetentionHourlyResolutionHours()),
-		MonthToDateRunInterval: env.GetCloudCostMonthToDateInterval(),
-		DailyQueryWindow:       timeutil.Day * time.Duration(env.GetCustomCostQueryWindowDays()),
-		HourlyQueryWindow:      time.Hour * time.Duration(env.GetCustomCostQueryWindowHours()),
-		PluginConfigDir:        env.GetPluginConfigDir(),
-		PluginExecutableDir:    env.GetPluginExecutableDir(),
+		DailyDuration:       timeutil.Day * time.Duration(env.GetDataRetentionDailyResolutionDays()),
+		HourlyDuration:      time.Hour * time.Duration(env.GetDataRetentionHourlyResolutionHours()),
+		DailyQueryWindow:    timeutil.Day * time.Duration(env.GetCustomCostQueryWindowDays()),
+		HourlyQueryWindow:   time.Hour * time.Duration(env.GetCustomCostQueryWindowHours()),
+		PluginConfigDir:     env.GetPluginConfigDir(),
+		PluginExecutableDir: env.GetPluginExecutableDir(),
 	}
 }
 
@@ -179,7 +178,6 @@ func (ing *CustomCostIngestor) buildSingleDomain(start, end time.Time, domain st
 		return
 	}
 
-	// TODO HAVE PLUG IN RETURN DATA AS PROTOBUF
 	// Request the plugin
 	raw, err := rpcClient.Dispense("CustomCostSource")
 	if err != nil {
@@ -236,7 +234,7 @@ func (ing *CustomCostIngestor) Start(rebuild bool) {
 func (ing *CustomCostIngestor) Stop() {
 	// If already stopping, log that and return.
 	if !ing.isStopping.CompareAndSwap(false, true) {
-		log.Infof("CloudCost: ingestor: is already stopping")
+		log.Infof("CustomCost: ingestor: is already stopping")
 		return
 	}
 

pkg/customcost/memoryrepository.go

@@ -42,12 +42,12 @@ func (m *MemoryRepository) Get(startTime time.Time, domain string) (*pb.CustomCo
 
 	domainData, ok := m.data[domain]
 	if !ok {
-		return nil, nil
+		return &pb.CustomCostResponse{}, nil
 	}
 
 	b, ook := domainData[startTime.UTC()]
 	if !ook {
-		return nil, nil
+		return &pb.CustomCostResponse{}, nil
 	}
 
 	ccr := &pb.CustomCostResponse{}

pkg/customcost/pipelineservice.go

@@ -22,7 +22,7 @@ var protocol = proto.HTTP()
 
 const execFmt = `%s/%s.ocplugin.%s.%s`
 
-// PipelineService exposes CloudCost pipeline controls and diagnostics endpoints
+// PipelineService exposes CustomCost pipeline controls and diagnostics endpoints
 type PipelineService struct {
 	hourlyIngestor, dailyIngestor *CustomCostIngestor
 	hourlyStore, dailyStore       Repository
@@ -197,10 +197,13 @@ func (s *PipelineService) GetCustomCostRebuildHandler() func(w http.ResponseWrit
 
 // GetCustomCostStatusHandler creates a handler from a http request which returns the custom cost ingestor status
 func (s *PipelineService) GetCustomCostStatusHandler() func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	// If Reporting Service is nil, always return 501
+
 	if s == nil {
+		resultStatus := Status{
+			Enabled: false,
+		}
 		return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-			http.Error(w, "Custom cost pipeline Service is nil", http.StatusNotImplemented)
+			protocol.WriteData(w, resultStatus)
 		}
 	}
 	if s.hourlyIngestor == nil || s.dailyIngestor == nil {
@@ -212,7 +215,8 @@ func (s *PipelineService) GetCustomCostStatusHandler() func(w http.ResponseWrite
 	// Return valid handler func
 	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 		w.Header().Set("Content-Type", "application/json")
-
-		protocol.WriteData(w, s.Status())
+		stat := s.Status()
+		stat.Enabled = true
+		protocol.WriteData(w, stat)
 	}
 }

pkg/customcost/repository.go

@@ -6,7 +6,7 @@ import (
 	"github.com/opencost/opencost/core/pkg/model/pb"
 )
 
-// Repository is an interface for storing and retrieving CloudCost data
+// Repository is an interface for storing and retrieving CustomCost data
 type Repository interface {
 	Has(time.Time, string) (bool, error)
 	Get(time.Time, string) (*pb.CustomCostResponse, error)

pkg/customcost/repositoryquerier.go

@@ -48,7 +48,7 @@ func (rq *RepositoryQuerier) QueryTotal(ctx context.Context, request CostTotalRe
 			ccResponse, err := repo.Get(queryStart, domain)
 			if err != nil {
 				return nil, fmt.Errorf("QueryTotal: %w", err)
-			} else if ccResponse == nil {
+			} else if ccResponse == nil || ccResponse.Start == nil || ccResponse.End == nil {
 				continue
 			}
 

pkg/customcost/status.go

@@ -4,24 +4,23 @@ import (
 	"time"
 
 	"github.com/opencost/opencost/core/pkg/opencost"
-	cloudconfig "github.com/opencost/opencost/pkg/cloud"
 )
 
-// Status gives the details and metadata of a CloudCost integration
+// Status gives the details and metadata of a CustomCost integration
 type Status struct {
-	Key               string                     `json:"key"`
-	Source            string                     `json:"source"`
-	Provider          string                     `json:"provider"`
-	Active            bool                       `json:"active"`
-	Valid             bool                       `json:"valid"`
-	LastRun           time.Time                  `json:"lastRun"`
-	NextRun           time.Time                  `json:"nextRun"`
-	RefreshRateDaily  string                     `json:"RefreshRateDaily"`
-	RefreshRateHourly string                     `json:"RefreshRateHourly"`
-	Created           time.Time                  `json:"created"`
-	Runs              int                        `json:"runs"`
-	CoverageHourly    map[string]opencost.Window `json:"coverageHourly"`
-	CoverageDaily     map[string]opencost.Window `json:"coverageDaily"`
-	ConnectionStatus  string                     `json:"connectionStatus"`
-	Config            cloudconfig.Config         `json:"config"`
+	Enabled           bool                       `json:"enabled"`
+	Key               string                     `json:"key,omitempty""`
+	Source            string                     `json:"source,omitempty""`
+	Provider          string                     `json:"provider,omitempty""`
+	Active            bool                       `json:"active,omitempty""`
+	Valid             bool                       `json:"valid,omitempty""`
+	LastRun           time.Time                  `json:"lastRun,omitempty""`
+	NextRun           time.Time                  `json:"nextRun,omitempty""`
+	RefreshRateDaily  string                     `json:"RefreshRateDaily,omitempty""`
+	RefreshRateHourly string                     `json:"RefreshRateHourly,omitempty""`
+	Created           time.Time                  `json:"created,omitempty""`
+	Runs              int                        `json:"runs,omitempty""`
+	CoverageHourly    map[string]opencost.Window `json:"coverageHourly,omitempty""`
+	CoverageDaily     map[string]opencost.Window `json:"coverageDaily,omitempty""`
+	ConnectionStatus  string                     `json:"connectionStatus,omitempty""`
 }