tweak test

Signed-off-by: Alex Meijer <ameijer@kubecost.com>

pkg/customcost/ingestor.go

@@ -21,11 +21,12 @@ import (
 
 // IngestorStatus includes diagnostic values for a given Ingestor
 type IngestorStatus struct {
-	Created  time.Time
-	LastRun  time.Time
-	NextRun  time.Time
-	Runs     int
-	Coverage map[string]opencost.Window
+	Created     time.Time
+	LastRun     time.Time
+	NextRun     time.Time
+	Runs        int
+	Coverage    map[string]opencost.Window
+	RefreshRate time.Duration
 }
 
 // CustomCost IngestorConfig is a configuration struct for an Ingestor
@@ -223,9 +224,9 @@ func (ing *CustomCostIngestor) Start(rebuild bool) {
 
 	// Build the store once, advancing backward in time from the earliest
 	// point of coverage.
-	//go ing.build(rebuild)
+	go ing.build(rebuild)
 
-	ing.run()
+	go ing.run()
 
 }
 
@@ -269,11 +270,12 @@ func (ing *CustomCostIngestor) Stop() {
 // Status returns an IngestorStatus that describes the current state of the ingestor
 func (ing *CustomCostIngestor) Status() IngestorStatus {
 	return IngestorStatus{
-		Created:  ing.creationTime,
-		LastRun:  ing.lastRun,
-		NextRun:  ing.lastRun.Add(ing.refreshRate).UTC(),
-		Runs:     ing.runs,
-		Coverage: ing.coverage,
+		Created:     ing.creationTime,
+		LastRun:     ing.lastRun,
+		NextRun:     ing.lastRun.Add(ing.refreshRate).UTC(),
+		Runs:        ing.runs,
+		Coverage:    ing.coverage,
+		RefreshRate: ing.refreshRate,
 	}
 }
 

pkg/customcost/pipelineservice.go

@@ -17,7 +17,6 @@ import (
 	proto "github.com/opencost/opencost/core/pkg/protocol"
 	"github.com/opencost/opencost/core/pkg/util/timeutil"
 	"github.com/opencost/opencost/core/pkg/version"
-	"github.com/opencost/opencost/pkg/env"
 )
 
 var protocol = proto.HTTP()
@@ -121,7 +120,7 @@ func NewPipelineService(hourlyrepo, dailyrepo Repository, ingConf CustomCostInge
 		return nil, err
 	}
 
-	//dailyIngestor.Start(false)
+	dailyIngestor.Start(false)
 	return &PipelineService{
 		hourlyIngestor: hourlyIngestor,
 		hourlyStore:    hourlyrepo,
@@ -134,18 +133,17 @@ func NewPipelineService(hourlyrepo, dailyrepo Repository, ingConf CustomCostInge
 func (dp *PipelineService) Status() Status {
 
 	// Pull config status from the config controller
-	ingstatus := dp.hourlyIngestor.Status()
-	dur, err := time.ParseDuration(env.GetCustomCostRefreshRateHours())
-	if err != nil {
-		log.Errorf("error parsing duration %s: %v", env.GetCustomCostRefreshRateHours(), err)
-		return Status{}
-	}
-	refreshRate := time.Hour * dur
+	ingstatusHourly := dp.hourlyIngestor.Status()
+
+	// Pull config status from the config controller
+	ingstatusDaily := dp.dailyIngestor.Status()
 
 	// These are the statuses
 	return Status{
-		Coverage:    ingstatus.Coverage,
-		RefreshRate: refreshRate.String(),
+		CoverageDaily:     ingstatusDaily.Coverage,
+		CoverageHourly:    ingstatusHourly.Coverage,
+		RefreshRateHourly: ingstatusHourly.RefreshRate.String(),
+		RefreshRateDaily:  ingstatusDaily.RefreshRate.String(),
 	}
 
 }

pkg/customcost/pipelineservice_test.go

@@ -56,7 +56,7 @@ func TestPipelineService(t *testing.T) {
 	config := DefaultIngestorConfiguration()
 
 	config.DailyDuration = 7 * timeutil.Day
-	config.HourlyDuration = 12 * time.Hour
+	config.HourlyDuration = 16 * time.Hour
 	pipeline, err := NewPipelineService(hourlyRepo, dailyRepo, config)
 	if err != nil {
 		t.Fatalf("error starting pipeline: %v", err)
@@ -69,17 +69,18 @@ func TestPipelineService(t *testing.T) {
 	for !ingestionComplete && loopCount < maxLoops {
 		status := pipeline.Status()
 		log.Debugf("got status: %v", status)
-		coverage, found := status.Coverage["datadog"]
-		if found {
+		coverageHourly, foundHourly := status.CoverageHourly["datadog"]
+		coverageDaily, foundDaily := status.CoverageDaily["datadog"]
+		if foundHourly && foundDaily {
 			// check coverage
 			minTime := time.Now().UTC().Add(-6 * timeutil.Day)
 			maxTime := time.Now().UTC().Add(-3 * time.Hour)
-			if coverage.Start().Before(minTime) && coverage.End().After(maxTime) {
+			if coverageDaily.Start().Before(minTime) && coverageHourly.End().After(maxTime) {
 				log.Infof("good coverage, breaking out of loop")
 				ingestionComplete = true
 				break
 			} else {
-				log.Infof("coverage not within range. Looking for coverage %v to %v, but current coverage is %v", minTime, maxTime, coverage)
+				log.Infof("coverage not within range. Looking for coverage %v to %v, but current coverage is %v/%v", minTime, maxTime, coverageDaily, coverageHourly)
 			}
 		} else {
 			log.Debugf("no coverage info ready yet for datadog")
@@ -97,7 +98,7 @@ func TestPipelineService(t *testing.T) {
 	pipeline.dailyIngestor.Stop()
 
 	// inspect data from yesterday
-	targetTime := time.Now().Add(-1 * timeutil.Day).Truncate(timeutil.Day)
+	targetTime := time.Now().UTC().Add(-1 * timeutil.Day).Truncate(timeutil.Day)
 	log.Infof("querying for data with window start: %v", targetTime)
 	// check for presence of hosts in DD response
 	ddCosts, err := dailyRepo.Get(targetTime, "datadog")
@@ -112,12 +113,31 @@ func TestPipelineService(t *testing.T) {
 	}
 
 	if !foundInfraHosts {
-		t.Fatal("expecting infra_hosts costs in response")
+		t.Fatal("expecting infra_hosts costs in daily response")
+	}
+
+	// query data from 4 hours ago (hourly)
+	targetTime = time.Now().UTC().Add(-12 * time.Hour).Truncate(time.Hour)
+	log.Infof("querying for data with window start: %v", targetTime)
+	// check for presence of hosts in DD response
+	ddCosts, err = hourlyRepo.Get(targetTime, "datadog")
+	if err != nil {
+		t.Fatalf("error getting results for targetTime")
+	}
+	foundInfraHosts = false
+	for _, cost := range ddCosts.Costs {
+		if cost.ResourceType == "infra_hosts" {
+			foundInfraHosts = true
+		}
+	}
+
+	if !foundInfraHosts {
+		t.Fatal("expecting infra_hosts costs in hourly response")
 	}
 }
 
 func downloadLatestPluginExec(dirName string, t *testing.T) {
-	ddPluginURL := "https://github.com/opencost/opencost-plugins/releases/download/v0.0.2/datadog.ocplugin." + runtime.GOOS + "." + version.Architecture
+	ddPluginURL := "https://github.com/opencost/opencost-plugins/releases/download/v0.0.3/datadog.ocplugin." + runtime.GOOS + "." + version.Architecture
 	out, err := os.OpenFile(dirName+"/datadog.ocplugin."+runtime.GOOS+"."+version.Architecture, 0755|os.O_CREATE, 0755)
 	if err != nil {
 		t.Fatalf("error creating executable file: %v", err)

pkg/customcost/status.go

@@ -9,17 +9,19 @@ import (
 
 // Status gives the details and metadata of a CloudCost integration
 type Status struct {
-	Key              string                     `json:"key"`
-	Source           string                     `json:"source"`
-	Provider         string                     `json:"provider"`
-	Active           bool                       `json:"active"`
-	Valid            bool                       `json:"valid"`
-	LastRun          time.Time                  `json:"lastRun"`
-	NextRun          time.Time                  `json:"nextRun"`
-	RefreshRate      string                     `json:"RefreshRate"`
-	Created          time.Time                  `json:"created"`
-	Runs             int                        `json:"runs"`
-	Coverage         map[string]opencost.Window `json:"coverage"`
-	ConnectionStatus string                     `json:"connectionStatus"`
-	Config           cloudconfig.Config         `json:"config"`
+	Key               string                     `json:"key"`
+	Source            string                     `json:"source"`
+	Provider          string                     `json:"provider"`
+	Active            bool                       `json:"active"`
+	Valid             bool                       `json:"valid"`
+	LastRun           time.Time                  `json:"lastRun"`
+	NextRun           time.Time                  `json:"nextRun"`
+	RefreshRateDaily  string                     `json:"RefreshRateDaily"`
+	RefreshRateHourly string                     `json:"RefreshRateHourly"`
+	Created           time.Time                  `json:"created"`
+	Runs              int                        `json:"runs"`
+	CoverageHourly    map[string]opencost.Window `json:"coverageHourly"`
+	CoverageDaily     map[string]opencost.Window `json:"coverageDaily"`
+	ConnectionStatus  string                     `json:"connectionStatus"`
+	Config            cloudconfig.Config         `json:"config"`
 }