Update retention variables, introduce prefixs (#3325)

Signed-off-by: Sean Holcomb <seanholcomb@gmail.com>

core/pkg/env/core.go

@@ -16,6 +16,10 @@ const (
 	PProfEnabledEnvVar = "PPROF_ENABLED"
 
 	InstallNamespaceEnvVar = "INSTALL_NAMESPACE"
+
+	Resolution1dRetentionEnvVar  = "RESOLUTION_1D_RETENTION"  // int: number of days
+	Resolution1hRetentionEnvVar  = "RESOLUTION_1H_RETENTION"  // int: number of hours
+	Resolution10mRetentionEnvVar = "RESOLUTION_10M_RETENTION" // int: number of 10m segments
 )
 
 // GetAPIPort returns the environment variable value for APIPortEnvVar which

core/pkg/env/env.go

@@ -198,3 +198,10 @@ func SetBool(key string, value bool) error {
 func SetDuration(key string, value time.Duration) error {
 	return envMapper.SetDuration(key, value)
 }
+
+// GetPrefixInt parses an int from the environment variable key parameter. It first checks the env var with the prefix
+// then checks the env var without the prefix If the environment variable is empty or fails to parse, the defaultValue
+// parameter is returned.
+func GetPrefixInt(prefix, key string, defaultValue int) int {
+	return envMapper.GetInt(prefix+key, envMapper.GetInt(key, defaultValue))
+}

modules/collector-source/pkg/collector/config.go

@@ -19,15 +19,15 @@ func NewOpenCostCollectorConfigFromEnv() CollectorConfig {
 		Resolutions: []util.ResolutionConfiguration{
 			{
 				Interval:  "10m",
-				Retention: env.GetCollector10mResolutionRetention(),
+				Retention: env.GetCollectorResolution10mRetention(),
 			},
 			{
 				Interval:  "1h",
-				Retention: env.GetCollector1hResolutionRetention(),
+				Retention: env.GetCollectorResolution1hRetention(),
 			},
 			{
 				Interval:  "1d",
-				Retention: env.GetCollection1dResolutionRetention(),
+				Retention: env.GetCollectionResolution1dRetention(),
 			},
 		},
 		ScrapeInterval:  env.GetCollectorScrapeIntervalSeconds(),

modules/collector-source/pkg/env/collectorenv.go

@@ -5,27 +5,25 @@ import (
 )
 
 const (
-	NetworkPortEnvVar               = "NETWORK_PORT"
-	Collector10mResolutionRetention = "COLLECTOR_10M_RESOLUTION_RETENTION"
-	Collector1hResolutionRetention  = "COLLECTOR_1H_RESOLUTION_RETENTION"
-	Collection1dResolutionRetention = "COLLECTOR_1D_RESOLUTION_RETENTION"
-	CollectorScrapeInterval         = "COLLECTOR_SCRAPE_INTERVAL"
+	CollectorEnvVarPrefix   = "COLLECTOR_"
+	CollectorScrapeInterval = "COLLECTOR_SCRAPE_INTERVAL"
+	NetworkPortEnvVar       = "NETWORK_PORT"
 )
 
 func GetNetworkPort() int {
 	return env.GetInt(NetworkPortEnvVar, 3001)
 }
 
-func GetCollector10mResolutionRetention() int {
-	return env.GetInt(Collector10mResolutionRetention, 36)
+func GetCollectorResolution10mRetention() int {
+	return env.GetPrefixInt(CollectorEnvVarPrefix, env.Resolution10mRetentionEnvVar, 36)
 }
 
-func GetCollector1hResolutionRetention() int {
-	return env.GetInt(Collector1hResolutionRetention, 49)
+func GetCollectorResolution1hRetention() int {
+	return env.GetPrefixInt(CollectorEnvVarPrefix, env.Resolution1hRetentionEnvVar, 49)
 }
 
-func GetCollection1dResolutionRetention() int {
-	return env.GetInt(Collection1dResolutionRetention, 15)
+func GetCollectionResolution1dRetention() int {
+	return env.GetPrefixInt(CollectorEnvVarPrefix, env.Resolution1dRetentionEnvVar, 15)
 }
 
 func GetCollectorScrapeIntervalSeconds() string {

pkg/cloudcost/ingestor.go

@@ -39,7 +39,7 @@ type IngestorConfig struct {
 func DefaultIngestorConfiguration() IngestorConfig {
 	return IngestorConfig{
 		Resolution:             timeutil.Day,
-		Duration:               timeutil.Day * time.Duration(env.GetDataRetentionDailyResolutionDays()),
+		Duration:               timeutil.Day * time.Duration(env.GetCloudCost1dRetention()),
 		MonthToDateRunInterval: env.GetCloudCostMonthToDateInterval(),
 		RefreshRate:            time.Hour * time.Duration(env.GetCloudCostRefreshRateHours()),
 		QueryWindow:            timeutil.Day * time.Duration(env.GetCloudCostQueryWindowDays()),

pkg/customcost/ingestor.go

@@ -42,8 +42,8 @@ type CustomCostIngestorConfig struct {
 // DefaultIngestorConfiguration retrieves an CustomCostIngestorConfig from env variables
 func DefaultIngestorConfiguration() CustomCostIngestorConfig {
 	return CustomCostIngestorConfig{
-		DailyDuration:       timeutil.Day * time.Duration(env.GetDataRetentionDailyResolutionDays()),
-		HourlyDuration:      time.Hour * time.Duration(env.GetDataRetentionHourlyResolutionHours()),
+		DailyDuration:       timeutil.Day * time.Duration(env.GetCustomCost1dRetention()),
+		HourlyDuration:      time.Hour * time.Duration(env.GetCustomCost1hRetention()),
 		DailyQueryWindow:    timeutil.Day * time.Duration(env.GetCustomCostQueryWindowDays()),
 		HourlyQueryWindow:   time.Hour * time.Duration(env.GetCustomCostQueryWindowHours()),
 		PluginConfigDir:     env.GetPluginConfigDir(),

pkg/customcost/querier.go

@@ -41,7 +41,7 @@ func getCustomCostAccumulateOption(window opencost.Window, from []opencost.Accum
 		from = allSteppedAccumulateOptions
 	}
 
-	hourlyStoreHours := env.GetDataRetentionHourlyResolutionHours()
+	hourlyStoreHours := env.GetCustomCost1hRetention()
 	hourlySteps := time.Duration(hourlyStoreHours) * time.Hour
 	oldestHourly := time.Now().Add(-1 * hourlySteps)
 
@@ -53,7 +53,7 @@ func getCustomCostAccumulateOption(window opencost.Window, from []opencost.Accum
 		return opencost.AccumulateOptionHour, nil
 	}
 
-	dailyStoreDays := env.GetDataRetentionDailyResolutionDays()
+	dailyStoreDays := env.GetCloudCost1dRetention()
 	dailySteps := time.Duration(dailyStoreDays) * timeutil.Day
 	oldestDaily := time.Now().Add(-1 * dailySteps)
 	// Use daily if...

pkg/env/cloudcost.go

@@ -11,12 +11,14 @@ const (
 )
 
 const (
+	CloudCostEnvVarPrefix           = "CLOUD_COST_"
 	CloudCostEnabledEnvVar          = "CLOUD_COST_ENABLED"
 	CloudCostMonthToDateIntervalVar = "CLOUD_COST_MONTH_TO_DATE_INTERVAL"
 	CloudCostRefreshRateHoursEnvVar = "CLOUD_COST_REFRESH_RATE_HOURS"
 	CloudCostQueryWindowDaysEnvVar  = "CLOUD_COST_QUERY_WINDOW_DAYS"
 	CloudCostRunWindowDaysEnvVar    = "CLOUD_COST_RUN_WINDOW_DAYS"
 
+	CustomCostEnvVarPrefix          = "CUSTOM_COST_"
 	CustomCostEnabledEnvVar         = "CUSTOM_COST_ENABLED"
 	CustomCostQueryWindowDaysEnvVar = "CUSTOM_COST_QUERY_WINDOW_DAYS"
 
@@ -50,6 +52,14 @@ func GetCloudCostQueryWindowDays() int64 {
 	return env.GetInt64(CloudCostQueryWindowDaysEnvVar, 7)
 }
 
+func GetCloudCostRunWindowDays() int64 {
+	return env.GetInt64(CloudCostRunWindowDaysEnvVar, 3)
+}
+
+func GetCloudCost1dRetention() int {
+	return env.GetPrefixInt(CloudCostEnvVarPrefix, env.Resolution1dRetentionEnvVar, 30)
+}
+
 func GetCustomCostQueryWindowHours() int64 {
 	return env.GetInt64(CustomCostQueryWindowDaysEnvVar, 1)
 }
@@ -58,8 +68,12 @@ func GetCustomCostQueryWindowDays() int64 {
 	return env.GetInt64(CustomCostQueryWindowDaysEnvVar, 7)
 }
 
-func GetCloudCostRunWindowDays() int64 {
-	return env.GetInt64(CloudCostRunWindowDaysEnvVar, 3)
+func GetCustomCost1dRetention() int {
+	return env.GetPrefixInt(CustomCostEnvVarPrefix, env.Resolution1dRetentionEnvVar, 30)
+}
+
+func GetCustomCost1hRetention() int {
+	return env.GetPrefixInt(CustomCostEnvVarPrefix, env.Resolution1hRetentionEnvVar, 49)
 }
 
 func GetPluginConfigDir() string {

pkg/env/costmodel.go

@@ -79,9 +79,6 @@ const (
 	ExportCSVLabelsAll  = "EXPORT_CSV_LABELS_ALL"
 	ExportCSVMaxDays    = "EXPORT_CSV_MAX_DAYS"
 
-	DataRetentionDailyResolutionDaysEnvVar   = "DATA_RETENTION_DAILY_RESOLUTION_DAYS"
-	DataRetentionHourlyResolutionHoursEnvVar = "DATA_RETENTION_HOURLY_RESOLUTION_HOURS"
-
 	CarbonEstimatesEnabledEnvVar = "CARBON_ESTIMATES_ENABLED"
 
 	KubernetesResourceAccessEnvVar = "KUBERNETES_RESOURCE_ACCESS"
@@ -332,14 +329,6 @@ func GetRegionOverrideList() []string {
 	return regionList
 }
 
-func GetDataRetentionDailyResolutionDays() int64 {
-	return env.GetInt64(DataRetentionDailyResolutionDaysEnvVar, 30)
-}
-
-func GetDataRetentionHourlyResolutionHours() int64 {
-	return env.GetInt64(DataRetentionHourlyResolutionHoursEnvVar, 49)
-}
-
 func IsKubernetesEnabled() bool {
 	return env.Get(KubernetesEnabledEnvVar, "") != ""
 }