Move files to use config path

CONTRIBUTING.md

@@ -33,13 +33,40 @@ To confirm that the server is running, you can hit [http://localhost:9003/costDa
 
 ## Running locally ##
 
-In order to run cost-model locally, or outside of the runtime of a Kubernetes cluster, you can set the environment variable `KUBECONFIG_PATH`.
+To run locally cd into `cmd/costmodel` and `go run main.go` 
+
+cost-model requires a connection to Prometheus in order to operate so setting the environment variable `PROMETHEUS_SERVER_ENDPOINT` is required. 
+In order to expose Prometheus to cost-model it may be required to port-forward using kubectl to your Prometheus endpoint.
+
+For example:
+```bash
+kubectl port-forward svc/kubecost-prometheus-server 9080:80
+```
+This would expose Prometheus on port 9080 and allow setting the environment variable as so:
+```bash
+PROMETHEUS_SERVER_ENDPOINT="http://127.0.0.1:9080"
+```
+
+If you want to run with a specific kubeconfig the environment variable `KUBECONFIG_PATH` can be used. cost-model will attempt to connect to your Kubernetes cluster in a similar fashion as kubectl so the env is not required. The order of precedence is `KUBECONFIG_PATH` > default kubeconfig file location ($HOME/.kube/config) > in cluster config
 
 Example:
 ```bash
 export KUBECONFIG_PATH=~/.kube/config
 ```
 
+There are two more environement variabes recommended to run locally. These should be set as the default file location used is `/var/` which usually requires more permissions than kubecost actually needs to run. They do not need to match but keeping everything together can help cleanup when no longer needed.
+
+```bash
+ETL_PATH_PREFIX="/my/cool/path/kubecost/var/config" 
+CONFIG_PATH="/my/cool/path/kubecost/var/config"
+```
+
+An example of the full command:
+```bash
+ETL_PATH_PREFIX="/my/cool/path/kubecost/var/config" CONFIG_PATH="/my/cool/path/kubecost/var/config" PROMETHEUS_SERVER_ENDPOINT="http://127.0.0.1:9090" go run main.go
+```
+
+
 ## Running the integration tests ##
 To run these tests:
 * Make sure you have a kubeconfig that can point to your cluster, and have permissions to create/modify a namespace called "test"

pkg/cloud/awsprovider.go

@@ -97,8 +97,6 @@ func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 // How often spot data is refreshed
 const SpotRefreshDuration = 15 * time.Minute
 
-const defaultConfigPath = "/var/configs/"
-
 var awsRegions = []string{
 	"us-east-2",
 	"us-east-1",

pkg/cmd/agent/agent.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"path"
 	"time"
 
 	"github.com/kubecost/cost-model/pkg/cloud"
@@ -179,9 +180,11 @@ func Execute(opts *AgentOpts) error {
 
 	clusterCache.SetConfigMapUpdateFunc(watchConfigFunc)
 
+	configPrefix := env.GetConfigPathWithDefault("/var/configs/")
+
 	// Initialize cluster exporting if it's enabled
 	if env.IsExportClusterCacheEnabled() {
-		cacheLocation := confManager.ConfigFileAt("/var/configs/cluster-cache.json")
+		cacheLocation := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-cache.json"))
 		clusterExporter = clustercache.NewClusterExporter(clusterCache, cacheLocation, ClusterExportInterval)
 		clusterExporter.Run()
 	}
@@ -191,7 +194,7 @@ func Execute(opts *AgentOpts) error {
 
 	var clusterInfoProvider clusters.ClusterInfoProvider
 	if env.IsExportClusterInfoEnabled() {
-		clusterInfoConf := confManager.ConfigFileAt("/var/configs/cluster-info.json")
+		clusterInfoConf := confManager.ConfigFileAt(path.Join(configPrefix, " cluster-info.json"))
 		clusterInfoProvider = costmodel.NewClusterInfoWriteOnRequest(localClusterInfo, clusterInfoConf)
 	} else {
 		clusterInfoProvider = localClusterInfo

pkg/costmodel/router.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"reflect"
 	"regexp"
 	"strconv"
@@ -1275,7 +1276,7 @@ func (a *Accesses) AddServiceKey(w http.ResponseWriter, r *http.Request, ps http
 
 	key := r.PostForm.Get("key")
 	k := []byte(key)
-	err := ioutil.WriteFile("/var/configs/key.json", k, 0644)
+	err := ioutil.WriteFile(path.Join(env.GetConfigPathWithDefault("/var/configs/"), "key.json"), k, 0644)
 	if err != nil {
 		fmt.Fprintf(w, "Error writing service key: "+err.Error())
 	}
@@ -1444,10 +1445,12 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 		LocalConfigPath:   "/",
 	})
 
+	configPrefix := env.GetConfigPathWithDefault("/var/configs/")
+
 	// Create Kubernetes Cluster Cache + Watchers
 	var k8sCache clustercache.ClusterCache
 	if env.IsClusterCacheFileEnabled() {
-		importLocation := confManager.ConfigFileAt("/var/configs/cluster-cache.json")
+		importLocation := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-cache.json"))
 		k8sCache = clustercache.NewClusterImporter(importLocation)
 	} else {
 		k8sCache = clustercache.NewKubernetesClusterCache(kubeClientset)
@@ -1533,7 +1536,7 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 	// ClusterInfo Provider to provide the cluster map with local and remote cluster data
 	var clusterInfoProvider clusters.ClusterInfoProvider
 	if env.IsClusterInfoFileEnabled() {
-		clusterInfoFile := confManager.ConfigFileAt("/var/configs/cluster-info.json")
+		clusterInfoFile := confManager.ConfigFileAt(path.Join(configPrefix, " cluster-info.json"))
 		clusterInfoProvider = NewConfiguredClusterInfoProvider(clusterInfoFile)
 	} else {
 		clusterInfoProvider = NewLocalClusterInfoProvider(kubeClientset, cloudProvider)

pkg/metrics/metricsconfig.go

@@ -5,13 +5,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"sync"
 
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/util/watcher"
 )
 
-var metricsConfigLock = new(sync.Mutex)
+var (
+	metricsConfigLock = new(sync.Mutex)
+	metricsFilePath   = path.Join(env.GetConfigPathWithDefault("/var/configs/"), "metrics.json")
+)
 
 type MetricsConfig struct {
 	DisabledMetrics []string `json:"disabledMetrics"`
@@ -33,7 +37,7 @@ func GetMetricsConfig() (*MetricsConfig, error) {
 	metricsConfigLock.Lock()
 	defer metricsConfigLock.Unlock()
 	mc := &MetricsConfig{}
-	body, err := ioutil.ReadFile("/var/configs/metrics.json")
+	body, err := ioutil.ReadFile(metricsFilePath)
 	if os.IsNotExist(err) {
 
 		return mc, nil
@@ -59,7 +63,7 @@ func UpdateMetricsConfig(mc *MetricsConfig) (*MetricsConfig, error) {
 		return nil, fmt.Errorf("error encoding metrics config struct: %s", err)
 	}
 
-	err = ioutil.WriteFile("/var/configs/metrics.json", mcb, 0644)
+	err = ioutil.WriteFile(metricsFilePath, mcb, 0644)
 	if err != nil {
 		return nil, fmt.Errorf("error writing to metrics config file: %s", err)
 	}

pkg/services/clusterservice.go

@@ -1,6 +1,11 @@
 package services
 
-import "github.com/kubecost/cost-model/pkg/services/clusters"
+import (
+	"path"
+
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/services/clusters"
+)
 
 // NewClusterManagerService creates a new HTTPService implementation driving cluster definition management
 // for the frontend
@@ -10,28 +15,8 @@ func NewClusterManagerService() HTTPService {
 
 // newClusterManager creates a new cluster manager instance for use in the service
 func newClusterManager() *clusters.ClusterManager {
-	clustersConfigFile := "/var/configs/clusters/default-clusters.yaml"
+	clustersConfigFile := path.Join(env.GetConfigPathWithDefault("/var/configs/"), "clusters/default-clusters.yaml")
 
 	// Return a memory-backed cluster manager populated by configmap
 	return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
-
-	// NOTE: The following should be used with a persistent disk store. Since the
-	// NOTE: configmap approach is currently the "persistent" source (entries are read-only
-	// NOTE: on the backend), we don't currently need to store on disk.
-	/*
-		path := env.GetConfigPath()
-		db, err := bolt.Open(path+"costmodel.db", 0600, nil)
-		if err != nil {
-			log.Errorf("[Error] Failed to create costmodel.db: %s", err.Error())
-			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
-		}
-
-		store, err := clusters.NewBoltDBClusterStorage("clusters", db)
-		if err != nil {
-			log.Errorf("[Error] Failed to Create Cluster Storage: %s", err.Error())
-			return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
-		}
-
-		return clusters.NewConfiguredClusterManager(store, clustersConfigFile)
-	*/
 }