Merge branch 'develop' into parcs-accum-support

go.mod

@@ -8,7 +8,7 @@ require (
 	cloud.google.com/go/storage v1.28.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-sdk-for-go v65.0.0+incompatible
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.1-0.20230323231529-14c481f239ec
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
 	github.com/Azure/azure-storage-blob-go v0.15.0
@@ -66,7 +66,7 @@ require (
 	cloud.google.com/go v0.110.0 // indirect
 	cloud.google.com/go/compute v1.18.0 // indirect
 	cloud.google.com/go/iam v0.12.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect

go.sum

@@ -58,10 +58,14 @@ github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9Eb
 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.1-0.20230323231529-14c481f239ec h1:S83Dzhd3VLyvN2bgFI7/Lgk1etamk3Pk8QQhn3iXt4s=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.1-0.20230323231529-14c481f239ec/go.mod h1:IoxiGSzhL1QHFXa/mlAXCD+sUaP0rxg//yn2w/JH7wg=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0 h1:xGLAFFd9D3iLGxYiUGPdITSzsFmU1K8VtfuUHWAoN7M=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag=
 github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=

pkg/cloud/aliyunprovider.go

@@ -15,6 +15,8 @@ import (
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/signers"
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -309,8 +311,8 @@ type AlibabaPricing struct {
 	NodeAttributes *AlibabaNodeAttributes
 	PVAttributes   *AlibabaPVAttributes
 	PricingTerms   *AlibabaPricingTerms
-	Node           *Node
-	PV             *PV
+	Node           *models.Node
+	PV             *models.PV
 }
 
 // Alibaba cloud's Provider struct
@@ -326,7 +328,7 @@ type Alibaba struct {
 
 	// The following fields are unexported because of avoiding any leak of secrets of these keys.
 	// Alibaba Access key used specifically in signer interface used to sign API calls
-	serviceAccountChecks *ServiceAccountChecks
+	serviceAccountChecks *models.ServiceAccountChecks
 	clusterAccountId     string
 	clusterRegion        string
 	accessKey            *credentials.AccessKeyCredential
@@ -511,7 +513,7 @@ func (alibaba *Alibaba) AllNodePricing() (interface{}, error) {
 }
 
 // NodePricing gives pricing information of a specific node given by the key
-func (alibaba *Alibaba) NodePricing(key Key) (*Node, error) {
+func (alibaba *Alibaba) NodePricing(key models.Key) (*models.Node, error) {
 	alibaba.DownloadPricingDataLock.RLock()
 	defer alibaba.DownloadPricingDataLock.RUnlock()
 
@@ -531,7 +533,7 @@ func (alibaba *Alibaba) NodePricing(key Key) (*Node, error) {
 }
 
 // PVPricing gives a pricing information of a specific PV given by PVkey
-func (alibaba *Alibaba) PVPricing(pvk PVKey) (*PV, error) {
+func (alibaba *Alibaba) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	alibaba.DownloadPricingDataLock.RLock()
 	defer alibaba.DownloadPricingDataLock.RUnlock()
 
@@ -550,7 +552,7 @@ func (alibaba *Alibaba) PVPricing(pvk PVKey) (*PV, error) {
 
 // Inter zone and Inter region network cost are defaulted based on https://www.alibabacloud.com/help/en/cloud-data-transmission/latest/cross-region-data-transfers
 // Internet cost is default based on https://www.alibabacloud.com/help/en/elastic-compute-service/latest/public-bandwidth to $0.123
-func (alibaba *Alibaba) NetworkPricing() (*Network, error) {
+func (alibaba *Alibaba) NetworkPricing() (*models.Network, error) {
 	cpricing, err := alibaba.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -568,7 +570,7 @@ func (alibaba *Alibaba) NetworkPricing() (*Network, error) {
 		return nil, err
 	}
 
-	return &Network{
+	return &models.Network{
 		ZoneNetworkEgressCost:     znec,
 		RegionNetworkEgressCost:   rnec,
 		InternetNetworkEgressCost: inec,
@@ -577,7 +579,7 @@ func (alibaba *Alibaba) NetworkPricing() (*Network, error) {
 
 // Alibaba loadbalancer has three different types https://www.alibabacloud.com/product/server-load-balancer,
 // defaulted price to classic load balancer https://www.alibabacloud.com/help/en/server-load-balancer/latest/pay-as-you-go.
-func (alibaba *Alibaba) LoadBalancerPricing() (*LoadBalancer, error) {
+func (alibaba *Alibaba) LoadBalancerPricing() (*models.LoadBalancer, error) {
 	cpricing, err := alibaba.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -586,12 +588,12 @@ func (alibaba *Alibaba) LoadBalancerPricing() (*LoadBalancer, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &LoadBalancer{
+	return &models.LoadBalancer{
 		Cost: lbPricing,
 	}, nil
 }
 
-func (alibaba *Alibaba) GetConfig() (*CustomPricing, error) {
+func (alibaba *Alibaba) GetConfig() (*models.CustomPricing, error) {
 	c, err := alibaba.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -603,7 +605,7 @@ func (alibaba *Alibaba) GetConfig() (*CustomPricing, error) {
 		c.NegotiatedDiscount = "0%"
 	}
 	if c.ShareTenancyCosts == "" {
-		c.ShareTenancyCosts = defaultShareTenancyCost
+		c.ShareTenancyCosts = models.DefaultShareTenancyCost
 	}
 
 	return c, nil
@@ -617,14 +619,14 @@ func (alibaba *Alibaba) loadAlibabaAuthSecretAndSetEnv(force bool) error {
 		return nil
 	}
 
-	exists, err := fileutil.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(models.AuthSecretPath)
 	if !exists || err != nil {
-		return fmt.Errorf("failed to locate service account file: %s with err: %w", authSecretPath, err)
+		return fmt.Errorf("failed to locate service account file: %s with err: %w", models.AuthSecretPath, err)
 	}
 
-	result, err := os.ReadFile(authSecretPath)
+	result, err := os.ReadFile(models.AuthSecretPath)
 	if err != nil {
-		return fmt.Errorf("failed to read service account file: %s with err: %w", authSecretPath, err)
+		return fmt.Errorf("failed to read service account file: %s with err: %w", models.AuthSecretPath, err)
 	}
 
 	var ak *AlibabaAccessKey
@@ -699,12 +701,12 @@ func (alibaba *Alibaba) GetDisks() ([]byte, error) {
 	return nil, nil
 }
 
-func (alibaba *Alibaba) GetOrphanedResources() ([]OrphanedResource, error) {
+func (alibaba *Alibaba) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	return nil, errors.New("not implemented")
 }
 
-func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
-	return alibaba.Config.Update(func(c *CustomPricing) error {
+func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
+	return alibaba.Config.Update(func(c *models.CustomPricing) error {
 		if updateType != "" {
 			return fmt.Errorf("UpdateConfig for Alibaba Provider doesn't support updateType %s at this time", updateType)
 
@@ -715,10 +717,10 @@ func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*CustomPri
 				return err
 			}
 			for k, v := range a {
-				kUpper := toTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
+				kUpper := utils.ToTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
 				vstr, ok := v.(string)
 				if ok {
-					err := SetCustomPricingField(c, kUpper, vstr)
+					err := models.SetCustomPricingField(c, kUpper, vstr)
 					if err != nil {
 						return err
 					}
@@ -729,7 +731,7 @@ func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*CustomPri
 		}
 
 		if env.IsRemoteEnabled() {
-			err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
+			err := utils.UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
 			if err != nil {
 				return err
 			}
@@ -738,7 +740,7 @@ func (alibaba *Alibaba) UpdateConfig(r io.Reader, updateType string) (*CustomPri
 	})
 }
 
-func (alibaba *Alibaba) UpdateConfigFromConfigMap(cm map[string]string) (*CustomPricing, error) {
+func (alibaba *Alibaba) UpdateConfigFromConfigMap(cm map[string]string) (*models.CustomPricing, error) {
 	return alibaba.Config.UpdateFromMap(cm)
 }
 
@@ -753,18 +755,18 @@ func (alibaba *Alibaba) GetLocalStorageQuery(window, offset time.Duration, rate
 }
 
 // Will look at this in Next PR if needed
-func (alibaba *Alibaba) ApplyReservedInstancePricing(nodes map[string]*Node) {
+func (alibaba *Alibaba) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
 
 }
 
 // Will look at this in Next PR if needed
-func (alibaba *Alibaba) ServiceAccountStatus() *ServiceAccountStatus {
-	return &ServiceAccountStatus{}
+func (alibaba *Alibaba) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return &models.ServiceAccountStatus{}
 }
 
 // Will look at this in Next PR if needed
-func (alibaba *Alibaba) PricingSourceStatus() map[string]*PricingSource {
-	return map[string]*PricingSource{}
+func (alibaba *Alibaba) PricingSourceStatus() map[string]*models.PricingSource {
+	return map[string]*models.PricingSource{}
 }
 
 // Will look at this in Next PR if needed
@@ -840,7 +842,7 @@ func (alibabaNodeKey *AlibabaNodeKey) GPUCount() int {
 }
 
 // Get's the key for the k8s node input
-func (alibaba *Alibaba) GetKey(mapValue map[string]string, node *v1.Node) Key {
+func (alibaba *Alibaba) GetKey(mapValue map[string]string, node *v1.Node) models.Key {
 	slimK8sNode := generateSlimK8sNodeFromV1Node(node)
 
 	var aak *credentials.AccessKeyCredential
@@ -906,7 +908,7 @@ type AlibabaPVKey struct {
 	SizeInGiB         string
 }
 
-func (alibaba *Alibaba) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (alibaba *Alibaba) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	regionID := defaultRegion
 	// If default Region is not passed default it to cluster region ID.
 	if defaultRegion == "" {
@@ -1065,7 +1067,7 @@ type DescribePriceResponse struct {
 }
 
 // processDescribePriceAndCreateAlibabaPricing processes the DescribePrice API and generates the pricing information for alibaba node resource and alibaba pv resource that's backed by cloud disk.
-func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface{}, signer *signers.AccessKeySigner, custom *CustomPricing) (pricing *AlibabaPricing, err error) {
+func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface{}, signer *signers.AccessKeySigner, custom *models.CustomPricing) (pricing *AlibabaPricing, err error) {
 	pricing = &AlibabaPricing{}
 	var response DescribePriceResponse
 
@@ -1091,7 +1093,7 @@ func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface
 				return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
 			}
 			// TO-DO : Ask in PR How to get the defaults is it equal to AWS/GCP defaults? And what needs to be returned
-			pricing.Node = &Node{
+			pricing.Node = &models.Node{
 				Cost:         fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
 				BaseCPUPrice: custom.CPU,
 				BaseRAMPrice: custom.RAM,
@@ -1116,7 +1118,7 @@ func processDescribePriceAndCreateAlibabaPricing(client *sdk.Client, i interface
 				return nil, fmt.Errorf("unable to unmarshall json response to custom struct with err: %w", err)
 			}
 			pricing.PVAttributes = NewAlibabaPVAttributes(disk)
-			pricing.PV = &PV{
+			pricing.PV = &models.PV{
 				Cost: fmt.Sprintf("%f", response.PriceInfo.Price.TradePrice),
 			}
 			// TO-DO : Disk has support for Hour and Month but pricing API is failing for month for disk(Research why?) and same challenge as node pricing no prepaid/postpaid distinction in v1.PersistentVolume object have to look at APIs for th information.

pkg/cloud/aliyunprovider_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk"
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/signers"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	v1 "k8s.io/api/core/v1"
 	resource "k8s.io/apimachinery/pkg/api/resource"
 )
@@ -408,7 +409,7 @@ func TestProcessDescribePriceAndCreateAlibabaPricing(t *testing.T) {
 			expectedError: nil,
 		},
 	}
-	custom := &CustomPricing{}
+	custom := &models.CustomPricing{}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			pricingObj, err := processDescribePriceAndCreateAlibabaPricing(client, c.teststruct, signer, custom)

pkg/cloud/awsprovider.go

@@ -15,6 +15,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/kubecost"
 
 	"github.com/opencost/opencost/pkg/clustercache"
@@ -68,11 +70,11 @@ var (
 	regionRx      = regexp.MustCompile("([a-z]+-[a-z]+-[0-9])")
 )
 
-func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
+func (aws *AWS) PricingSourceStatus() map[string]*models.PricingSource {
 
-	sources := make(map[string]*PricingSource)
+	sources := make(map[string]*models.PricingSource)
 
-	sps := &PricingSource{
+	sps := &models.PricingSource{
 		Name:    SpotPricingSource,
 		Enabled: true,
 	}
@@ -96,7 +98,7 @@ func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 	}
 	sources[SpotPricingSource] = sps
 
-	rps := &PricingSource{
+	rps := &models.PricingSource{
 		Name:    ReservedInstancePricingSource,
 		Enabled: true,
 	}
@@ -177,7 +179,7 @@ type AWS struct {
 	ProjectID                   string
 	DownloadPricingDataLock     sync.RWMutex
 	Config                      *ProviderConfig
-	serviceAccountChecks        *ServiceAccountChecks
+	serviceAccountChecks        *models.ServiceAccountChecks
 	clusterManagementPrice      float64
 	clusterRegion               string
 	clusterAccountID            string
@@ -294,7 +296,7 @@ type AWSProductTerms struct {
 	Storage  string        `json:"storage"`
 	VCpu     string        `json:"vcpu"`
 	GPU      string        `json:"gpu"` // GPU represents the number of GPU on the instance
-	PV       *PV           `json:"pv"`
+	PV       *models.PV    `json:"pv"`
 }
 
 // ClusterIdEnvVar is the environment variable in which one can manually set the ClusterId
@@ -450,7 +452,7 @@ func (aws *AWS) GetManagementPlatform() (string, error) {
 	return "", nil
 }
 
-func (aws *AWS) GetConfig() (*CustomPricing, error) {
+func (aws *AWS) GetConfig() (*models.CustomPricing, error) {
 	c, err := aws.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -462,7 +464,7 @@ func (aws *AWS) GetConfig() (*CustomPricing, error) {
 		c.NegotiatedDiscount = "0%"
 	}
 	if c.ShareTenancyCosts == "" {
-		c.ShareTenancyCosts = defaultShareTenancyCost
+		c.ShareTenancyCosts = models.DefaultShareTenancyCost
 	}
 
 	return c, nil
@@ -518,12 +520,12 @@ func (aws *AWS) GetAWSAthenaInfo() (*AwsAthenaInfo, error) {
 	}, nil
 }
 
-func (aws *AWS) UpdateConfigFromConfigMap(cm map[string]string) (*CustomPricing, error) {
+func (aws *AWS) UpdateConfigFromConfigMap(cm map[string]string) (*models.CustomPricing, error) {
 	return aws.Config.UpdateFromMap(cm)
 }
 
-func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
-	return aws.Config.Update(func(c *CustomPricing) error {
+func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
+	return aws.Config.Update(func(c *models.CustomPricing) error {
 		if updateType == SpotInfoUpdateType {
 			asfi := AwsSpotFeedInfo{}
 			err := json.NewDecoder(r).Decode(&asfi)
@@ -568,10 +570,10 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 				return err
 			}
 			for k, v := range a {
-				kUpper := toTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
+				kUpper := utils.ToTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
 				vstr, ok := v.(string)
 				if ok {
-					err := SetCustomPricingField(c, kUpper, vstr)
+					err := models.SetCustomPricingField(c, kUpper, vstr)
 					if err != nil {
 						return err
 					}
@@ -582,7 +584,7 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 		}
 
 		if env.IsRemoteEnabled() {
-			err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
+			err := utils.UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
 			if err != nil {
 				return err
 			}
@@ -654,11 +656,11 @@ func (k *awsKey) getUsageType(labels map[string]string) string {
 	return ""
 }
 
-func (aws *AWS) PVPricing(pvk PVKey) (*PV, error) {
+func (aws *AWS) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	pricing, ok := aws.Pricing[pvk.Features()]
 	if !ok {
 		log.Debugf("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
-		return &PV{}, nil
+		return &models.PV{}, nil
 	}
 	return pricing.PV, nil
 }
@@ -672,7 +674,7 @@ type awsPVKey struct {
 	ProviderID             string
 }
 
-func (aws *AWS) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (aws *AWS) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	providerID := ""
 	if pv.Spec.AWSElasticBlockStore != nil {
 		providerID = pv.Spec.AWSElasticBlockStore.VolumeID
@@ -717,7 +719,7 @@ func (key *awsPVKey) Features() string {
 }
 
 // GetKey maps node labels to information needed to retrieve pricing data
-func (aws *AWS) GetKey(labels map[string]string, n *v1.Node) Key {
+func (aws *AWS) GetKey(labels map[string]string, n *v1.Node) models.Key {
 	return &awsKey{
 		SpotLabelName:  aws.SpotLabelName,
 		SpotLabelValue: aws.SpotLabelValue,
@@ -847,7 +849,7 @@ func (aws *AWS) DownloadPricingData() error {
 		}
 	}
 
-	pvkeys := make(map[string]PVKey)
+	pvkeys := make(map[string]models.PVKey)
 	for _, pv := range pvList {
 		params, ok := storageClassMap[pv.Spec.StorageClassName]
 		if !ok {
@@ -996,7 +998,7 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 					usageTypeNoRegion := usageTypeMatch[len(usageTypeMatch)-1]
 					key := locationToRegion[product.Attributes.Location] + "," + usageTypeNoRegion
 					spotKey := key + ",preemptible"
-					pv := &PV{
+					pv := &models.PV{
 						Class:  volTypes[usageTypeNoRegion],
 						Region: locationToRegion[product.Attributes.Location],
 					}
@@ -1113,7 +1115,7 @@ func (aws *AWS) refreshSpotPricing(force bool) {
 }
 
 // Stubbed NetworkPricing for AWS. Pull directly from aws.json for now
-func (aws *AWS) NetworkPricing() (*Network, error) {
+func (aws *AWS) NetworkPricing() (*models.Network, error) {
 	cpricing, err := aws.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -1131,14 +1133,14 @@ func (aws *AWS) NetworkPricing() (*Network, error) {
 		return nil, err
 	}
 
-	return &Network{
+	return &models.Network{
 		ZoneNetworkEgressCost:     znec,
 		RegionNetworkEgressCost:   rnec,
 		InternetNetworkEgressCost: inec,
 	}, nil
 }
 
-func (aws *AWS) LoadBalancerPricing() (*LoadBalancer, error) {
+func (aws *AWS) LoadBalancerPricing() (*models.LoadBalancer, error) {
 	fffrc := 0.025
 	afrc := 0.010
 	lbidc := 0.008
@@ -1152,7 +1154,7 @@ func (aws *AWS) LoadBalancerPricing() (*LoadBalancer, error) {
 	} else {
 		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
 	}
-	return &LoadBalancer{
+	return &models.LoadBalancer{
 		Cost: totalCost,
 	}, nil
 }
@@ -1188,7 +1190,7 @@ func (aws *AWS) savingsPlanPricing(instanceID string) (*SavingsPlanData, bool) {
 	return data, ok
 }
 
-func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*Node, error) {
+func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Key) (*models.Node, error) {
 	key := k.Features()
 
 	if spotInfo, ok := aws.spotPricing(k.ID()); ok {
@@ -1200,7 +1202,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 		} else {
 			log.Infof("Spot data for node %s is missing", k.ID())
 		}
-		return &Node{
+		return &models.Node{
 			Cost:         spotcost,
 			VCPU:         terms.VCpu,
 			RAM:          terms.Memory,
@@ -1213,7 +1215,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 		}, nil
 	} else if aws.isPreemptible(key) { // Preemptible but we don't have any data in the pricing report.
 		log.DedupedWarningf(5, "Node %s marked preemptible but we have no data in spot feed", k.ID())
-		return &Node{
+		return &models.Node{
 			VCPU:         terms.VCpu,
 			VCPUCost:     aws.BaseSpotCPUPrice,
 			RAM:          terms.Memory,
@@ -1226,7 +1228,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 		}, nil
 	} else if sp, ok := aws.savingsPlanPricing(k.ID()); ok {
 		strCost := fmt.Sprintf("%f", sp.EffectiveCost)
-		return &Node{
+		return &models.Node{
 			Cost:         strCost,
 			VCPU:         terms.VCpu,
 			RAM:          terms.Memory,
@@ -1240,7 +1242,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 
 	} else if ri, ok := aws.reservedInstancePricing(k.ID()); ok {
 		strCost := fmt.Sprintf("%f", ri.EffectiveCost)
-		return &Node{
+		return &models.Node{
 			Cost:         strCost,
 			VCPU:         terms.VCpu,
 			RAM:          terms.Memory,
@@ -1267,7 +1269,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 		}
 	}
 
-	return &Node{
+	return &models.Node{
 		Cost:         cost,
 		VCPU:         terms.VCpu,
 		RAM:          terms.Memory,
@@ -1281,7 +1283,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k Key) (*No
 }
 
 // NodePricing takes in a key from GetKey and returns a Node object for use in building the cost model.
-func (aws *AWS) NodePricing(k Key) (*Node, error) {
+func (aws *AWS) NodePricing(k models.Key) (*models.Node, error) {
 	aws.DownloadPricingDataLock.RLock()
 	defer aws.DownloadPricingDataLock.RUnlock()
 
@@ -1299,7 +1301,7 @@ func (aws *AWS) NodePricing(k Key) (*Node, error) {
 		err := aws.DownloadPricingData()
 		aws.DownloadPricingDataLock.RLock()
 		if err != nil {
-			return &Node{
+			return &models.Node{
 				Cost:             aws.BaseCPUPrice,
 				BaseCPUPrice:     aws.BaseCPUPrice,
 				BaseRAMPrice:     aws.BaseRAMPrice,
@@ -1310,7 +1312,7 @@ func (aws *AWS) NodePricing(k Key) (*Node, error) {
 		}
 		terms, termsOk := aws.Pricing[key]
 		if !termsOk {
-			return &Node{
+			return &models.Node{
 				Cost:             aws.BaseCPUPrice,
 				BaseCPUPrice:     aws.BaseCPUPrice,
 				BaseRAMPrice:     aws.BaseRAMPrice,
@@ -1381,7 +1383,7 @@ func (aws *AWS) ConfigureAuth() error {
 }
 
 // updates the authentication to the latest values (via config or secret)
-func (aws *AWS) ConfigureAuthWith(config *CustomPricing) error {
+func (aws *AWS) ConfigureAuthWith(config *models.CustomPricing) error {
 	accessKeyID, accessKeySecret := aws.getAWSAuth(false, config)
 	if accessKeyID != "" && accessKeySecret != "" { // credentials may exist on the actual AWS node-- if so, use those. If not, override with the service key
 		err := env.Set(env.AWSAccessKeyIDEnvVar, accessKeyID)
@@ -1397,11 +1399,11 @@ func (aws *AWS) ConfigureAuthWith(config *CustomPricing) error {
 }
 
 // Gets the aws key id and secret
-func (aws *AWS) getAWSAuth(forceReload bool, cp *CustomPricing) (string, string) {
+func (aws *AWS) getAWSAuth(forceReload bool, cp *models.CustomPricing) (string, string) {
 
 	// 1. Check config values first (set from frontend UI)
 	if cp.ServiceKeyName != "" && cp.ServiceKeySecret != "" {
-		aws.serviceAccountChecks.set("hasKey", &ServiceAccountCheck{
+		aws.serviceAccountChecks.Set("hasKey", &models.ServiceAccountCheck{
 			Message: "AWS ServiceKey exists",
 			Status:  true,
 		})
@@ -1411,7 +1413,7 @@ func (aws *AWS) getAWSAuth(forceReload bool, cp *CustomPricing) (string, string)
 	// 2. Check for secret
 	s, _ := aws.loadAWSAuthSecret(forceReload)
 	if s != nil && s.AccessKeyID != "" && s.SecretAccessKey != "" {
-		aws.serviceAccountChecks.set("hasKey", &ServiceAccountCheck{
+		aws.serviceAccountChecks.Set("hasKey", &models.ServiceAccountCheck{
 			Message: "AWS ServiceKey exists",
 			Status:  true,
 		})
@@ -1420,12 +1422,12 @@ func (aws *AWS) getAWSAuth(forceReload bool, cp *CustomPricing) (string, string)
 
 	// 3. Fall back to env vars
 	if env.GetAWSAccessKeyID() == "" || env.GetAWSAccessKeySecret() == "" {
-		aws.serviceAccountChecks.set("hasKey", &ServiceAccountCheck{
+		aws.serviceAccountChecks.Set("hasKey", &models.ServiceAccountCheck{
 			Message: "AWS ServiceKey exists",
 			Status:  false,
 		})
 	} else {
-		aws.serviceAccountChecks.set("hasKey", &ServiceAccountCheck{
+		aws.serviceAccountChecks.Set("hasKey", &models.ServiceAccountCheck{
 			Message: "AWS ServiceKey exists",
 			Status:  true,
 		})
@@ -1442,12 +1444,12 @@ func (aws *AWS) loadAWSAuthSecret(force bool) (*AWSAccessKey, error) {
 	}
 	loadedAWSSecret = true
 
-	exists, err := fileutil.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(models.AuthSecretPath)
 	if !exists || err != nil {
-		return nil, fmt.Errorf("Failed to locate service account file: %s", authSecretPath)
+		return nil, fmt.Errorf("Failed to locate service account file: %s", models.AuthSecretPath)
 	}
 
-	result, err := os.ReadFile(authSecretPath)
+	result, err := os.ReadFile(models.AuthSecretPath)
 	if err != nil {
 		return nil, err
 	}
@@ -1682,7 +1684,7 @@ func (aws *AWS) isDiskOrphaned(vol *ec2Types.Volume) bool {
 	return true
 }
 
-func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
+func (aws *AWS) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	volumes, err := aws.getAllDisks()
 	if err != nil {
 		return nil, err
@@ -1693,7 +1695,7 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 		return nil, err
 	}
 
-	var orphanedResources []OrphanedResource
+	var orphanedResources []models.OrphanedResource
 
 	for _, volume := range volumes {
 		if aws.isDiskOrphaned(volume) {
@@ -1720,7 +1722,7 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 				url = "https://console.aws.amazon.com/ec2/home?#Volumes:sort=desc:createTime"
 			}
 
-			or := OrphanedResource{
+			or := models.OrphanedResource{
 				Kind:        "disk",
 				Region:      zone,
 				Size:        &volumeSize,
@@ -1749,7 +1751,7 @@ func (aws *AWS) GetOrphanedResources() ([]OrphanedResource, error) {
 				}
 			}
 
-			or := OrphanedResource{
+			or := models.OrphanedResource{
 				Kind:        "address",
 				Address:     *address.PublicIp,
 				Description: desc,
@@ -2152,14 +2154,14 @@ func (aws *AWS) parseSpotData(bucket string, prefix string, projectID string, re
 	}
 	lso, err := cli.ListObjects(context.TODO(), ls)
 	if err != nil {
-		aws.serviceAccountChecks.set("bucketList", &ServiceAccountCheck{
+		aws.serviceAccountChecks.Set("bucketList", &models.ServiceAccountCheck{
 			Message:        "Bucket List Permissions Available",
 			Status:         false,
 			AdditionalInfo: err.Error(),
 		})
 		return nil, err
 	} else {
-		aws.serviceAccountChecks.set("bucketList", &ServiceAccountCheck{
+		aws.serviceAccountChecks.Set("bucketList", &models.ServiceAccountCheck{
 			Message: "Bucket List Permissions Available",
 			Status:  true,
 		})
@@ -2204,14 +2206,14 @@ func (aws *AWS) parseSpotData(bucket string, prefix string, projectID string, re
 		buf := manager.NewWriteAtBuffer([]byte{})
 		_, err := downloader.Download(context.TODO(), buf, getObj)
 		if err != nil {
-			aws.serviceAccountChecks.set("objectList", &ServiceAccountCheck{
+			aws.serviceAccountChecks.Set("objectList", &models.ServiceAccountCheck{
 				Message:        "Object Get Permissions Available",
 				Status:         false,
 				AdditionalInfo: err.Error(),
 			})
 			return nil, err
 		} else {
-			aws.serviceAccountChecks.set("objectList", &ServiceAccountCheck{
+			aws.serviceAccountChecks.Set("objectList", &models.ServiceAccountCheck{
 				Message: "Object Get Permissions Available",
 				Status:  true,
 			})
@@ -2282,12 +2284,12 @@ func (aws *AWS) parseSpotData(bucket string, prefix string, projectID string, re
 }
 
 // ApplyReservedInstancePricing TODO
-func (aws *AWS) ApplyReservedInstancePricing(nodes map[string]*Node) {
+func (aws *AWS) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
 
 }
 
-func (aws *AWS) ServiceAccountStatus() *ServiceAccountStatus {
-	return aws.serviceAccountChecks.getStatus()
+func (aws *AWS) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return aws.serviceAccountChecks.GetStatus()
 }
 
 func (aws *AWS) CombinedDiscountForNode(instanceType string, isPreemptible bool, defaultDiscount, negotiatedDiscount float64) float64 {

pkg/cloud/awsprovider_test.go

@@ -7,6 +7,8 @@ import (
 	"net/url"
 	"reflect"
 	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/models"
 )
 
 func Test_awsKey_getUsageType(t *testing.T) {
@@ -333,7 +335,7 @@ func Test_populate_pricing(t *testing.T) {
 				},
 			},
 		},
-		PV: &PV{
+		PV: &models.PV{
 			Cost:       "0.00010958904109589041",
 			CostPerIO:  "",
 			Class:      "gp3",
@@ -472,7 +474,7 @@ func Test_populate_pricing(t *testing.T) {
 				},
 			},
 		},
-		PV: &PV{
+		PV: &models.PV{
 			Cost:       "0.0007276712328767123",
 			CostPerIO:  "",
 			Class:      "gp3",

pkg/cloud/azureprovider.go → pkg/cloud/azure/azureprovider.go

@@ -1,4 +1,4 @@
-package cloud
+package azure
 
 import (
 	"context"
@@ -21,7 +21,8 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 
-	pricesheet "github.com/opencost/opencost/pkg/cloud/azurepricesheet"
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -387,21 +388,22 @@ type AzureRetailPricingAttributes struct {
 
 // AzurePricing either contains a Node or PV
 type AzurePricing struct {
-	Node *Node
-	PV   *PV
+	Node *models.Node
+	PV   *models.PV
 }
 
 type Azure struct {
-	Pricing                        map[string]*AzurePricing
-	DownloadPricingDataLock        sync.RWMutex
-	Clientset                      clustercache.ClusterCache
-	Config                         *ProviderConfig
-	serviceAccountChecks           *ServiceAccountChecks
+	Pricing                 map[string]*AzurePricing
+	DownloadPricingDataLock sync.RWMutex
+	Clientset               clustercache.ClusterCache
+	Config                  models.ProviderConfig
+	ServiceAccountChecks    *models.ServiceAccountChecks
+	ClusterAccountID        string
+	ClusterRegion           string
+
 	pricingSource                  string
 	rateCardPricingError           error
 	priceSheetPricingError         error
-	clusterAccountID               string
-	clusterRegion                  string
 	loadedAzureSecret              bool
 	azureSecret                    *AzureServiceKey
 	loadedAzureStorageConfigSecret bool
@@ -531,7 +533,7 @@ func (ask *AzureServiceKey) IsValid() bool {
 }
 
 // Loads the azure authentication via configuration or a secret set at install time.
-func (az *Azure) getAzureRateCardAuth(forceReload bool, cp *CustomPricing) (subscriptionID, clientID, clientSecret, tenantID string) {
+func (az *Azure) getAzureRateCardAuth(forceReload bool, cp *models.CustomPricing) (subscriptionID, clientID, clientSecret, tenantID string) {
 	// 1. Check for secret (secret values will always be used if they are present)
 	s, _ := az.loadAzureAuthSecret(forceReload)
 	if s != nil && s.IsValid() {
@@ -561,7 +563,7 @@ func (az *Azure) getAzureRateCardAuth(forceReload bool, cp *CustomPricing) (subs
 }
 
 // GetAzureStorageConfig retrieves storage config from secret and sets default values
-func (az *Azure) GetAzureStorageConfig(forceReload bool, cp *CustomPricing) (*AzureStorageConfig, error) {
+func (az *Azure) GetAzureStorageConfig(forceReload bool, cp *models.CustomPricing) (*AzureStorageConfig, error) {
 	// default subscription id
 	defaultSubscriptionID := cp.AzureSubscriptionID
 
@@ -577,7 +579,7 @@ func (az *Azure) GetAzureStorageConfig(forceReload bool, cp *CustomPricing) (*Az
 
 	// check for required fields
 	if asc != nil && asc.AccessKey != "" && asc.AccountName != "" && asc.ContainerName != "" && asc.SubscriptionId != "" {
-		az.serviceAccountChecks.set("hasStorage", &ServiceAccountCheck{
+		az.ServiceAccountChecks.Set("hasStorage", &models.ServiceAccountCheck{
 			Message: "Azure Storage Config exists",
 			Status:  true,
 		})
@@ -596,7 +598,7 @@ func (az *Azure) GetAzureStorageConfig(forceReload bool, cp *CustomPricing) (*Az
 		}
 		// check for required fields
 		if asc.AccessKey != "" && asc.AccountName != "" && asc.ContainerName != "" && asc.SubscriptionId != "" {
-			az.serviceAccountChecks.set("hasStorage", &ServiceAccountCheck{
+			az.ServiceAccountChecks.Set("hasStorage", &models.ServiceAccountCheck{
 				Message: "Azure Storage Config exists",
 				Status:  true,
 			})
@@ -605,7 +607,7 @@ func (az *Azure) GetAzureStorageConfig(forceReload bool, cp *CustomPricing) (*Az
 		}
 	}
 
-	az.serviceAccountChecks.set("hasStorage", &ServiceAccountCheck{
+	az.ServiceAccountChecks.Set("hasStorage", &models.ServiceAccountCheck{
 		Message: "Azure Storage Config exists",
 		Status:  false,
 	})
@@ -622,12 +624,12 @@ func (az *Azure) loadAzureAuthSecret(force bool) (*AzureServiceKey, error) {
 	}
 	az.loadedAzureSecret = true
 
-	exists, err := fileutil.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(models.AuthSecretPath)
 	if !exists || err != nil {
-		return nil, fmt.Errorf("Failed to locate service account file: %s", authSecretPath)
+		return nil, fmt.Errorf("Failed to locate service account file: %s", models.AuthSecretPath)
 	}
 
-	result, err := os.ReadFile(authSecretPath)
+	result, err := os.ReadFile(models.AuthSecretPath)
 	if err != nil {
 		return nil, err
 	}
@@ -651,12 +653,12 @@ func (az *Azure) loadAzureStorageConfig(force bool) (*AzureStorageConfig, error)
 	}
 	az.loadedAzureStorageConfigSecret = true
 
-	exists, err := fileutil.FileExists(storageConfigSecretPath)
+	exists, err := fileutil.FileExists(models.StorageConfigSecretPath)
 	if !exists || err != nil {
-		return nil, fmt.Errorf("Failed to locate azure storage config file: %s", storageConfigSecretPath)
+		return nil, fmt.Errorf("Failed to locate azure storage config file: %s", models.StorageConfigSecretPath)
 	}
 
-	result, err := os.ReadFile(storageConfigSecretPath)
+	result, err := os.ReadFile(models.StorageConfigSecretPath)
 	if err != nil {
 		return nil, err
 	}
@@ -671,7 +673,7 @@ func (az *Azure) loadAzureStorageConfig(force bool) (*AzureStorageConfig, error)
 	return &asc, nil
 }
 
-func (az *Azure) GetKey(labels map[string]string, n *v1.Node) Key {
+func (az *Azure) GetKey(labels map[string]string, n *v1.Node) models.Key {
 	cfg, err := az.GetConfig()
 	if err != nil {
 		log.Infof("Error loading azure custom pricing information")
@@ -805,7 +807,7 @@ func (az *Azure) DownloadPricingData() error {
 
 	var authorizer autorest.Authorizer
 
-	azureEnv := determineCloudByRegion(az.clusterRegion)
+	azureEnv := determineCloudByRegion(az.ClusterRegion)
 
 	if config.AzureClientID != "" && config.AzureClientSecret != "" && config.AzureTenantID != "" {
 		credentialsConfig := NewClientCredentialsConfig(config.AzureClientID, config.AzureClientSecret, config.AzureTenantID, azureEnv)
@@ -877,7 +879,7 @@ func (az *Azure) DownloadPricingData() error {
 
 	// If we've got a billing account set, kick off downloading the custom pricing data.
 	if config.AzureBillingAccount != "" {
-		downloader := pricesheet.Downloader[AzurePricing]{
+		downloader := PriceSheetDownloader{
 			TenantID:       config.AzureTenantID,
 			ClientID:       config.AzureClientID,
 			ClientSecret:   config.AzureClientSecret,
@@ -957,7 +959,7 @@ func convertMeterToPricings(info commerce.MeterInfo, regions map[string]string,
 				log.Debugf("Adding PV.Key: %s, Cost: %s", key, priceStr)
 				return map[string]*AzurePricing{
 					key: {
-						PV: &PV{
+						PV: &models.PV{
 							Cost:   priceStr,
 							Region: region,
 						},
@@ -1007,7 +1009,7 @@ func convertMeterToPricings(info commerce.MeterInfo, regions map[string]string,
 
 		key := fmt.Sprintf("%s,%s,%s", region, instanceType, usageType)
 		pricing := &AzurePricing{
-			Node: &Node{
+			Node: &models.Node{
 				Cost:         priceStr,
 				BaseCPUPrice: baseCPUPrice,
 				UsageType:    usageType,
@@ -1028,7 +1030,7 @@ func addAzureFilePricing(prices map[string]*AzurePricing, regions map[string]str
 		key := region + "," + AzureFileStandardStorageClass
 		log.Debugf("Adding PV.Key: %s, Cost: %s", key, zeroPrice)
 		prices[key] = &AzurePricing{
-			PV: &PV{
+			PV: &models.PV{
 				Cost:   zeroPrice,
 				Region: region,
 			},
@@ -1075,7 +1077,7 @@ func (az *Azure) AllNodePricing() (interface{}, error) {
 }
 
 // NodePricing returns Azure pricing data for a single node
-func (az *Azure) NodePricing(key Key) (*Node, error) {
+func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
 	pricingDataExists := true
@@ -1112,7 +1114,7 @@ func (az *Azure) NodePricing(key Key) (*Node, error) {
 			if azKey.isValidGPUNode() {
 				gpu = "1"
 			}
-			spotNode := &Node{
+			spotNode := &models.Node{
 				Cost:      spotCost,
 				UsageType: "spot",
 				GPU:       gpu,
@@ -1143,7 +1145,7 @@ func (az *Azure) NodePricing(key Key) (*Node, error) {
 
 	// GPU Node
 	if azKey.isValidGPUNode() {
-		return &Node{
+		return &models.Node{
 			VCPUCost:         c.CPU,
 			RAMCost:          c.RAM,
 			UsesBaseCPUPrice: true,
@@ -1156,14 +1158,14 @@ func (az *Azure) NodePricing(key Key) (*Node, error) {
 	// scheduled to this node. Azure does not charge for this node. Set costs to
 	// zero.
 	if azKey.Labels["kubernetes.io/hostname"] == "virtual-node-aci-linux" {
-		return &Node{
+		return &models.Node{
 			VCPUCost: "0",
 			RAMCost:  "0",
 		}, nil
 	}
 
 	// Regular Node
-	return &Node{
+	return &models.Node{
 		VCPUCost:         c.CPU,
 		RAMCost:          c.RAM,
 		UsesBaseCPUPrice: true,
@@ -1171,7 +1173,7 @@ func (az *Azure) NodePricing(key Key) (*Node, error) {
 }
 
 // Stubbed NetworkPricing for Azure. Pull directly from azure.json for now
-func (az *Azure) NetworkPricing() (*Network, error) {
+func (az *Azure) NetworkPricing() (*models.Network, error) {
 	cpricing, err := az.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -1189,7 +1191,7 @@ func (az *Azure) NetworkPricing() (*Network, error) {
 		return nil, err
 	}
 
-	return &Network{
+	return &models.Network{
 		ZoneNetworkEgressCost:     znec,
 		RegionNetworkEgressCost:   rnec,
 		InternetNetworkEgressCost: inec,
@@ -1200,8 +1202,8 @@ func (az *Azure) NetworkPricing() (*Network, error) {
 // services will be that of a standard static public IP https://azure.microsoft.com/en-us/pricing/details/ip-addresses/.
 // Azure still has load balancers which follow the standard pricing scheme based on rules
 // https://azure.microsoft.com/en-us/pricing/details/load-balancer/, they are created on a per-cluster basis.
-func (azr *Azure) LoadBalancerPricing() (*LoadBalancer, error) {
-	return &LoadBalancer{
+func (azr *Azure) LoadBalancerPricing() (*models.LoadBalancer, error) {
+	return &models.LoadBalancer{
 		Cost: 0.005,
 	}, nil
 }
@@ -1214,7 +1216,7 @@ type azurePvKey struct {
 	ProviderId             string
 }
 
-func (az *Azure) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (az *Azure) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	providerID := ""
 	if pv.Spec.AzureDisk != nil {
 		providerID = pv.Spec.AzureDisk.DiskName
@@ -1289,7 +1291,7 @@ func (az *Azure) getDisks() ([]*compute.Disk, error) {
 
 	var authorizer autorest.Authorizer
 
-	azureEnv := determineCloudByRegion(az.clusterRegion)
+	azureEnv := determineCloudByRegion(az.ClusterRegion)
 
 	if config.AzureClientID != "" && config.AzureClientSecret != "" && config.AzureTenantID != "" {
 		credentialsConfig := NewClientCredentialsConfig(config.AzureClientID, config.AzureClientSecret, config.AzureTenantID, azureEnv)
@@ -1344,13 +1346,13 @@ func (az *Azure) isDiskOrphaned(disk *compute.Disk) bool {
 	return disk.DiskState == "Unattached" || disk.DiskState == "Reserved"
 }
 
-func (az *Azure) GetOrphanedResources() ([]OrphanedResource, error) {
+func (az *Azure) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	disks, err := az.getDisks()
 	if err != nil {
 		return nil, err
 	}
 
-	var orphanedResources []OrphanedResource
+	var orphanedResources []models.OrphanedResource
 
 	for _, d := range disks {
 		if az.isDiskOrphaned(d) {
@@ -1383,7 +1385,7 @@ func (az *Azure) GetOrphanedResources() ([]OrphanedResource, error) {
 				}
 			}
 
-			or := OrphanedResource{
+			or := models.OrphanedResource{
 				Kind:        "disk",
 				Region:      diskRegion,
 				Description: desc,
@@ -1435,20 +1437,20 @@ func (az *Azure) ClusterInfo() (map[string]string, error) {
 		m["name"] = c.ClusterName
 	}
 	m["provider"] = kubecost.AzureProvider
-	m["account"] = az.clusterAccountID
-	m["region"] = az.clusterRegion
+	m["account"] = az.ClusterAccountID
+	m["region"] = az.ClusterRegion
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 	m["id"] = env.GetClusterID()
 	return m, nil
 
 }
 
-func (az *Azure) UpdateConfigFromConfigMap(a map[string]string) (*CustomPricing, error) {
+func (az *Azure) UpdateConfigFromConfigMap(a map[string]string) (*models.CustomPricing, error) {
 	return az.Config.UpdateFromMap(a)
 }
 
-func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
-	return az.Config.Update(func(c *CustomPricing) error {
+func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
+	return az.Config.Update(func(c *models.CustomPricing) error {
 		if updateType == AzureStorageUpdateType {
 			asc := &AzureStorageConfig{}
 			err := json.NewDecoder(r).Decode(&asc)
@@ -1481,10 +1483,10 @@ func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, e
 
 			for k, v := range a {
 				// Just so we consistently supply / receive the same values, uppercase the first letter.
-				kUpper := toTitle.String(k)
+				kUpper := utils.ToTitle.String(k)
 				vstr, ok := v.(string)
 				if ok {
-					err := SetCustomPricingField(c, kUpper, vstr)
+					err := models.SetCustomPricingField(c, kUpper, vstr)
 					if err != nil {
 						return fmt.Errorf("error setting custom pricing field on AzureStorageConfig: %s", err)
 					}
@@ -1495,7 +1497,7 @@ func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, e
 		}
 
 		if env.IsRemoteEnabled() {
-			err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
+			err := utils.UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
 			if err != nil {
 				return fmt.Errorf("error updating cluster metadata: %s", err)
 			}
@@ -1505,7 +1507,7 @@ func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, e
 	})
 }
 
-func (az *Azure) GetConfig() (*CustomPricing, error) {
+func (az *Azure) GetConfig() (*models.CustomPricing, error) {
 	c, err := az.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -1527,7 +1529,7 @@ func (az *Azure) GetConfig() (*CustomPricing, error) {
 		c.AzureOfferDurableID = "MS-AZR-0003p"
 	}
 	if c.ShareTenancyCosts == "" {
-		c.ShareTenancyCosts = defaultShareTenancyCost
+		c.ShareTenancyCosts = models.DefaultShareTenancyCost
 	}
 	if c.SpotLabel == "" {
 		c.SpotLabel = defaultSpotLabel
@@ -1538,18 +1540,18 @@ func (az *Azure) GetConfig() (*CustomPricing, error) {
 	return c, nil
 }
 
-func (az *Azure) ApplyReservedInstancePricing(nodes map[string]*Node) {
+func (az *Azure) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
 
 }
 
-func (az *Azure) PVPricing(pvk PVKey) (*PV, error) {
+func (az *Azure) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
 
 	pricing, ok := az.Pricing[pvk.Features()]
 	if !ok {
 		log.Debugf("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
-		return &PV{}, nil
+		return &models.PV{}, nil
 	}
 	return pricing.PV, nil
 }
@@ -1558,8 +1560,8 @@ func (az *Azure) GetLocalStorageQuery(window, offset time.Duration, rate bool, u
 	return ""
 }
 
-func (az *Azure) ServiceAccountStatus() *ServiceAccountStatus {
-	return az.serviceAccountChecks.getStatus()
+func (az *Azure) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return az.ServiceAccountChecks.GetStatus()
 }
 
 const (
@@ -1568,15 +1570,15 @@ const (
 )
 
 // PricingSourceStatus returns the status of the rate card api
-func (az *Azure) PricingSourceStatus() map[string]*PricingSource {
+func (az *Azure) PricingSourceStatus() map[string]*models.PricingSource {
 	az.DownloadPricingDataLock.Lock()
 	defer az.DownloadPricingDataLock.Unlock()
-	sources := make(map[string]*PricingSource)
+	sources := make(map[string]*models.PricingSource)
 	errMsg := ""
 	if az.rateCardPricingError != nil {
 		errMsg = az.rateCardPricingError.Error()
 	}
-	rcps := &PricingSource{
+	rcps := &models.PricingSource{
 		Name:    rateCardPricingSource,
 		Enabled: az.pricingSource == rateCardPricingSource,
 		Error:   errMsg,
@@ -1594,7 +1596,7 @@ func (az *Azure) PricingSourceStatus() map[string]*PricingSource {
 	if az.priceSheetPricingError != nil {
 		errMsg = az.priceSheetPricingError.Error()
 	}
-	psps := &PricingSource{
+	psps := &models.PricingSource{
 		Name:    priceSheetPricingSource,
 		Enabled: az.pricingSource == priceSheetPricingSource,
 		Error:   errMsg,
@@ -1635,7 +1637,7 @@ func (az *Azure) Regions() []string {
 	return azureRegions
 }
 
-func parseAzureSubscriptionID(id string) string {
+func ParseAzureSubscriptionID(id string) string {
 	match := azureSubRegex.FindStringSubmatch(id)
 	if len(match) >= 2 {
 		return match[1]

pkg/cloud/azureprovider_test.go → pkg/cloud/azure/azureprovider_test.go

@@ -1,10 +1,12 @@
-package cloud
+package azure
 
 import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/services/preview/commerce/mgmt/2015-06-01-preview/commerce"
 	"github.com/stretchr/testify/require"
+
+	"github.com/opencost/opencost/pkg/cloud/models"
 )
 
 func TestParseAzureSubscriptionID(t *testing.T) {
@@ -31,7 +33,7 @@ func TestParseAzureSubscriptionID(t *testing.T) {
 	}
 
 	for _, test := range cases {
-		result := parseAzureSubscriptionID(test.input)
+		result := ParseAzureSubscriptionID(test.input)
 		if result != test.expected {
 			t.Errorf("Input: %s, Expected: %s, Actual: %s", test.input, test.expected, result)
 		}
@@ -71,7 +73,7 @@ func TestConvertMeterToPricings(t *testing.T) {
 
 		expected := map[string]*AzurePricing{
 			"useast,premium_ssd": {
-				PV: &PV{Cost: "0.085616", Region: "useast"},
+				PV: &models.PV{Cost: "0.085616", Region: "useast"},
 			},
 		}
 		require.Equal(t, expected, results)
@@ -84,10 +86,10 @@ func TestConvertMeterToPricings(t *testing.T) {
 
 		expected := map[string]*AzurePricing{
 			"japanwest,Standard_E96a_v4,preemptible": {
-				Node: &Node{Cost: "10.000000", BaseCPUPrice: "0.30000", UsageType: "preemptible"},
+				Node: &models.Node{Cost: "10.000000", BaseCPUPrice: "0.30000", UsageType: "preemptible"},
 			},
 			"japanwest,Standard_E96as_v4,preemptible": {
-				Node: &Node{Cost: "10.000000", BaseCPUPrice: "0.30000", UsageType: "preemptible"},
+				Node: &models.Node{Cost: "10.000000", BaseCPUPrice: "0.30000", UsageType: "preemptible"},
 			},
 		}
 		require.Equal(t, expected, results)

pkg/cloud/azurepricesheet/client.go → pkg/cloud/azure/client.go

@@ -1,4 +1,4 @@
-package azurepricesheet
+package azure
 
 import (
 	"context"
@@ -34,11 +34,11 @@ type PriceSheetClient struct {
 	pl               runtime.Pipeline
 }
 
-// NewClient creates a new instance of PriceSheetClient with the specified values.
+// NewPriceSheetClient creates a new instance of PriceSheetClient with the specified values.
 // billingAccountId - Azure Billing Account ID.
 // credential - used to authorize requests. Usually a credential from azidentity.
 // options - pass nil to accept the default values.
-func NewClient(billingAccountID string, credential azcore.TokenCredential, options *arm.ClientOptions) (*PriceSheetClient, error) {
+func NewPriceSheetClient(billingAccountID string, credential azcore.TokenCredential, options *arm.ClientOptions) (*PriceSheetClient, error) {
 	if options == nil {
 		options = &arm.ClientOptions{}
 	}

pkg/cloud/azurepricesheet/downloader.go → pkg/cloud/azure/downloader.go

@@ -1,4 +1,4 @@
-package azurepricesheet
+package azure
 
 import (
 	"bufio"
@@ -20,18 +20,18 @@ import (
 	"github.com/opencost/opencost/pkg/log"
 )
 
-type Downloader[T any] struct {
+type PriceSheetDownloader struct {
 	TenantID         string
 	ClientID         string
 	ClientSecret     string
 	BillingAccount   string
 	OfferID          string
-	ConvertMeterInfo func(info commerce.MeterInfo) (map[string]*T, error)
+	ConvertMeterInfo func(info commerce.MeterInfo) (map[string]*AzurePricing, error)
 }
 
-func (d *Downloader[T]) GetPricing(ctx context.Context) (map[string]*T, error) {
+func (d *PriceSheetDownloader) GetPricing(ctx context.Context) (map[string]*AzurePricing, error) {
 	log.Infof("requesting pricesheet download link")
-	url, err := d.getPricesheetDownloadURL(ctx)
+	url, err := d.getDownloadURL(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("getting download URL: %w", err)
 	}
@@ -50,12 +50,12 @@ func (d *Downloader[T]) GetPricing(ctx context.Context) (map[string]*T, error) {
 	return prices, nil
 }
 
-func (d *Downloader[T]) getPricesheetDownloadURL(ctx context.Context) (string, error) {
+func (d *PriceSheetDownloader) getDownloadURL(ctx context.Context) (string, error) {
 	cred, err := azidentity.NewClientSecretCredential(d.TenantID, d.ClientID, d.ClientSecret, nil)
 	if err != nil {
 		return "", fmt.Errorf("creating credential: %w", err)
 	}
-	client, err := NewClient(d.BillingAccount, cred, nil)
+	client, err := NewPriceSheetClient(d.BillingAccount, cred, nil)
 	if err != nil {
 		return "", fmt.Errorf("creating pricesheet client: %w", err)
 	}
@@ -72,7 +72,7 @@ func (d *Downloader[T]) getPricesheetDownloadURL(ctx context.Context) (string, e
 	return resp.Properties.DownloadURL, nil
 }
 
-func (d Downloader[T]) saveData(ctx context.Context, url, tempName string) (io.ReadCloser, error) {
+func (d PriceSheetDownloader) saveData(ctx context.Context, url, tempName string) (io.ReadCloser, error) {
 	// Download file from URL in response.
 	out, err := os.CreateTemp("", tempName)
 	if err != nil {
@@ -113,7 +113,7 @@ func (r *removeOnClose) Close() error {
 	return os.Remove(r.Name())
 }
 
-func (d *Downloader[T]) readPricesheet(ctx context.Context, data io.Reader) (map[string]*T, error) {
+func (d *PriceSheetDownloader) readPricesheet(ctx context.Context, data io.Reader) (map[string]*AzurePricing, error) {
 	// Avoid double-buffering.
 	buf, ok := (data).(*bufio.Reader)
 	if !ok {
@@ -144,7 +144,7 @@ func (d *Downloader[T]) readPricesheet(ctx context.Context, data io.Reader) (map
 
 	units := make(map[string]bool)
 
-	results := make(map[string]*T)
+	results := make(map[string]*AzurePricing)
 	lines := 2
 	for {
 		row, err := reader.Read()

pkg/cloud/azurepricesheet/downloader_test.go → pkg/cloud/azure/downloader_test.go

@@ -1,4 +1,4 @@
-package azurepricesheet
+package azure
 
 import (
 	"context"
@@ -7,11 +7,12 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/2020-09-01/commerce/mgmt/commerce"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/stretchr/testify/require"
 )
 
 func TestDownloader(t *testing.T) {
-	d := Downloader[fakePricing]{
+	d := PriceSheetDownloader{
 		TenantID:         "test-tenant-id",
 		ClientID:         "test-client-id",
 		ClientSecret:     "test-client-secret",
@@ -26,11 +27,11 @@ func TestDownloader(t *testing.T) {
 
 		// Units and prices are normalised.
 		// Info for saving plans and other offers is skipped.
-		expected := map[string]*fakePricing{
-			"DC96as_v4": {price: "10.505", unit: "1 Hour"},
-			"DC2as_v4":  {price: "0.219", unit: "1 Hour"},
-			"VM1":       {price: "1.0", unit: "1 Hour"},
-			"VM2":       {price: "2.0", unit: "1 Hour"},
+		expected := map[string]*AzurePricing{
+			"DC96as_v4 1 Hour": {Node: &models.Node{Cost: "10.505"}},
+			"DC2as_v4 1 Hour":  {Node: &models.Node{Cost: "0.219"}},
+			"VM1 1 Hour":       {Node: &models.Node{Cost: "1.0"}},
+			"VM2 1 Hour":       {Node: &models.Node{Cost: "2.0"}},
 		}
 		require.Equal(t, expected, results)
 	})
@@ -48,13 +49,13 @@ func TestDownloader(t *testing.T) {
 	})
 
 	t.Run("no matching prices", func(t *testing.T) {
-		d := Downloader[fakePricing]{
+		d := PriceSheetDownloader{
 			TenantID:       "test-tenant-id",
 			ClientID:       "test-client-id",
 			ClientSecret:   "test-client-secret",
 			BillingAccount: "test-billing-account",
 			OfferID:        "my-offer-id",
-			ConvertMeterInfo: func(commerce.MeterInfo) (map[string]*fakePricing, error) {
+			ConvertMeterInfo: func(commerce.MeterInfo) (map[string]*AzurePricing, error) {
 				return nil, nil
 			},
 		}
@@ -63,29 +64,26 @@ func TestDownloader(t *testing.T) {
 	})
 }
 
-func convertMeter(info commerce.MeterInfo) (map[string]*fakePricing, error) {
+func convertMeter(info commerce.MeterInfo) (map[string]*AzurePricing, error) {
 	switch *info.MeterName {
 	case "skip-this":
 		return nil, nil
 	case "multiple-prices":
-		return map[string]*fakePricing{
-			"VM1": {price: "1.0", unit: "1 Hour"},
-			"VM2": {price: "2.0", unit: "1 Hour"},
+		return map[string]*AzurePricing{
+			"VM1 1 Hour": {Node: &models.Node{Cost: "1.0"}},
+			"VM2 1 Hour": {Node: &models.Node{Cost: "2.0"}},
 		}, nil
 	case "error":
 		return nil, fmt.Errorf("there was an error handling this row!")
 	default:
-		return map[string]*fakePricing{
-			*info.MeterName: {price: fmt.Sprintf("%0.3f", *info.MeterRates["0"]), unit: *info.Unit},
+		return map[string]*AzurePricing{
+			*info.MeterName + " " + *info.Unit: {
+				Node: &models.Node{Cost: fmt.Sprintf("%0.3f", *info.MeterRates["0"])},
+			},
 		}, nil
 	}
 }
 
-type fakePricing struct {
-	price string
-	unit  string
-}
-
 const pricesheetData = `Price Sheet Report for billing period - 202304
 
 Meter ID,Meter name,Meter category,Meter sub-category,Meter region,Unit,Unit of measure,Part number,Unit price,Currency code,Included quantity,Offer Id,Term,Price type

pkg/cloud/csvprovider.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/util"
 
@@ -222,31 +223,31 @@ func (k *csvKey) ID() string {
 	return k.ProviderID
 }
 
-func (c *CSVProvider) NodePricing(key Key) (*Node, error) {
+func (c *CSVProvider) NodePricing(key models.Key) (*models.Node, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()
-	var node *Node
+	var node *models.Node
 	if p, ok := c.Pricing[key.ID()]; ok {
-		node = &Node{
+		node = &models.Node{
 			Cost:        p.MarketPriceHourly,
-			PricingType: CsvExact,
+			PricingType: models.CsvExact,
 		}
 	}
 	s := strings.Split(key.ID(), ",") // Try without a region to be sure
 	if len(s) == 2 {
 		if p, ok := c.Pricing[s[1]]; ok {
-			node = &Node{
+			node = &models.Node{
 				Cost:        p.MarketPriceHourly,
-				PricingType: CsvExact,
+				PricingType: models.CsvExact,
 			}
 		}
 	}
 	classKey := key.Features() // Use node attributes to try and do a class match
 	if cost, ok := c.NodeClassPricing[classKey]; ok {
 		log.Infof("Unable to find provider ID `%s`, using features:`%s`", key.ID(), key.Features())
-		node = &Node{
+		node = &models.Node{
 			Cost:        fmt.Sprintf("%f", cost),
-			PricingType: CsvClass,
+			PricingType: models.CsvClass,
 		}
 	}
 
@@ -346,7 +347,7 @@ func PVValueFromMapField(m string, n *v1.PersistentVolume) string {
 	}
 }
 
-func (c *CSVProvider) GetKey(l map[string]string, n *v1.Node) Key {
+func (c *CSVProvider) GetKey(l map[string]string, n *v1.Node) models.Key {
 	id := NodeValueFromMapField(c.NodeMapField, n, c.UsesRegion)
 	var gpuCount int64
 	gpuCount = 0
@@ -382,7 +383,7 @@ func (key *csvPVKey) Features() string {
 	return key.ProviderID
 }
 
-func (c *CSVProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (c *CSVProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	id := PVValueFromMapField(c.PVMapField, pv)
 	return &csvPVKey{
 		Labels:                 pv.Labels,
@@ -394,22 +395,22 @@ func (c *CSVProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]st
 	}
 }
 
-func (c *CSVProvider) PVPricing(pvk PVKey) (*PV, error) {
+func (c *CSVProvider) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()
 	pricing, ok := c.PricingPV[pvk.Features()]
 	if !ok {
 		log.Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
-		return &PV{}, nil
+		return &models.PV{}, nil
 	}
-	return &PV{
+	return &models.PV{
 		Cost: pricing.MarketPriceHourly,
 	}, nil
 }
 
-func (c *CSVProvider) ServiceAccountStatus() *ServiceAccountStatus {
-	return &ServiceAccountStatus{
-		Checks: []*ServiceAccountCheck{},
+func (c *CSVProvider) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return &models.ServiceAccountStatus{
+		Checks: []*models.ServiceAccountCheck{},
 	}
 }
 

pkg/cloud/customprovider.go

@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -59,7 +61,7 @@ func (*CustomProvider) GetLocalStorageQuery(window, offset time.Duration, rate b
 	return ""
 }
 
-func (cp *CustomProvider) GetConfig() (*CustomPricing, error) {
+func (cp *CustomProvider) GetConfig() (*models.CustomPricing, error) {
 	return cp.Config.GetCustomPricingData()
 }
 
@@ -67,15 +69,15 @@ func (*CustomProvider) GetManagementPlatform() (string, error) {
 	return "", nil
 }
 
-func (*CustomProvider) ApplyReservedInstancePricing(nodes map[string]*Node) {
+func (*CustomProvider) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
 
 }
 
-func (cp *CustomProvider) UpdateConfigFromConfigMap(a map[string]string) (*CustomPricing, error) {
+func (cp *CustomProvider) UpdateConfigFromConfigMap(a map[string]string) (*models.CustomPricing, error) {
 	return cp.Config.UpdateFromMap(a)
 }
 
-func (cp *CustomProvider) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
+func (cp *CustomProvider) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
 	// Parse config updates from reader
 	a := make(map[string]interface{})
 	err := json.NewDecoder(r).Decode(&a)
@@ -84,12 +86,12 @@ func (cp *CustomProvider) UpdateConfig(r io.Reader, updateType string) (*CustomP
 	}
 
 	// Update Config
-	c, err := cp.Config.Update(func(c *CustomPricing) error {
+	c, err := cp.Config.Update(func(c *models.CustomPricing) error {
 		for k, v := range a {
-			kUpper := toTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
+			kUpper := utils.ToTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
 			vstr, ok := v.(string)
 			if ok {
-				err := SetCustomPricingField(c, kUpper, vstr)
+				err := models.SetCustomPricingField(c, kUpper, vstr)
 				if err != nil {
 					return err
 				}
@@ -133,7 +135,7 @@ func (*CustomProvider) GetDisks() ([]byte, error) {
 	return nil, nil
 }
 
-func (*CustomProvider) GetOrphanedResources() ([]OrphanedResource, error) {
+func (*CustomProvider) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	return nil, errors.New("not implemented")
 }
 
@@ -144,7 +146,7 @@ func (cp *CustomProvider) AllNodePricing() (interface{}, error) {
 	return cp.Pricing, nil
 }
 
-func (cp *CustomProvider) NodePricing(key Key) (*Node, error) {
+func (cp *CustomProvider) NodePricing(key models.Key) (*models.Node, error) {
 	cp.DownloadPricingDataLock.RLock()
 	defer cp.DownloadPricingDataLock.RUnlock()
 
@@ -172,7 +174,7 @@ func (cp *CustomProvider) NodePricing(key Key) (*Node, error) {
 		gpuCost = pricing.GPU
 	}
 
-	return &Node{
+	return &models.Node{
 		VCPUCost: cpuCost,
 		RAMCost:  ramCost,
 		GPUCost:  gpuCost,
@@ -212,7 +214,7 @@ func (cp *CustomProvider) DownloadPricingData() error {
 	return nil
 }
 
-func (cp *CustomProvider) GetKey(labels map[string]string, n *v1.Node) Key {
+func (cp *CustomProvider) GetKey(labels map[string]string, n *v1.Node) models.Key {
 	return &customProviderKey{
 		SpotLabel:      cp.SpotLabel,
 		SpotLabelValue: cp.SpotLabelValue,
@@ -225,7 +227,7 @@ func (cp *CustomProvider) GetKey(labels map[string]string, n *v1.Node) Key {
 // ExternalAllocations represents tagged assets outside the scope of kubernetes.
 // "start" and "end" are dates of the format YYYY-MM-DD
 // "aggregator" is the tag used to determine how to allocate those assets, ie namespace, pod, etc.
-func (*CustomProvider) ExternalAllocations(start string, end string, aggregator []string, filterType string, filterValue string, crossCluster bool) ([]*OutOfClusterAllocation, error) {
+func (*CustomProvider) ExternalAllocations(start string, end string, aggregator []string, filterType string, filterValue string, crossCluster bool) ([]*models.OutOfClusterAllocation, error) {
 	return nil, nil // TODO: transform the QuerySQL lines into the new OutOfClusterAllocation Struct
 }
 
@@ -233,17 +235,17 @@ func (*CustomProvider) QuerySQL(query string) ([]byte, error) {
 	return nil, nil
 }
 
-func (cp *CustomProvider) PVPricing(pvk PVKey) (*PV, error) {
+func (cp *CustomProvider) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	cpricing, err := cp.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
 	}
-	return &PV{
+	return &models.PV{
 		Cost: cpricing.Storage,
 	}, nil
 }
 
-func (cp *CustomProvider) NetworkPricing() (*Network, error) {
+func (cp *CustomProvider) NetworkPricing() (*models.Network, error) {
 	cpricing, err := cp.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -261,14 +263,14 @@ func (cp *CustomProvider) NetworkPricing() (*Network, error) {
 		return nil, err
 	}
 
-	return &Network{
+	return &models.Network{
 		ZoneNetworkEgressCost:     znec,
 		RegionNetworkEgressCost:   rnec,
 		InternetNetworkEgressCost: inec,
 	}, nil
 }
 
-func (cp *CustomProvider) LoadBalancerPricing() (*LoadBalancer, error) {
+func (cp *CustomProvider) LoadBalancerPricing() (*models.LoadBalancer, error) {
 	cpricing, err := cp.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -294,12 +296,12 @@ func (cp *CustomProvider) LoadBalancerPricing() (*LoadBalancer, error) {
 	} else {
 		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
 	}
-	return &LoadBalancer{
+	return &models.LoadBalancer{
 		Cost: totalCost,
 	}, nil
 }
 
-func (*CustomProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (*CustomProvider) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	return &awsPVKey{
 		Labels:                 pv.Labels,
 		StorageClassName:       pv.Spec.StorageClassName,
@@ -330,14 +332,14 @@ func (cpk *customProviderKey) Features() string {
 	return "default" // TODO: multiple custom pricing support.
 }
 
-func (cp *CustomProvider) ServiceAccountStatus() *ServiceAccountStatus {
-	return &ServiceAccountStatus{
-		Checks: []*ServiceAccountCheck{},
+func (cp *CustomProvider) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return &models.ServiceAccountStatus{
+		Checks: []*models.ServiceAccountCheck{},
 	}
 }
 
-func (cp *CustomProvider) PricingSourceStatus() map[string]*PricingSource {
-	return make(map[string]*PricingSource)
+func (cp *CustomProvider) PricingSourceStatus() map[string]*models.PricingSource {
+	return make(map[string]*models.PricingSource)
 }
 
 func (cp *CustomProvider) CombinedDiscountForNode(instanceType string, isPreemptible bool, defaultDiscount, negotiatedDiscount float64) float64 {

pkg/cloud/gcpprovider.go

@@ -14,6 +14,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/kubecost"
 
 	"github.com/opencost/opencost/pkg/clustercache"
@@ -153,7 +155,7 @@ func (gcp *GCP) GetLocalStorageQuery(window, offset time.Duration, rate bool, us
 	return fmt.Sprintf(fmtQuery, baseMetric, fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
 }
 
-func (gcp *GCP) GetConfig() (*CustomPricing, error) {
+func (gcp *GCP) GetConfig() (*models.CustomPricing, error) {
 	c, err := gcp.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -168,7 +170,7 @@ func (gcp *GCP) GetConfig() (*CustomPricing, error) {
 		c.CurrencyCode = "USD"
 	}
 	if c.ShareTenancyCosts == "" {
-		c.ShareTenancyCosts = defaultShareTenancyCost
+		c.ShareTenancyCosts = models.DefaultShareTenancyCost
 	}
 	return c, nil
 }
@@ -210,7 +212,7 @@ func (*GCP) loadGCPAuthSecret() {
 		return
 	}
 
-	exists, err := fileutil.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(models.AuthSecretPath)
 	if !exists || err != nil {
 		errMessage := "Secret does not exist"
 		if err != nil {
@@ -221,7 +223,7 @@ func (*GCP) loadGCPAuthSecret() {
 		return
 	}
 
-	result, err := os.ReadFile(authSecretPath)
+	result, err := os.ReadFile(models.AuthSecretPath)
 	if err != nil {
 		log.Warnf("Failed to load auth secret, or was not mounted: %s", err.Error())
 		return
@@ -233,12 +235,12 @@ func (*GCP) loadGCPAuthSecret() {
 	}
 }
 
-func (gcp *GCP) UpdateConfigFromConfigMap(a map[string]string) (*CustomPricing, error) {
+func (gcp *GCP) UpdateConfigFromConfigMap(a map[string]string) (*models.CustomPricing, error) {
 	return gcp.Config.UpdateFromMap(a)
 }
 
-func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
-	return gcp.Config.Update(func(c *CustomPricing) error {
+func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
+	return gcp.Config.Update(func(c *models.CustomPricing) error {
 		if updateType == BigqueryUpdateType {
 			a := BigQueryConfig{}
 			err := json.NewDecoder(r).Decode(&a)
@@ -285,10 +287,10 @@ func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 				return err
 			}
 			for k, v := range a {
-				kUpper := toTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
+				kUpper := utils.ToTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
 				vstr, ok := v.(string)
 				if ok {
-					err := SetCustomPricingField(c, kUpper, vstr)
+					err := models.SetCustomPricingField(c, kUpper, vstr)
 					if err != nil {
 						return err
 					}
@@ -299,7 +301,7 @@ func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 		}
 
 		if env.IsRemoteEnabled() {
-			err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
+			err := utils.UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
 			if err != nil {
 				return err
 			}
@@ -444,7 +446,7 @@ func (gcp *GCP) isDiskOrphaned(disk *compute.Disk) (bool, error) {
 	return true, nil
 }
 
-func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
+func (gcp *GCP) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	disks, err := gcp.getAllDisks()
 	if err != nil {
 		return nil, err
@@ -455,7 +457,7 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 		return nil, err
 	}
 
-	var orphanedResources []OrphanedResource
+	var orphanedResources []models.OrphanedResource
 
 	for _, diskList := range disks.Items {
 		if len(diskList.Disks) == 0 {
@@ -488,7 +490,7 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 					zone = ""
 				}
 
-				or := OrphanedResource{
+				or := models.OrphanedResource{
 					Kind:        "disk",
 					Region:      zone,
 					Description: desc,
@@ -518,7 +520,7 @@ func (gcp *GCP) GetOrphanedResources() ([]OrphanedResource, error) {
 					region = ""
 				}
 
-				or := OrphanedResource{
+				or := models.OrphanedResource{
 					Kind:   "address",
 					Region: region,
 					Description: map[string]string{
@@ -572,8 +574,8 @@ type GCPPricing struct {
 	ServiceRegions      []string         `json:"serviceRegions"`
 	PricingInfo         []*PricingInfo   `json:"pricingInfo"`
 	ServiceProviderName string           `json:"serviceProviderName"`
-	Node                *Node            `json:"node"`
-	PV                  *PV              `json:"pv"`
+	Node                *models.Node     `json:"node"`
+	PV                  *models.PV       `json:"pv"`
 }
 
 // PricingInfo contains metadata about a cost.
@@ -615,7 +617,7 @@ type GCPResourceInfo struct {
 	UsageType          string `json:"usageType"`
 }
 
-func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[string]PVKey) (map[string]*GCPPricing, string, error) {
+func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys map[string]models.PVKey) (map[string]*GCPPricing, string, error) {
 	gcpPricingList := make(map[string]*GCPPricing)
 	var nextPageToken string
 	dec := json.NewDecoder(r)
@@ -656,7 +658,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						region := sr
 						candidateKey := region + "," + "ssd"
 						if _, ok := pvKeys[candidateKey]; ok {
-							product.PV = &PV{
+							product.PV = &models.PV{
 								Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 							}
 							gcpPricingList[candidateKey] = product
@@ -678,7 +680,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						region := sr
 						candidateKey := region + "," + "ssd" + "," + "regional"
 						if _, ok := pvKeys[candidateKey]; ok {
-							product.PV = &PV{
+							product.PV = &models.PV{
 								Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 							}
 							gcpPricingList[candidateKey] = product
@@ -699,7 +701,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						region := sr
 						candidateKey := region + "," + "pdstandard"
 						if _, ok := pvKeys[candidateKey]; ok {
-							product.PV = &PV{
+							product.PV = &models.PV{
 								Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 							}
 							gcpPricingList[candidateKey] = product
@@ -720,7 +722,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 						region := sr
 						candidateKey := region + "," + "pdstandard" + "," + "regional"
 						if _, ok := pvKeys[candidateKey]; ok {
-							product.PV = &PV{
+							product.PV = &models.PV{
 								Cost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 							}
 							gcpPricingList[candidateKey] = product
@@ -838,7 +840,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 										pl.Node.GPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 										pl.Node.GPU = "1"
 									} else {
-										product.Node = &Node{
+										product.Node = &models.Node{
 											GPUName: gpuType,
 											GPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 											GPU:     "1",
@@ -880,7 +882,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									gcpPricingList[candidateKey].Node.RAMCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
 									product = &GCPPricing{}
-									product.Node = &Node{
+									product.Node = &models.Node{
 										RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
@@ -896,7 +898,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 								} else {
 									log.Infof("Adding RAM %f for %s", hourlyPrice, candidateKeyGPU)
 									product = &GCPPricing{}
-									product.Node = &Node{
+									product.Node = &models.Node{
 										RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
@@ -912,7 +914,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									gcpPricingList[candidateKey].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
 									product = &GCPPricing{}
-									product.Node = &Node{
+									product.Node = &models.Node{
 										VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
@@ -926,7 +928,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									gcpPricingList[candidateKeyGPU].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
 									product = &GCPPricing{}
-									product.Node = &Node{
+									product.Node = &models.Node{
 										VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
@@ -959,7 +961,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 	return gcpPricingList, nextPageToken, nil
 }
 
-func (gcp *GCP) parsePages(inputKeys map[string]Key, pvKeys map[string]PVKey) (map[string]*GCPPricing, error) {
+func (gcp *GCP) parsePages(inputKeys map[string]models.Key, pvKeys map[string]models.PVKey) (map[string]*GCPPricing, error) {
 	var pages []map[string]*GCPPricing
 	c, err := gcp.GetConfig()
 	if err != nil {
@@ -1044,7 +1046,7 @@ func (gcp *GCP) DownloadPricingData() error {
 	gcp.BillingDataDataset = c.BillingDataDataset
 
 	nodeList := gcp.Clientset.GetAllNodes()
-	inputkeys := make(map[string]Key)
+	inputkeys := make(map[string]models.Key)
 
 	defaultRegion := "" // Sometimes, PVs may be missing the region label. In that case assume that they are in the same region as the nodes
 	for _, n := range nodeList {
@@ -1073,7 +1075,7 @@ func (gcp *GCP) DownloadPricingData() error {
 		}
 	}
 
-	pvkeys := make(map[string]PVKey)
+	pvkeys := make(map[string]models.PVKey)
 	for _, pv := range pvList {
 		params, ok := storageClassMap[pv.Spec.StorageClassName]
 		if !ok {
@@ -1107,19 +1109,19 @@ func (gcp *GCP) DownloadPricingData() error {
 	return nil
 }
 
-func (gcp *GCP) PVPricing(pvk PVKey) (*PV, error) {
+func (gcp *GCP) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	gcp.DownloadPricingDataLock.RLock()
 	defer gcp.DownloadPricingDataLock.RUnlock()
 	pricing, ok := gcp.Pricing[pvk.Features()]
 	if !ok {
 		log.Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
-		return &PV{}, nil
+		return &models.PV{}, nil
 	}
 	return pricing.PV, nil
 }
 
 // Stubbed NetworkPricing for GCP. Pull directly from gcp.json for now
-func (gcp *GCP) NetworkPricing() (*Network, error) {
+func (gcp *GCP) NetworkPricing() (*models.Network, error) {
 	cpricing, err := gcp.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -1137,14 +1139,14 @@ func (gcp *GCP) NetworkPricing() (*Network, error) {
 		return nil, err
 	}
 
-	return &Network{
+	return &models.Network{
 		ZoneNetworkEgressCost:     znec,
 		RegionNetworkEgressCost:   rnec,
 		InternetNetworkEgressCost: inec,
 	}, nil
 }
 
-func (gcp *GCP) LoadBalancerPricing() (*LoadBalancer, error) {
+func (gcp *GCP) LoadBalancerPricing() (*models.LoadBalancer, error) {
 	fffrc := 0.025
 	afrc := 0.010
 	lbidc := 0.008
@@ -1158,7 +1160,7 @@ func (gcp *GCP) LoadBalancerPricing() (*LoadBalancer, error) {
 	} else {
 		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
 	}
-	return &LoadBalancer{
+	return &models.LoadBalancer{
 		Cost: totalCost,
 	}, nil
 }
@@ -1218,7 +1220,7 @@ var gcpReservedInstancePlans map[string]*GCPReservedInstancePlan = map[string]*G
 	},
 }
 
-func (gcp *GCP) ApplyReservedInstancePricing(nodes map[string]*Node) {
+func (gcp *GCP) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
 	numReserved := len(gcp.ReservedInstances)
 
 	// Early return if no reserved instance data loaded
@@ -1276,7 +1278,7 @@ func (gcp *GCP) ApplyReservedInstancePricing(nodes map[string]*Node) {
 			continue
 		}
 
-		node.Reserved = &ReservedInstanceData{
+		node.Reserved = &models.ReservedInstanceData{
 			ReservedCPU: 0,
 			ReservedRAM: 0,
 		}
@@ -1402,7 +1404,7 @@ func (key *pvKey) GetStorageClass() string {
 	return key.StorageClass
 }
 
-func (gcp *GCP) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (gcp *GCP) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	providerID := ""
 	if pv.Spec.GCEPersistentDisk != nil {
 		providerID = pv.Spec.GCEPersistentDisk.PDName
@@ -1441,7 +1443,7 @@ type gcpKey struct {
 	Labels map[string]string
 }
 
-func (gcp *GCP) GetKey(labels map[string]string, n *v1.Node) Key {
+func (gcp *GCP) GetKey(labels map[string]string, n *v1.Node) models.Key {
 	return &gcpKey{
 		Labels: labels,
 	}
@@ -1522,13 +1524,13 @@ func (gcp *GCP) AllNodePricing() (interface{}, error) {
 	return gcp.Pricing, nil
 }
 
-func (gcp *GCP) getPricing(key Key) (*GCPPricing, bool) {
+func (gcp *GCP) getPricing(key models.Key) (*GCPPricing, bool) {
 	gcp.DownloadPricingDataLock.RLock()
 	defer gcp.DownloadPricingDataLock.RUnlock()
 	n, ok := gcp.Pricing[key.Features()]
 	return n, ok
 }
-func (gcp *GCP) isValidPricingKey(key Key) bool {
+func (gcp *GCP) isValidPricingKey(key models.Key) bool {
 	gcp.DownloadPricingDataLock.RLock()
 	defer gcp.DownloadPricingDataLock.RUnlock()
 	_, ok := gcp.ValidPricingKeys[key.Features()]
@@ -1536,7 +1538,7 @@ func (gcp *GCP) isValidPricingKey(key Key) bool {
 }
 
 // NodePricing returns GCP pricing data for a single node
-func (gcp *GCP) NodePricing(key Key) (*Node, error) {
+func (gcp *GCP) NodePricing(key models.Key) (*models.Node, error) {
 	if n, ok := gcp.getPricing(key); ok {
 		log.Debugf("Returning pricing for node %s: %+v from SKU %s", key, n.Node, n.Name)
 		n.Node.BaseCPUPrice = gcp.BaseCPUPrice
@@ -1557,14 +1559,14 @@ func (gcp *GCP) NodePricing(key Key) (*Node, error) {
 	return nil, fmt.Errorf("Warning: no pricing data found for %s", key)
 }
 
-func (gcp *GCP) ServiceAccountStatus() *ServiceAccountStatus {
-	return &ServiceAccountStatus{
-		Checks: []*ServiceAccountCheck{},
+func (gcp *GCP) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return &models.ServiceAccountStatus{
+		Checks: []*models.ServiceAccountCheck{},
 	}
 }
 
-func (gcp *GCP) PricingSourceStatus() map[string]*PricingSource {
-	return make(map[string]*PricingSource)
+func (gcp *GCP) PricingSourceStatus() map[string]*models.PricingSource {
+	return make(map[string]*models.PricingSource)
 }
 
 func (gcp *GCP) CombinedDiscountForNode(instanceType string, isPreemptible bool, defaultDiscount, negotiatedDiscount float64) float64 {

pkg/cloud/gcpprovider_test.go

@@ -5,6 +5,8 @@ import (
 	"io/ioutil"
 	"reflect"
 	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/models"
 )
 
 func TestParseGCPInstanceTypeLabel(t *testing.T) {
@@ -282,7 +284,7 @@ func TestParsePage(t *testing.T) {
 
 	testGcp := &GCP{}
 
-	inputKeys := map[string]Key{
+	inputKeys := map[string]models.Key{
 		"us-central1,a2highgpu,ondemand,gpu": &gcpKey{
 			Labels: map[string]string{
 				"node.kubernetes.io/instance-type": "a2-highgpu-1g",
@@ -293,7 +295,7 @@ func TestParsePage(t *testing.T) {
 		},
 	}
 
-	pvKeys := map[string]PVKey{}
+	pvKeys := map[string]models.PVKey{}
 
 	actualPrices, token, err := testGcp.parsePage(reader, inputKeys, pvKeys)
 	if err != nil {
@@ -342,7 +344,7 @@ func TestParsePage(t *testing.T) {
 				},
 			},
 			ServiceProviderName: "Google",
-			Node: &Node{
+			Node: &models.Node{
 				VCPUCost:         "0.031611",
 				RAMCost:          "0.004237",
 				UsesBaseCPUPrice: false,
@@ -352,7 +354,7 @@ func TestParsePage(t *testing.T) {
 			},
 		},
 		"us-central1,a2highgpu,ondemand": &GCPPricing{
-			Node: &Node{
+			Node: &models.Node{
 				VCPUCost:         "0.031611",
 				RAMCost:          "0.004237",
 				UsesBaseCPUPrice: false,

pkg/cloud/models/models.go

@@ -0,0 +1,301 @@
+package models
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/microcosm-cc/bluemonday"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/opencost/opencost/pkg/config"
+	"github.com/opencost/opencost/pkg/log"
+)
+
+var (
+	sanitizePolicy = bluemonday.UGCPolicy()
+)
+
+const (
+	AuthSecretPath          = "/var/secrets/service-key.json"
+	StorageConfigSecretPath = "/var/azure-storage-config/azure-storage-config.json"
+	DefaultShareTenancyCost = "true"
+)
+
+// ReservedInstanceData keeps record of resources on a node should be
+// priced at reserved rates
+type ReservedInstanceData struct {
+	ReservedCPU int64   `json:"reservedCPU"`
+	ReservedRAM int64   `json:"reservedRAM"`
+	CPUCost     float64 `json:"CPUHourlyCost"`
+	RAMCost     float64 `json:"RAMHourlyCost"`
+}
+
+// Node is the interface by which the provider and cost model communicate Node prices.
+// The provider will best-effort try to fill out this struct.
+type Node struct {
+	Cost             string                `json:"hourlyCost"`
+	VCPU             string                `json:"CPU"`
+	VCPUCost         string                `json:"CPUHourlyCost"`
+	RAM              string                `json:"RAM"`
+	RAMBytes         string                `json:"RAMBytes"`
+	RAMCost          string                `json:"RAMGBHourlyCost"`
+	Storage          string                `json:"storage"`
+	StorageCost      string                `json:"storageHourlyCost"`
+	UsesBaseCPUPrice bool                  `json:"usesDefaultPrice"`
+	BaseCPUPrice     string                `json:"baseCPUPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
+	BaseRAMPrice     string                `json:"baseRAMPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
+	BaseGPUPrice     string                `json:"baseGPUPrice"`
+	UsageType        string                `json:"usageType"`
+	GPU              string                `json:"gpu"` // GPU represents the number of GPU on the instance
+	GPUName          string                `json:"gpuName"`
+	GPUCost          string                `json:"gpuCost"`
+	InstanceType     string                `json:"instanceType,omitempty"`
+	Region           string                `json:"region,omitempty"`
+	Reserved         *ReservedInstanceData `json:"reserved,omitempty"`
+	ProviderID       string                `json:"providerID,omitempty"`
+	PricingType      PricingType           `json:"pricingType,omitempty"`
+}
+
+// IsSpot determines whether or not a Node uses spot by usage type
+func (n *Node) IsSpot() bool {
+	if n != nil {
+		return strings.Contains(n.UsageType, "spot") || strings.Contains(n.UsageType, "emptible")
+	} else {
+		return false
+	}
+}
+
+type OrphanedResource struct {
+	Kind        string            `json:"resourceKind"`
+	Region      string            `json:"region"`
+	Description map[string]string `json:"description"`
+	Size        *int64            `json:"diskSizeInGB,omitempty"`
+	DiskName    string            `json:"diskName,omitempty"`
+	Url         string            `json:"url"`
+	Address     string            `json:"ipAddress,omitempty"`
+	MonthlyCost *float64          `json:"monthlyCost"`
+}
+
+// PV is the interface by which the provider and cost model communicate PV prices.
+// The provider will best-effort try to fill out this struct.
+type PV struct {
+	Cost       string            `json:"hourlyCost"`
+	CostPerIO  string            `json:"costPerIOOperation"`
+	Class      string            `json:"storageClass"`
+	Size       string            `json:"size"`
+	Region     string            `json:"region"`
+	ProviderID string            `json:"providerID,omitempty"`
+	Parameters map[string]string `json:"parameters"`
+}
+
+// Key represents a way for nodes to match between the k8s API and a pricing API
+type Key interface {
+	ID() string       // ID represents an exact match
+	Features() string // Features are a comma separated string of node metadata that could match pricing
+	GPUType() string  // GPUType returns "" if no GPU exists or GPUs, but the name of the GPU otherwise
+	GPUCount() int    // GPUCount returns 0 if no GPU exists or GPUs, but the number of attached GPUs otherwise
+}
+
+type PVKey interface {
+	Features() string
+	GetStorageClass() string
+	ID() string
+}
+
+// OutOfClusterAllocation represents a cloud provider cost not associated with kubernetes
+type OutOfClusterAllocation struct {
+	Aggregator  string  `json:"aggregator"`
+	Environment string  `json:"environment"`
+	Service     string  `json:"service"`
+	Cost        float64 `json:"cost"`
+	Cluster     string  `json:"cluster"`
+}
+
+type CustomPricing struct {
+	Provider    string `json:"provider"`
+	Description string `json:"description"`
+	// CPU a string-encoded float describing cost per core-hour of CPU.
+	CPU string `json:"CPU"`
+	// CPU a string-encoded float describing cost per core-hour of CPU for spot
+	// nodes.
+	SpotCPU string `json:"spotCPU"`
+	// RAM a string-encoded float describing cost per GiB-hour of RAM/memory.
+	RAM string `json:"RAM"`
+	// SpotRAM a string-encoded float describing cost per GiB-hour of RAM/memory
+	// for spot nodes.
+	SpotRAM string `json:"spotRAM"`
+	GPU     string `json:"GPU"`
+	SpotGPU string `json:"spotGPU"`
+	// Storage is a string-encoded float describing cost per GB-hour of storage
+	// (e.g. PV, disk) resources.
+	Storage                      string `json:"storage"`
+	ZoneNetworkEgress            string `json:"zoneNetworkEgress"`
+	RegionNetworkEgress          string `json:"regionNetworkEgress"`
+	InternetNetworkEgress        string `json:"internetNetworkEgress"`
+	FirstFiveForwardingRulesCost string `json:"firstFiveForwardingRulesCost"`
+	AdditionalForwardingRuleCost string `json:"additionalForwardingRuleCost"`
+	LBIngressDataCost            string `json:"LBIngressDataCost"`
+	SpotLabel                    string `json:"spotLabel,omitempty"`
+	SpotLabelValue               string `json:"spotLabelValue,omitempty"`
+	GpuLabel                     string `json:"gpuLabel,omitempty"`
+	GpuLabelValue                string `json:"gpuLabelValue,omitempty"`
+	ServiceKeyName               string `json:"awsServiceKeyName,omitempty"`
+	ServiceKeySecret             string `json:"awsServiceKeySecret,omitempty"`
+	AlibabaServiceKeyName        string `json:"alibabaServiceKeyName,omitempty"`
+	AlibabaServiceKeySecret      string `json:"alibabaServiceKeySecret,omitempty"`
+	AlibabaClusterRegion         string `json:"alibabaClusterRegion,omitempty"`
+	SpotDataRegion               string `json:"awsSpotDataRegion,omitempty"`
+	SpotDataBucket               string `json:"awsSpotDataBucket,omitempty"`
+	SpotDataPrefix               string `json:"awsSpotDataPrefix,omitempty"`
+	ProjectID                    string `json:"projectID,omitempty"`
+	AthenaProjectID              string `json:"athenaProjectID,omitempty"`
+	AthenaBucketName             string `json:"athenaBucketName"`
+	AthenaRegion                 string `json:"athenaRegion"`
+	AthenaDatabase               string `json:"athenaDatabase"`
+	AthenaTable                  string `json:"athenaTable"`
+	AthenaWorkgroup              string `json:"athenaWorkgroup"`
+	MasterPayerARN               string `json:"masterPayerARN"`
+	BillingDataDataset           string `json:"billingDataDataset,omitempty"`
+	CustomPricesEnabled          string `json:"customPricesEnabled"`
+	DefaultIdle                  string `json:"defaultIdle"`
+	AzureSubscriptionID          string `json:"azureSubscriptionID"`
+	AzureClientID                string `json:"azureClientID"`
+	AzureClientSecret            string `json:"azureClientSecret"`
+	AzureTenantID                string `json:"azureTenantID"`
+	AzureBillingRegion           string `json:"azureBillingRegion"`
+	AzureBillingAccount          string `json:"azureBillingAccount"`
+	AzureOfferDurableID          string `json:"azureOfferDurableID"`
+	AzureStorageSubscriptionID   string `json:"azureStorageSubscriptionID"`
+	AzureStorageAccount          string `json:"azureStorageAccount"`
+	AzureStorageAccessKey        string `json:"azureStorageAccessKey"`
+	AzureStorageContainer        string `json:"azureStorageContainer"`
+	AzureContainerPath           string `json:"azureContainerPath"`
+	AzureCloud                   string `json:"azureCloud"`
+	CurrencyCode                 string `json:"currencyCode"`
+	Discount                     string `json:"discount"`
+	NegotiatedDiscount           string `json:"negotiatedDiscount"`
+	SharedOverhead               string `json:"sharedOverhead"`
+	ClusterName                  string `json:"clusterName"`
+	ClusterAccountID             string `json:"clusterAccount,omitempty"`
+	SharedNamespaces             string `json:"sharedNamespaces"`
+	SharedLabelNames             string `json:"sharedLabelNames"`
+	SharedLabelValues            string `json:"sharedLabelValues"`
+	ShareTenancyCosts            string `json:"shareTenancyCosts"` // TODO clean up configuration so we can use a type other that string (this should be a bool, but the app panics if it's not a string)
+	ReadOnly                     string `json:"readOnly"`
+	EditorAccess                 string `json:"editorAccess"`
+	KubecostToken                string `json:"kubecostToken"`
+	GoogleAnalyticsTag           string `json:"googleAnalyticsTag"`
+	ExcludeProviderID            string `json:"excludeProviderID"`
+	DefaultLBPrice               string `json:"defaultLBPrice"`
+}
+
+// GetSharedOverheadCostPerMonth parses and returns a float64 representation
+// of the configured monthly shared overhead cost. If the string version cannot
+// be parsed into a float, an error is logged and 0.0 is returned.
+func (cp *CustomPricing) GetSharedOverheadCostPerMonth() float64 {
+	// Empty string should be interpreted as "no cost", i.e. 0.0
+	if cp.SharedOverhead == "" {
+		return 0.0
+	}
+
+	// Attempt to parse, but log and return 0.0 if that fails.
+	sharedCostPerMonth, err := strconv.ParseFloat(cp.SharedOverhead, 64)
+	if err != nil {
+		log.Errorf("SharedOverhead: failed to parse shared overhead \"%s\": %s", cp.SharedOverhead, err)
+		return 0.0
+	}
+
+	return sharedCostPerMonth
+}
+
+func SetCustomPricingField(obj *CustomPricing, name string, value string) error {
+
+	structValue := reflect.ValueOf(obj).Elem()
+	structFieldValue := structValue.FieldByName(name)
+
+	if !structFieldValue.IsValid() {
+		return fmt.Errorf("No such field: %s in obj", name)
+	}
+
+	if !structFieldValue.CanSet() {
+		return fmt.Errorf("Cannot set %s field value", name)
+	}
+
+	structFieldType := structFieldValue.Type()
+	value = sanitizePolicy.Sanitize(value)
+	val := reflect.ValueOf(value)
+	if structFieldType != val.Type() {
+		return fmt.Errorf("Provided value type didn't match custom pricing field type")
+	}
+
+	structFieldValue.Set(val)
+	return nil
+}
+
+type PricingSources struct {
+	PricingSources map[string]*PricingSource
+}
+
+type PricingSource struct {
+	Name      string `json:"name"`
+	Enabled   bool   `json:"enabled"`
+	Available bool   `json:"available"`
+	Error     string `json:"error"`
+}
+
+type PricingType string
+
+const (
+	Api           PricingType = "api"
+	Spot          PricingType = "spot"
+	Reserved      PricingType = "reserved"
+	SavingsPlan   PricingType = "savingsPlan"
+	CsvExact      PricingType = "csvExact"
+	CsvClass      PricingType = "csvClass"
+	DefaultPrices PricingType = "defaultPrices"
+)
+
+type PricingMatchMetadata struct {
+	TotalNodes        int                 `json:"TotalNodes"`
+	PricingTypeCounts map[PricingType]int `json:"PricingType"`
+}
+
+// Provider represents a k8s provider.
+type Provider interface {
+	ClusterInfo() (map[string]string, error)
+	GetAddresses() ([]byte, error)
+	GetDisks() ([]byte, error)
+	GetOrphanedResources() ([]OrphanedResource, error)
+	NodePricing(Key) (*Node, error)
+	PVPricing(PVKey) (*PV, error)
+	NetworkPricing() (*Network, error)           // TODO: add key interface arg for dynamic price fetching
+	LoadBalancerPricing() (*LoadBalancer, error) // TODO: add key interface arg for dynamic price fetching
+	AllNodePricing() (interface{}, error)
+	DownloadPricingData() error
+	GetKey(map[string]string, *v1.Node) Key
+	GetPVKey(*v1.PersistentVolume, map[string]string, string) PVKey
+	UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error)
+	UpdateConfigFromConfigMap(map[string]string) (*CustomPricing, error)
+	GetConfig() (*CustomPricing, error)
+	GetManagementPlatform() (string, error)
+	GetLocalStorageQuery(time.Duration, time.Duration, bool, bool) string
+	ApplyReservedInstancePricing(map[string]*Node)
+	ServiceAccountStatus() *ServiceAccountStatus
+	PricingSourceStatus() map[string]*PricingSource
+	ClusterManagementPricing() (string, float64, error)
+	CombinedDiscountForNode(string, bool, float64, float64) float64
+	Regions() []string
+	PricingSourceSummary() interface{}
+}
+
+// ProviderConfig describes config storage common to all providers.
+type ProviderConfig interface {
+	ConfigFileManager() *config.ConfigFileManager
+	GetCustomPricingData() (*CustomPricing, error)
+	Update(func(*CustomPricing) error) (*CustomPricing, error)
+	UpdateFromMap(map[string]string) (*CustomPricing, error)
+}

pkg/cloud/models/network.go

@@ -0,0 +1,21 @@
+package models
+
+// TODO: used for dynamic cloud provider price fetching.
+// determine what identifies a load balancer in the json returned from the cloud provider pricing API call
+// type LBKey interface {
+// }
+
+// Network is the interface by which the provider and cost model communicate network egress prices.
+// The provider will best-effort try to fill out this struct.
+type Network struct {
+	ZoneNetworkEgressCost     float64
+	RegionNetworkEgressCost   float64
+	InternetNetworkEgressCost float64
+}
+
+// LoadBalancer is the interface by which the provider and cost model communicate LoadBalancer prices.
+// The provider will best-effort try to fill out this struct.
+type LoadBalancer struct {
+	IngressIPAddresses []string `json:"IngressIPAddresses"`
+	Cost               float64  `json:"hourlyCost"`
+}

pkg/cloud/models/serviceaccounts.go

@@ -0,0 +1,45 @@
+package models
+
+import "sync"
+
+type ServiceAccountStatus struct {
+	Checks []*ServiceAccountCheck `json:"checks"`
+}
+
+// ServiceAccountChecks is a thread safe map for holding ServiceAccountCheck objects
+type ServiceAccountChecks struct {
+	sync.RWMutex
+	serviceAccountChecks map[string]*ServiceAccountCheck
+}
+
+// NewServiceAccountChecks initialize ServiceAccountChecks
+func NewServiceAccountChecks() *ServiceAccountChecks {
+	return &ServiceAccountChecks{
+		serviceAccountChecks: make(map[string]*ServiceAccountCheck),
+	}
+}
+
+func (sac *ServiceAccountChecks) Set(key string, check *ServiceAccountCheck) {
+	sac.Lock()
+	defer sac.Unlock()
+	sac.serviceAccountChecks[key] = check
+}
+
+// getStatus extracts ServiceAccountCheck objects into a slice and returns them in a ServiceAccountStatus
+func (sac *ServiceAccountChecks) GetStatus() *ServiceAccountStatus {
+	sac.Lock()
+	defer sac.Unlock()
+	checks := []*ServiceAccountCheck{}
+	for _, v := range sac.serviceAccountChecks {
+		checks = append(checks, v)
+	}
+	return &ServiceAccountStatus{
+		Checks: checks,
+	}
+}
+
+type ServiceAccountCheck struct {
+	Message        string `json:"message"`
+	Status         bool   `json:"status"`
+	AdditionalInfo string `json:"additionalInfo"`
+}

pkg/cloud/provider.go

@@ -1,21 +1,15 @@
 package cloud
 
 import (
-	"database/sql"
 	"errors"
-	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"regexp"
-	"strconv"
 	"strings"
-	"sync"
 	"time"
 
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
-
+	"github.com/opencost/opencost/pkg/cloud/azure"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/kubecost"
 
 	"github.com/opencost/opencost/pkg/util"
@@ -32,331 +26,12 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
-const authSecretPath = "/var/secrets/service-key.json"
-const storageConfigSecretPath = "/var/azure-storage-config/azure-storage-config.json"
-const defaultShareTenancyCost = "true"
-
 const KarpenterCapacityTypeLabel = "karpenter.sh/capacity-type"
 const KarpenterCapacitySpotTypeValue = "spot"
 
-var toTitle = cases.Title(language.Und, cases.NoLower)
-
-var createTableStatements = []string{
-	`CREATE TABLE IF NOT EXISTS names (
-		cluster_id VARCHAR(255) NOT NULL,
-		cluster_name VARCHAR(255) NULL,
-		PRIMARY KEY (cluster_id)
-	);`,
-}
-
-// ReservedInstanceData keeps record of resources on a node should be
-// priced at reserved rates
-type ReservedInstanceData struct {
-	ReservedCPU int64   `json:"reservedCPU"`
-	ReservedRAM int64   `json:"reservedRAM"`
-	CPUCost     float64 `json:"CPUHourlyCost"`
-	RAMCost     float64 `json:"RAMHourlyCost"`
-}
-
-// Node is the interface by which the provider and cost model communicate Node prices.
-// The provider will best-effort try to fill out this struct.
-type Node struct {
-	Cost             string                `json:"hourlyCost"`
-	VCPU             string                `json:"CPU"`
-	VCPUCost         string                `json:"CPUHourlyCost"`
-	RAM              string                `json:"RAM"`
-	RAMBytes         string                `json:"RAMBytes"`
-	RAMCost          string                `json:"RAMGBHourlyCost"`
-	Storage          string                `json:"storage"`
-	StorageCost      string                `json:"storageHourlyCost"`
-	UsesBaseCPUPrice bool                  `json:"usesDefaultPrice"`
-	BaseCPUPrice     string                `json:"baseCPUPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
-	BaseRAMPrice     string                `json:"baseRAMPrice"` // Used to compute an implicit RAM GB/Hr price when RAM pricing is not provided.
-	BaseGPUPrice     string                `json:"baseGPUPrice"`
-	UsageType        string                `json:"usageType"`
-	GPU              string                `json:"gpu"` // GPU represents the number of GPU on the instance
-	GPUName          string                `json:"gpuName"`
-	GPUCost          string                `json:"gpuCost"`
-	InstanceType     string                `json:"instanceType,omitempty"`
-	Region           string                `json:"region,omitempty"`
-	Reserved         *ReservedInstanceData `json:"reserved,omitempty"`
-	ProviderID       string                `json:"providerID,omitempty"`
-	PricingType      PricingType           `json:"pricingType,omitempty"`
-}
-
-// IsSpot determines whether or not a Node uses spot by usage type
-func (n *Node) IsSpot() bool {
-	if n != nil {
-		return strings.Contains(n.UsageType, "spot") || strings.Contains(n.UsageType, "emptible")
-	} else {
-		return false
-	}
-}
-
-// LoadBalancer is the interface by which the provider and cost model communicate LoadBalancer prices.
-// The provider will best-effort try to fill out this struct.
-type LoadBalancer struct {
-	IngressIPAddresses []string `json:"IngressIPAddresses"`
-	Cost               float64  `json:"hourlyCost"`
-}
-
-// TODO: used for dynamic cloud provider price fetching.
-// determine what identifies a load balancer in the json returned from the cloud provider pricing API call
-// type LBKey interface {
-// }
-
-// Network is the interface by which the provider and cost model communicate network egress prices.
-// The provider will best-effort try to fill out this struct.
-type Network struct {
-	ZoneNetworkEgressCost     float64
-	RegionNetworkEgressCost   float64
-	InternetNetworkEgressCost float64
-}
-
-type OrphanedResource struct {
-	Kind        string            `json:"resourceKind"`
-	Region      string            `json:"region"`
-	Description map[string]string `json:"description"`
-	Size        *int64            `json:"diskSizeInGB,omitempty"`
-	DiskName    string            `json:"diskName,omitempty"`
-	Url         string            `json:"url"`
-	Address     string            `json:"ipAddress,omitempty"`
-	MonthlyCost *float64          `json:"monthlyCost"`
-}
-
-// PV is the interface by which the provider and cost model communicate PV prices.
-// The provider will best-effort try to fill out this struct.
-type PV struct {
-	Cost       string            `json:"hourlyCost"`
-	CostPerIO  string            `json:"costPerIOOperation"`
-	Class      string            `json:"storageClass"`
-	Size       string            `json:"size"`
-	Region     string            `json:"region"`
-	ProviderID string            `json:"providerID,omitempty"`
-	Parameters map[string]string `json:"parameters"`
-}
-
-// Key represents a way for nodes to match between the k8s API and a pricing API
-type Key interface {
-	ID() string       // ID represents an exact match
-	Features() string // Features are a comma separated string of node metadata that could match pricing
-	GPUType() string  // GPUType returns "" if no GPU exists or GPUs, but the name of the GPU otherwise
-	GPUCount() int    // GPUCount returns 0 if no GPU exists or GPUs, but the number of attached GPUs otherwise
-}
-
-type PVKey interface {
-	Features() string
-	GetStorageClass() string
-	ID() string
-}
-
-// OutOfClusterAllocation represents a cloud provider cost not associated with kubernetes
-type OutOfClusterAllocation struct {
-	Aggregator  string  `json:"aggregator"`
-	Environment string  `json:"environment"`
-	Service     string  `json:"service"`
-	Cost        float64 `json:"cost"`
-	Cluster     string  `json:"cluster"`
-}
-
-type CustomPricing struct {
-	Provider    string `json:"provider"`
-	Description string `json:"description"`
-	// CPU a string-encoded float describing cost per core-hour of CPU.
-	CPU string `json:"CPU"`
-	// CPU a string-encoded float describing cost per core-hour of CPU for spot
-	// nodes.
-	SpotCPU string `json:"spotCPU"`
-	// RAM a string-encoded float describing cost per GiB-hour of RAM/memory.
-	RAM string `json:"RAM"`
-	// SpotRAM a string-encoded float describing cost per GiB-hour of RAM/memory
-	// for spot nodes.
-	SpotRAM string `json:"spotRAM"`
-	GPU     string `json:"GPU"`
-	SpotGPU string `json:"spotGPU"`
-	// Storage is a string-encoded float describing cost per GB-hour of storage
-	// (e.g. PV, disk) resources.
-	Storage                      string `json:"storage"`
-	ZoneNetworkEgress            string `json:"zoneNetworkEgress"`
-	RegionNetworkEgress          string `json:"regionNetworkEgress"`
-	InternetNetworkEgress        string `json:"internetNetworkEgress"`
-	FirstFiveForwardingRulesCost string `json:"firstFiveForwardingRulesCost"`
-	AdditionalForwardingRuleCost string `json:"additionalForwardingRuleCost"`
-	LBIngressDataCost            string `json:"LBIngressDataCost"`
-	SpotLabel                    string `json:"spotLabel,omitempty"`
-	SpotLabelValue               string `json:"spotLabelValue,omitempty"`
-	GpuLabel                     string `json:"gpuLabel,omitempty"`
-	GpuLabelValue                string `json:"gpuLabelValue,omitempty"`
-	ServiceKeyName               string `json:"awsServiceKeyName,omitempty"`
-	ServiceKeySecret             string `json:"awsServiceKeySecret,omitempty"`
-	AlibabaServiceKeyName        string `json:"alibabaServiceKeyName,omitempty"`
-	AlibabaServiceKeySecret      string `json:"alibabaServiceKeySecret,omitempty"`
-	AlibabaClusterRegion         string `json:"alibabaClusterRegion,omitempty"`
-	SpotDataRegion               string `json:"awsSpotDataRegion,omitempty"`
-	SpotDataBucket               string `json:"awsSpotDataBucket,omitempty"`
-	SpotDataPrefix               string `json:"awsSpotDataPrefix,omitempty"`
-	ProjectID                    string `json:"projectID,omitempty"`
-	AthenaProjectID              string `json:"athenaProjectID,omitempty"`
-	AthenaBucketName             string `json:"athenaBucketName"`
-	AthenaRegion                 string `json:"athenaRegion"`
-	AthenaDatabase               string `json:"athenaDatabase"`
-	AthenaTable                  string `json:"athenaTable"`
-	AthenaWorkgroup              string `json:"athenaWorkgroup"`
-	MasterPayerARN               string `json:"masterPayerARN"`
-	BillingDataDataset           string `json:"billingDataDataset,omitempty"`
-	CustomPricesEnabled          string `json:"customPricesEnabled"`
-	DefaultIdle                  string `json:"defaultIdle"`
-	AzureSubscriptionID          string `json:"azureSubscriptionID"`
-	AzureClientID                string `json:"azureClientID"`
-	AzureClientSecret            string `json:"azureClientSecret"`
-	AzureTenantID                string `json:"azureTenantID"`
-	AzureBillingRegion           string `json:"azureBillingRegion"`
-	AzureBillingAccount          string `json:"azureBillingAccount"`
-	AzureOfferDurableID          string `json:"azureOfferDurableID"`
-	AzureStorageSubscriptionID   string `json:"azureStorageSubscriptionID"`
-	AzureStorageAccount          string `json:"azureStorageAccount"`
-	AzureStorageAccessKey        string `json:"azureStorageAccessKey"`
-	AzureStorageContainer        string `json:"azureStorageContainer"`
-	AzureContainerPath           string `json:"azureContainerPath"`
-	AzureCloud                   string `json:"azureCloud"`
-	CurrencyCode                 string `json:"currencyCode"`
-	Discount                     string `json:"discount"`
-	NegotiatedDiscount           string `json:"negotiatedDiscount"`
-	SharedOverhead               string `json:"sharedOverhead"`
-	ClusterName                  string `json:"clusterName"`
-	ClusterAccountID             string `json:"clusterAccount,omitempty"`
-	SharedNamespaces             string `json:"sharedNamespaces"`
-	SharedLabelNames             string `json:"sharedLabelNames"`
-	SharedLabelValues            string `json:"sharedLabelValues"`
-	ShareTenancyCosts            string `json:"shareTenancyCosts"` // TODO clean up configuration so we can use a type other that string (this should be a bool, but the app panics if it's not a string)
-	ReadOnly                     string `json:"readOnly"`
-	EditorAccess                 string `json:"editorAccess"`
-	KubecostToken                string `json:"kubecostToken"`
-	GoogleAnalyticsTag           string `json:"googleAnalyticsTag"`
-	ExcludeProviderID            string `json:"excludeProviderID"`
-	DefaultLBPrice               string `json:"defaultLBPrice"`
-}
-
-// GetSharedOverheadCostPerMonth parses and returns a float64 representation
-// of the configured monthly shared overhead cost. If the string version cannot
-// be parsed into a float, an error is logged and 0.0 is returned.
-func (cp *CustomPricing) GetSharedOverheadCostPerMonth() float64 {
-	// Empty string should be interpreted as "no cost", i.e. 0.0
-	if cp.SharedOverhead == "" {
-		return 0.0
-	}
-
-	// Attempt to parse, but log and return 0.0 if that fails.
-	sharedCostPerMonth, err := strconv.ParseFloat(cp.SharedOverhead, 64)
-	if err != nil {
-		log.Errorf("SharedOverhead: failed to parse shared overhead \"%s\": %s", cp.SharedOverhead, err)
-		return 0.0
-	}
-
-	return sharedCostPerMonth
-}
-
-type ServiceAccountStatus struct {
-	Checks []*ServiceAccountCheck `json:"checks"`
-}
-
-// ServiceAccountChecks is a thread safe map for holding ServiceAccountCheck objects
-type ServiceAccountChecks struct {
-	sync.RWMutex
-	serviceAccountChecks map[string]*ServiceAccountCheck
-}
-
-// NewServiceAccountChecks initialize ServiceAccountChecks
-func NewServiceAccountChecks() *ServiceAccountChecks {
-	return &ServiceAccountChecks{
-		serviceAccountChecks: make(map[string]*ServiceAccountCheck),
-	}
-}
-
-func (sac *ServiceAccountChecks) set(key string, check *ServiceAccountCheck) {
-	sac.Lock()
-	defer sac.Unlock()
-	sac.serviceAccountChecks[key] = check
-}
-
-// getStatus extracts ServiceAccountCheck objects into a slice and returns them in a ServiceAccountStatus
-func (sac *ServiceAccountChecks) getStatus() *ServiceAccountStatus {
-	sac.Lock()
-	defer sac.Unlock()
-	checks := []*ServiceAccountCheck{}
-	for _, v := range sac.serviceAccountChecks {
-		checks = append(checks, v)
-	}
-	return &ServiceAccountStatus{
-		Checks: checks,
-	}
-}
-
-type ServiceAccountCheck struct {
-	Message        string `json:"message"`
-	Status         bool   `json:"status"`
-	AdditionalInfo string `json:"additionalInfo"`
-}
-
-type PricingSources struct {
-	PricingSources map[string]*PricingSource
-}
-
-type PricingSource struct {
-	Name      string `json:"name"`
-	Enabled   bool   `json:"enabled"`
-	Available bool   `json:"available"`
-	Error     string `json:"error"`
-}
-
-type PricingType string
-
-const (
-	Api           PricingType = "api"
-	Spot          PricingType = "spot"
-	Reserved      PricingType = "reserved"
-	SavingsPlan   PricingType = "savingsPlan"
-	CsvExact      PricingType = "csvExact"
-	CsvClass      PricingType = "csvClass"
-	DefaultPrices PricingType = "defaultPrices"
-)
-
-type PricingMatchMetadata struct {
-	TotalNodes        int                 `json:"TotalNodes"`
-	PricingTypeCounts map[PricingType]int `json:"PricingType"`
-}
-
-// Provider represents a k8s provider.
-type Provider interface {
-	ClusterInfo() (map[string]string, error)
-	GetAddresses() ([]byte, error)
-	GetDisks() ([]byte, error)
-	GetOrphanedResources() ([]OrphanedResource, error)
-	NodePricing(Key) (*Node, error)
-	PVPricing(PVKey) (*PV, error)
-	NetworkPricing() (*Network, error)           // TODO: add key interface arg for dynamic price fetching
-	LoadBalancerPricing() (*LoadBalancer, error) // TODO: add key interface arg for dynamic price fetching
-	AllNodePricing() (interface{}, error)
-	DownloadPricingData() error
-	GetKey(map[string]string, *v1.Node) Key
-	GetPVKey(*v1.PersistentVolume, map[string]string, string) PVKey
-	UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error)
-	UpdateConfigFromConfigMap(map[string]string) (*CustomPricing, error)
-	GetConfig() (*CustomPricing, error)
-	GetManagementPlatform() (string, error)
-	GetLocalStorageQuery(time.Duration, time.Duration, bool, bool) string
-	ApplyReservedInstancePricing(map[string]*Node)
-	ServiceAccountStatus() *ServiceAccountStatus
-	PricingSourceStatus() map[string]*PricingSource
-	ClusterManagementPricing() (string, float64, error)
-	CombinedDiscountForNode(string, bool, float64, float64) float64
-	Regions() []string
-	PricingSourceSummary() interface{}
-}
-
 // ClusterName returns the name defined in cluster info, defaulting to the
 // CLUSTER_ID environment variable
-func ClusterName(p Provider) string {
+func ClusterName(p models.Provider) string {
 	info, err := p.ClusterInfo()
 	if err != nil {
 		return env.GetClusterID()
@@ -372,7 +47,7 @@ func ClusterName(p Provider) string {
 
 // CustomPricesEnabled returns the boolean equivalent of the cloup provider's custom prices flag,
 // indicating whether or not the cluster is using custom pricing.
-func CustomPricesEnabled(p Provider) bool {
+func CustomPricesEnabled(p models.Provider) bool {
 	config, err := p.GetConfig()
 	if err != nil {
 		return false
@@ -387,7 +62,7 @@ func CustomPricesEnabled(p Provider) bool {
 
 // ConfigWatcherFor returns a new ConfigWatcher instance which watches changes to the "pricing-configs"
 // configmap
-func ConfigWatcherFor(p Provider) *watcher.ConfigMapWatcher {
+func ConfigWatcherFor(p models.Provider) *watcher.ConfigMapWatcher {
 	return &watcher.ConfigMapWatcher{
 		ConfigMapName: env.GetPricingConfigmapName(),
 		WatchFunc: func(name string, data map[string]string) error {
@@ -398,7 +73,7 @@ func ConfigWatcherFor(p Provider) *watcher.ConfigMapWatcher {
 }
 
 // AllocateIdleByDefault returns true if the application settings specify to allocate idle by default
-func AllocateIdleByDefault(p Provider) bool {
+func AllocateIdleByDefault(p models.Provider) bool {
 	config, err := p.GetConfig()
 	if err != nil {
 		return false
@@ -408,7 +83,7 @@ func AllocateIdleByDefault(p Provider) bool {
 }
 
 // SharedNamespace returns a list of names of shared namespaces, as defined in the application settings
-func SharedNamespaces(p Provider) []string {
+func SharedNamespaces(p models.Provider) []string {
 	namespaces := []string{}
 
 	config, err := p.GetConfig()
@@ -429,7 +104,7 @@ func SharedNamespaces(p Provider) []string {
 // SharedLabel returns the configured set of shared labels as a parallel tuple of keys to values; e.g.
 // for app:kubecost,type:staging this returns (["app", "type"], ["kubecost", "staging"]) in order to
 // match the signature of the NewSharedResourceInfo
-func SharedLabels(p Provider) ([]string, []string) {
+func SharedLabels(p models.Provider) ([]string, []string) {
 	names := []string{}
 	values := []string{}
 
@@ -459,7 +134,7 @@ func SharedLabels(p Provider) ([]string, []string) {
 
 // ShareTenancyCosts returns true if the application settings specify to share
 // tenancy costs by default.
-func ShareTenancyCosts(p Provider) bool {
+func ShareTenancyCosts(p models.Provider) bool {
 	config, err := p.GetConfig()
 	if err != nil {
 		return false
@@ -469,7 +144,7 @@ func ShareTenancyCosts(p Provider) bool {
 }
 
 // NewProvider looks at the nodespec or provider metadata server to decide which provider to instantiate.
-func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.ConfigFileManager) (Provider, error) {
+func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.ConfigFileManager) (models.Provider, error) {
 	nodes := cache.GetAllNodes()
 	if len(nodes) == 0 {
 		log.Infof("Could not locate any nodes for cluster.") // valid in ETL readonly mode
@@ -528,16 +203,16 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 			Config:               NewProviderConfig(config, cp.configFileName),
 			clusterRegion:        cp.region,
 			clusterAccountID:     cp.accountID,
-			serviceAccountChecks: NewServiceAccountChecks(),
+			serviceAccountChecks: models.NewServiceAccountChecks(),
 		}, nil
 	case kubecost.AzureProvider:
 		log.Info("Found ProviderID starting with \"azure\", using Azure Provider")
-		return &Azure{
+		return &azure.Azure{
 			Clientset:            cache,
 			Config:               NewProviderConfig(config, cp.configFileName),
-			clusterRegion:        cp.region,
-			clusterAccountID:     cp.accountID,
-			serviceAccountChecks: NewServiceAccountChecks(),
+			ClusterRegion:        cp.region,
+			ClusterAccountID:     cp.accountID,
+			ServiceAccountChecks: models.NewServiceAccountChecks(),
 		}, nil
 	case kubecost.AlibabaProvider:
 		log.Info("Found ProviderID starting with \"alibaba\", using Alibaba Cloud Provider")
@@ -546,7 +221,7 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 			Config:               NewProviderConfig(config, cp.configFileName),
 			clusterRegion:        cp.region,
 			clusterAccountId:     cp.accountID,
-			serviceAccountChecks: NewServiceAccountChecks(),
+			serviceAccountChecks: models.NewServiceAccountChecks(),
 		}, nil
 	case kubecost.ScalewayProvider:
 		log.Info("Found ProviderID starting with \"scaleway\", using Scaleway Provider")
@@ -597,7 +272,7 @@ func getClusterProperties(node *v1.Node) clusterProperties {
 	} else if strings.HasPrefix(providerID, "azure") {
 		cp.provider = kubecost.AzureProvider
 		cp.configFileName = "azure.json"
-		cp.accountID = parseAzureSubscriptionID(providerID)
+		cp.accountID = azure.ParseAzureSubscriptionID(providerID)
 	} else if strings.HasPrefix(providerID, "scaleway") { // the scaleway provider ID looks like scaleway://instance/<instance_id>
 		cp.provider = kubecost.ScalewayProvider
 		cp.configFileName = "scaleway.json"
@@ -612,95 +287,6 @@ func getClusterProperties(node *v1.Node) clusterProperties {
 	return cp
 }
 
-func UpdateClusterMeta(cluster_id, cluster_name string) error {
-	pw := env.GetRemotePW()
-	address := env.GetSQLAddress()
-	connStr := fmt.Sprintf("postgres://postgres:%s@%s:5432?sslmode=disable", pw, address)
-	db, err := sql.Open("postgres", connStr)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-	updateStmt := `UPDATE names SET cluster_name = $1 WHERE cluster_id = $2;`
-	_, err = db.Exec(updateStmt, cluster_name, cluster_id)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func CreateClusterMeta(cluster_id, cluster_name string) error {
-	pw := env.GetRemotePW()
-	address := env.GetSQLAddress()
-	connStr := fmt.Sprintf("postgres://postgres:%s@%s:5432?sslmode=disable", pw, address)
-	db, err := sql.Open("postgres", connStr)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-	for _, stmt := range createTableStatements {
-		_, err := db.Exec(stmt)
-		if err != nil {
-			return err
-		}
-	}
-	insertStmt := `INSERT INTO names (cluster_id, cluster_name) VALUES ($1, $2);`
-	_, err = db.Exec(insertStmt, cluster_id, cluster_name)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func GetClusterMeta(cluster_id string) (string, string, error) {
-	pw := env.GetRemotePW()
-	address := env.GetSQLAddress()
-	connStr := fmt.Sprintf("postgres://postgres:%s@%s:5432?sslmode=disable", pw, address)
-	db, err := sql.Open("postgres", connStr)
-	if err != nil {
-		return "", "", err
-	}
-	defer db.Close()
-	query := `SELECT cluster_id, cluster_name
-	FROM names
-	WHERE cluster_id = ?`
-
-	rows, err := db.Query(query, cluster_id)
-	if err != nil {
-		return "", "", err
-	}
-	defer rows.Close()
-	var (
-		sql_cluster_id string
-		cluster_name   string
-	)
-	for rows.Next() {
-		if err := rows.Scan(&sql_cluster_id, &cluster_name); err != nil {
-			return "", "", err
-		}
-	}
-
-	return sql_cluster_id, cluster_name, nil
-}
-
-func GetOrCreateClusterMeta(cluster_id, cluster_name string) (string, string, error) {
-	id, name, err := GetClusterMeta(cluster_id)
-	if err != nil {
-		err := CreateClusterMeta(cluster_id, cluster_name)
-		if err != nil {
-			return "", "", err
-		}
-	}
-	if id == "" {
-		err := CreateClusterMeta(cluster_id, cluster_name)
-		if err != nil {
-			return "", "", err
-		}
-	}
-
-	return id, name, nil
-}
-
 var (
 	// It's of the form aws:///us-east-2a/i-0fea4fd46592d050b and we want i-0fea4fd46592d050b, if it exists
 	providerAWSRegex = regexp.MustCompile("aws://[^/]*/[^/]*/([^/]+)")

pkg/cloud/providerconfig.go

@@ -5,11 +5,11 @@ import (
 	"io/ioutil"
 	"os"
 	gopath "path"
-	"reflect"
 	"strconv"
 	"sync"
 
-	"github.com/microcosm-cc/bluemonday"
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/config"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"
@@ -18,15 +18,13 @@ import (
 
 const closedSourceConfigMount = "models/"
 
-var sanitizePolicy = bluemonday.UGCPolicy()
-
 // ProviderConfig is a utility class that provides a thread-safe configuration storage/cache for all Provider
 // implementations
 type ProviderConfig struct {
 	lock            *sync.Mutex
 	configManager   *config.ConfigFileManager
 	configFile      *config.ConfigFile
-	customPricing   *CustomPricing
+	customPricing   *models.CustomPricing
 	watcherHandleID config.HandlerID
 }
 
@@ -59,7 +57,7 @@ func (pc *ProviderConfig) onConfigFileUpdated(changeType config.ChangeType, data
 		pc.lock.Lock()
 		defer pc.lock.Unlock()
 
-		customPricing := new(CustomPricing)
+		customPricing := new(models.CustomPricing)
 		err := json.Unmarshal(data, customPricing)
 		if err != nil {
 			log.Infof("Could not decode Custom Pricing file at path %s. Using default.", pc.configFile.Path())
@@ -72,14 +70,14 @@ func (pc *ProviderConfig) onConfigFileUpdated(changeType config.ChangeType, data
 		}
 
 		if pc.customPricing.ShareTenancyCosts == "" {
-			pc.customPricing.ShareTenancyCosts = defaultShareTenancyCost
+			pc.customPricing.ShareTenancyCosts = models.DefaultShareTenancyCost
 		}
 	}
 }
 
 // Non-ThreadSafe logic to load the config file if a cache does not exist. Flag to write
 // the default config if the config file doesn't exist.
-func (pc *ProviderConfig) loadConfig(writeIfNotExists bool) (*CustomPricing, error) {
+func (pc *ProviderConfig) loadConfig(writeIfNotExists bool) (*models.CustomPricing, error) {
 	if pc.customPricing != nil {
 		return pc.customPricing, nil
 	}
@@ -130,7 +128,7 @@ func (pc *ProviderConfig) loadConfig(writeIfNotExists bool) (*CustomPricing, err
 		return DefaultPricing(), err
 	}
 
-	var customPricing CustomPricing
+	var customPricing models.CustomPricing
 	err = json.Unmarshal(byteValue, &customPricing)
 	if err != nil {
 		log.Infof("Could not decode Custom Pricing file at path %s", pc.configFile.Path())
@@ -143,14 +141,14 @@ func (pc *ProviderConfig) loadConfig(writeIfNotExists bool) (*CustomPricing, err
 	}
 
 	if pc.customPricing.ShareTenancyCosts == "" {
-		pc.customPricing.ShareTenancyCosts = defaultShareTenancyCost
+		pc.customPricing.ShareTenancyCosts = models.DefaultShareTenancyCost
 	}
 
 	return pc.customPricing, nil
 }
 
 // ThreadSafe method for retrieving the custom pricing config.
-func (pc *ProviderConfig) GetCustomPricingData() (*CustomPricing, error) {
+func (pc *ProviderConfig) GetCustomPricingData() (*models.CustomPricing, error) {
 	pc.lock.Lock()
 	defer pc.lock.Unlock()
 
@@ -166,7 +164,7 @@ func (pc *ProviderConfig) ConfigFileManager() *config.ConfigFileManager {
 
 // Allows a call to manually update the configuration while maintaining proper thread-safety
 // for read/write methods.
-func (pc *ProviderConfig) Update(updateFunc func(*CustomPricing) error) (*CustomPricing, error) {
+func (pc *ProviderConfig) Update(updateFunc func(*models.CustomPricing) error) (*models.CustomPricing, error) {
 	pc.lock.Lock()
 	defer pc.lock.Unlock()
 
@@ -198,12 +196,12 @@ func (pc *ProviderConfig) Update(updateFunc func(*CustomPricing) error) (*Custom
 }
 
 // ThreadSafe update of the config using a string map
-func (pc *ProviderConfig) UpdateFromMap(a map[string]string) (*CustomPricing, error) {
+func (pc *ProviderConfig) UpdateFromMap(a map[string]string) (*models.CustomPricing, error) {
 	// Run our Update() method using SetCustomPricingField logic
-	return pc.Update(func(c *CustomPricing) error {
+	return pc.Update(func(c *models.CustomPricing) error {
 		for k, v := range a {
 			// Just so we consistently supply / receive the same values, uppercase the first letter.
-			kUpper := toTitle.String(k)
+			kUpper := utils.ToTitle.String(k)
 			if kUpper == "CPU" || kUpper == "SpotCPU" || kUpper == "RAM" || kUpper == "SpotRAM" || kUpper == "GPU" || kUpper == "Storage" {
 				val, err := strconv.ParseFloat(v, 64)
 				if err != nil {
@@ -212,7 +210,7 @@ func (pc *ProviderConfig) UpdateFromMap(a map[string]string) (*CustomPricing, er
 				v = fmt.Sprintf("%f", val/730)
 			}
 
-			err := SetCustomPricingField(c, kUpper, v)
+			err := models.SetCustomPricingField(c, kUpper, v)
 			if err != nil {
 				return err
 			}
@@ -223,9 +221,9 @@ func (pc *ProviderConfig) UpdateFromMap(a map[string]string) (*CustomPricing, er
 }
 
 // DefaultPricing should be returned so we can do computation even if no file is supplied.
-func DefaultPricing() *CustomPricing {
+func DefaultPricing() *models.CustomPricing {
 	// https://cloud.google.com/compute/all-pricing
-	return &CustomPricing{
+	return &models.CustomPricing{
 		Provider:    "base",
 		Description: "Default prices based on GCP us-central1",
 
@@ -257,30 +255,6 @@ func DefaultPricing() *CustomPricing {
 	}
 }
 
-func SetCustomPricingField(obj *CustomPricing, name string, value string) error {
-
-	structValue := reflect.ValueOf(obj).Elem()
-	structFieldValue := structValue.FieldByName(name)
-
-	if !structFieldValue.IsValid() {
-		return fmt.Errorf("No such field: %s in obj", name)
-	}
-
-	if !structFieldValue.CanSet() {
-		return fmt.Errorf("Cannot set %s field value", name)
-	}
-
-	structFieldType := structFieldValue.Type()
-	value = sanitizePolicy.Sanitize(value)
-	val := reflect.ValueOf(value)
-	if structFieldType != val.Type() {
-		return fmt.Errorf("Provided value type didn't match custom pricing field type")
-	}
-
-	structFieldValue.Set(val)
-	return nil
-}
-
 // Returns the configuration directory concatenated with a specific config file name
 func configPathFor(filename string) string {
 	path := env.GetConfigPathWithDefault("/models/")
@@ -295,23 +269,23 @@ func filenameInConfigPath(fqfn string) string {
 
 // ReturnPricingFromConfigs is a safe function to return pricing from configs of opensource to the closed source
 // before defaulting it with the above function DefaultPricing
-func ReturnPricingFromConfigs(filename string) (*CustomPricing, error) {
+func ReturnPricingFromConfigs(filename string) (*models.CustomPricing, error) {
 	if _, err := os.Stat(closedSourceConfigMount); os.IsNotExist(err) {
-		return &CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: %s likely running in provider config in opencost itself with err: %v", closedSourceConfigMount, err)
+		return &models.CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: %s likely running in provider config in opencost itself with err: %v", closedSourceConfigMount, err)
 	}
 	providerConfigFile := gopath.Join(closedSourceConfigMount, filename)
 	if _, err := os.Stat(providerConfigFile); err != nil {
-		return &CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to find file %s with err: %v", providerConfigFile, err)
+		return &models.CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to find file %s with err: %v", providerConfigFile, err)
 	}
 	configFile, err := ioutil.ReadFile(providerConfigFile)
 	if err != nil {
-		return &CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to open file %s with err: %v", providerConfigFile, err)
+		return &models.CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to open file %s with err: %v", providerConfigFile, err)
 	}
 
-	defaultPricing := &CustomPricing{}
+	defaultPricing := &models.CustomPricing{}
 	err = json.Unmarshal(configFile, defaultPricing)
 	if err != nil {
-		return &CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to open file %s with err: %v", providerConfigFile, err)
+		return &models.CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to open file %s with err: %v", providerConfigFile, err)
 	}
 	return defaultPricing, nil
 }

pkg/cloud/scalewayprovider.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/kubecost"
 
 	"github.com/opencost/opencost/pkg/clustercache"
@@ -131,7 +133,7 @@ func (k *scalewayKey) ID() string {
 	return ""
 }
 
-func (c *Scaleway) NodePricing(key Key) (*Node, error) {
+func (c *Scaleway) NodePricing(key models.Key) (*models.Node, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()
 
@@ -139,9 +141,9 @@ func (c *Scaleway) NodePricing(key Key) (*Node, error) {
 	split := strings.Split(key.Features(), ",")
 	if pricing, ok := c.Pricing[split[0]]; ok {
 		if info, ok := pricing.NodesInfos[split[1]]; ok {
-			return &Node{
+			return &models.Node{
 				Cost:        fmt.Sprintf("%f", info.HourlyPrice),
-				PricingType: DefaultPrices,
+				PricingType: models.DefaultPrices,
 				VCPU:        fmt.Sprintf("%d", info.Ncpus),
 				RAM:         fmt.Sprintf("%d", info.RAM),
 				// This is tricky, as instances can have local volumes or not
@@ -158,24 +160,24 @@ func (c *Scaleway) NodePricing(key Key) (*Node, error) {
 	return nil, fmt.Errorf("Unable to find node pricing matching thes features `%s`", key.Features())
 }
 
-func (c *Scaleway) LoadBalancerPricing() (*LoadBalancer, error) {
+func (c *Scaleway) LoadBalancerPricing() (*models.LoadBalancer, error) {
 	// Different LB types, lets take the cheaper for now, we can't get the type
 	// without a service specifying the type in the annotations
-	return &LoadBalancer{
+	return &models.LoadBalancer{
 		Cost: 0.014,
 	}, nil
 }
 
-func (c *Scaleway) NetworkPricing() (*Network, error) {
+func (c *Scaleway) NetworkPricing() (*models.Network, error) {
 	// it's free baby!
-	return &Network{
+	return &models.Network{
 		ZoneNetworkEgressCost:     0,
 		RegionNetworkEgressCost:   0,
 		InternetNetworkEgressCost: 0,
 	}, nil
 }
 
-func (c *Scaleway) GetKey(l map[string]string, n *v1.Node) Key {
+func (c *Scaleway) GetKey(l map[string]string, n *v1.Node) models.Key {
 	return &scalewayKey{
 		Labels: l,
 	}
@@ -202,7 +204,7 @@ func (key *scalewayPVKey) Features() string {
 	return key.Zone
 }
 
-func (c *Scaleway) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) PVKey {
+func (c *Scaleway) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	// the csi volume handle is the form <az>/<volume-id>
 	zone := strings.Split(pv.Spec.CSI.VolumeHandle, "/")[0]
 	return &scalewayPVKey{
@@ -214,24 +216,24 @@ func (c *Scaleway) GetPVKey(pv *v1.PersistentVolume, parameters map[string]strin
 	}
 }
 
-func (c *Scaleway) PVPricing(pvk PVKey) (*PV, error) {
+func (c *Scaleway) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()
 
 	pricing, ok := c.Pricing[pvk.Features()]
 	if !ok {
 		log.Infof("Persistent Volume pricing not found for %s: %s", pvk.GetStorageClass(), pvk.Features())
-		return &PV{}, nil
+		return &models.PV{}, nil
 	}
-	return &PV{
+	return &models.PV{
 		Cost:  fmt.Sprintf("%f", pricing.PVCost),
 		Class: pvk.GetStorageClass(),
 	}, nil
 }
 
-func (c *Scaleway) ServiceAccountStatus() *ServiceAccountStatus {
-	return &ServiceAccountStatus{
-		Checks: []*ServiceAccountCheck{},
+func (c *Scaleway) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return &models.ServiceAccountStatus{
+		Checks: []*models.ServiceAccountCheck{},
 	}
 }
 
@@ -260,7 +262,7 @@ func (c *Scaleway) Regions() []string {
 	return zones
 }
 
-func (*Scaleway) ApplyReservedInstancePricing(map[string]*Node) {}
+func (*Scaleway) ApplyReservedInstancePricing(map[string]*models.Node) {}
 
 func (*Scaleway) GetAddresses() ([]byte, error) {
 	return nil, nil
@@ -270,7 +272,7 @@ func (*Scaleway) GetDisks() ([]byte, error) {
 	return nil, nil
 }
 
-func (*Scaleway) GetOrphanedResources() ([]OrphanedResource, error) {
+func (*Scaleway) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	return nil, errors.New("not implemented")
 }
 
@@ -295,24 +297,24 @@ func (scw *Scaleway) ClusterInfo() (map[string]string, error) {
 
 }
 
-func (c *Scaleway) UpdateConfigFromConfigMap(a map[string]string) (*CustomPricing, error) {
+func (c *Scaleway) UpdateConfigFromConfigMap(a map[string]string) (*models.CustomPricing, error) {
 	return c.Config.UpdateFromMap(a)
 }
 
-func (c *Scaleway) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
+func (c *Scaleway) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
 	defer c.DownloadPricingData()
 
-	return c.Config.Update(func(c *CustomPricing) error {
+	return c.Config.Update(func(c *models.CustomPricing) error {
 		a := make(map[string]interface{})
 		err := json.NewDecoder(r).Decode(&a)
 		if err != nil {
 			return err
 		}
 		for k, v := range a {
-			kUpper := toTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
+			kUpper := utils.ToTitle.String(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
 			vstr, ok := v.(string)
 			if ok {
-				err := SetCustomPricingField(c, kUpper, vstr)
+				err := models.SetCustomPricingField(c, kUpper, vstr)
 				if err != nil {
 					return err
 				}
@@ -322,7 +324,7 @@ func (c *Scaleway) UpdateConfig(r io.Reader, updateType string) (*CustomPricing,
 		}
 
 		if env.IsRemoteEnabled() {
-			err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
+			err := utils.UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
 			if err != nil {
 				return err
 			}
@@ -331,7 +333,7 @@ func (c *Scaleway) UpdateConfig(r io.Reader, updateType string) (*CustomPricing,
 		return nil
 	})
 }
-func (scw *Scaleway) GetConfig() (*CustomPricing, error) {
+func (scw *Scaleway) GetConfig() (*models.CustomPricing, error) {
 	c, err := scw.Config.GetCustomPricingData()
 	if err != nil {
 		return nil, err
@@ -367,9 +369,9 @@ func (scw *Scaleway) GetManagementPlatform() (string, error) {
 	return "", nil
 }
 
-func (c *Scaleway) PricingSourceStatus() map[string]*PricingSource {
-	return map[string]*PricingSource{
-		InstanceAPIPricing: &PricingSource{
+func (c *Scaleway) PricingSourceStatus() map[string]*models.PricingSource {
+	return map[string]*models.PricingSource{
+		InstanceAPIPricing: &models.PricingSource{
 			Name:      InstanceAPIPricing,
 			Enabled:   true,
 			Available: true,

pkg/cloud/utils/utils.go

@@ -0,0 +1,113 @@
+package utils
+
+import (
+	"database/sql"
+	"fmt"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+
+	"github.com/opencost/opencost/pkg/env"
+)
+
+var ToTitle = cases.Title(language.Und, cases.NoLower)
+
+var createTableStatements = []string{
+	`CREATE TABLE IF NOT EXISTS names (
+		cluster_id VARCHAR(255) NOT NULL,
+		cluster_name VARCHAR(255) NULL,
+		PRIMARY KEY (cluster_id)
+	);`,
+}
+
+// TODO: these don't really fit in this package, should they move
+// somewhere else?
+
+func UpdateClusterMeta(cluster_id, cluster_name string) error {
+	pw := env.GetRemotePW()
+	address := env.GetSQLAddress()
+	connStr := fmt.Sprintf("postgres://postgres:%s@%s:5432?sslmode=disable", pw, address)
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	updateStmt := `UPDATE names SET cluster_name = $1 WHERE cluster_id = $2;`
+	_, err = db.Exec(updateStmt, cluster_name, cluster_id)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func CreateClusterMeta(cluster_id, cluster_name string) error {
+	pw := env.GetRemotePW()
+	address := env.GetSQLAddress()
+	connStr := fmt.Sprintf("postgres://postgres:%s@%s:5432?sslmode=disable", pw, address)
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	for _, stmt := range createTableStatements {
+		_, err := db.Exec(stmt)
+		if err != nil {
+			return err
+		}
+	}
+	insertStmt := `INSERT INTO names (cluster_id, cluster_name) VALUES ($1, $2);`
+	_, err = db.Exec(insertStmt, cluster_id, cluster_name)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func GetClusterMeta(cluster_id string) (string, string, error) {
+	pw := env.GetRemotePW()
+	address := env.GetSQLAddress()
+	connStr := fmt.Sprintf("postgres://postgres:%s@%s:5432?sslmode=disable", pw, address)
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return "", "", err
+	}
+	defer db.Close()
+	query := `SELECT cluster_id, cluster_name
+	FROM names
+	WHERE cluster_id = ?`
+
+	rows, err := db.Query(query, cluster_id)
+	if err != nil {
+		return "", "", err
+	}
+	defer rows.Close()
+	var (
+		sql_cluster_id string
+		cluster_name   string
+	)
+	for rows.Next() {
+		if err := rows.Scan(&sql_cluster_id, &cluster_name); err != nil {
+			return "", "", err
+		}
+	}
+
+	return sql_cluster_id, cluster_name, nil
+}
+
+func GetOrCreateClusterMeta(cluster_id, cluster_name string) (string, string, error) {
+	id, name, err := GetClusterMeta(cluster_id)
+	if err != nil {
+		err := CreateClusterMeta(cluster_id, cluster_name)
+		if err != nil {
+			return "", "", err
+		}
+	}
+	if id == "" {
+		err := CreateClusterMeta(cluster_id, cluster_name)
+		if err != nil {
+			return "", "", err
+		}
+	}
+
+	return id, name, nil
+}

pkg/cmd/costmodel/costmodel.go

@@ -2,8 +2,8 @@ package costmodel
 
 import (
 	"context"
+	"fmt"
 	"net/http"
-	"os"
 	"time"
 
 	"github.com/julienschmidt/httprouter"
@@ -34,7 +34,10 @@ func Execute(opts *CostModelOpts) error {
 	log.Infof("Starting cost-model version %s", version.FriendlyVersion())
 	a := costmodel.Initialize()
 
-	StartExportWorker(context.Background(), a.Model)
+	err := StartExportWorker(context.Background(), a.Model)
+	if err != nil {
+		log.Errorf("couldn't start CSV export worker: %v", err)
+	}
 
 	rootMux := http.NewServeMux()
 	a.Router.GET("/healthz", Healthz)
@@ -48,17 +51,14 @@ func Execute(opts *CostModelOpts) error {
 	return http.ListenAndServe(":9003", errors.PanicHandlerMiddleware(handler))
 }
 
-func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) {
-	exportPath := os.Getenv(env.ExportCSVFile)
+func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) error {
+	exportPath := env.GetExportCSVFile()
 	if exportPath == "" {
-		log.Infof("%s is not set, skipping CSV exporter", env.ExportCSVFile)
-		return
+		return fmt.Errorf("%s is not set, skipping CSV exporter", exportPath)
 	}
-
 	fm, err := filemanager.NewFileManager(exportPath)
 	if err != nil {
-		log.Errorf("could not start CSV exporter: %v", err)
-		return
+		return fmt.Errorf("could not create file manager: %v", err)
 	}
 	go func() {
 		log.Info("Starting CSV exporter worker...")
@@ -70,7 +70,7 @@ func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) {
 			case <-ctx.Done():
 				return
 			case <-time.After(nextRunAt.Sub(time.Now())):
-				err := costmodel.UpdateCSV(ctx, fm, model)
+				err := costmodel.UpdateCSV(ctx, fm, model, env.GetExportCSVLabelsAll(), env.GetExportCSVLabelsList())
 				if err != nil {
 					// it's background worker, log error and carry on, maybe next time it will work
 					log.Errorf("Error updating CSV: %s", err)
@@ -82,4 +82,5 @@ func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) {
 			}
 		}
 	}()
+	return nil
 }

pkg/costmodel/aggregation.go

@@ -10,11 +10,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/opencost/opencost/pkg/util/httputil"
-	"github.com/opencost/opencost/pkg/util/timeutil"
-
 	"github.com/julienschmidt/httprouter"
+	"github.com/patrickmn/go-cache"
+	prometheusClient "github.com/prometheus/client_golang/api"
+
 	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/errors"
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -22,9 +23,9 @@ import (
 	"github.com/opencost/opencost/pkg/prom"
 	"github.com/opencost/opencost/pkg/thanos"
 	"github.com/opencost/opencost/pkg/util"
+	"github.com/opencost/opencost/pkg/util/httputil"
 	"github.com/opencost/opencost/pkg/util/json"
-	"github.com/patrickmn/go-cache"
-	prometheusClient "github.com/prometheus/client_golang/api"
+	"github.com/opencost/opencost/pkg/util/timeutil"
 )
 
 const (
@@ -181,7 +182,7 @@ func NewSharedResourceInfo(shareResources bool, sharedNamespaces []string, label
 	return sr
 }
 
-func GetTotalContainerCost(costData map[string]*CostData, rate string, cp cloud.Provider, discount float64, customDiscount float64, idleCoefficients map[string]float64) float64 {
+func GetTotalContainerCost(costData map[string]*CostData, rate string, cp models.Provider, discount float64, customDiscount float64, idleCoefficients map[string]float64) float64 {
 	totalContainerCost := 0.0
 	for _, costDatum := range costData {
 		clusterID := costDatum.ClusterID
@@ -197,7 +198,7 @@ func GetTotalContainerCost(costData map[string]*CostData, rate string, cp cloud.
 	return totalContainerCost
 }
 
-func (a *Accesses) ComputeIdleCoefficient(costData map[string]*CostData, cli prometheusClient.Client, cp cloud.Provider, discount float64, customDiscount float64, window, offset time.Duration) (map[string]float64, error) {
+func (a *Accesses) ComputeIdleCoefficient(costData map[string]*CostData, cli prometheusClient.Client, cp models.Provider, discount float64, customDiscount float64, window, offset time.Duration) (map[string]float64, error) {
 	coefficients := make(map[string]float64)
 
 	profileName := "ComputeIdleCoefficient: ComputeClusterCosts"
@@ -287,7 +288,7 @@ func clampAverage(requestsAvg float64, usedAverage float64, allocationAvg float6
 // AggregateCostData aggregates raw cost data by field; e.g. namespace, cluster, service, or label. In the case of label, callers
 // must pass a slice of subfields indicating the labels by which to group. Provider is used to define custom resource pricing.
 // See AggregationOptions for optional parameters.
-func AggregateCostData(costData map[string]*CostData, field string, subfields []string, cp cloud.Provider, opts *AggregationOptions) map[string]*Aggregation {
+func AggregateCostData(costData map[string]*CostData, field string, subfields []string, cp models.Provider, opts *AggregationOptions) map[string]*Aggregation {
 	discount := opts.Discount
 	customDiscount := opts.CustomDiscount
 	idleCoefficients := opts.IdleCoefficients
@@ -584,7 +585,7 @@ func AggregateCostData(costData map[string]*CostData, field string, subfields []
 	return aggregations
 }
 
-func aggregateDatum(cp cloud.Provider, aggregations map[string]*Aggregation, costDatum *CostData, field string, subfields []string, rate string, key string, discount float64, customDiscount float64, idleCoefficient float64, includeProperties bool) {
+func aggregateDatum(cp models.Provider, aggregations map[string]*Aggregation, costDatum *CostData, field string, subfields []string, rate string, key string, discount float64, customDiscount float64, idleCoefficient float64, includeProperties bool) {
 	// add new entry to aggregation results if a new key is encountered
 	if _, ok := aggregations[key]; !ok {
 		agg := &Aggregation{
@@ -617,7 +618,7 @@ func aggregateDatum(cp cloud.Provider, aggregations map[string]*Aggregation, cos
 	mergeVectors(cp, costDatum, aggregations[key], rate, discount, customDiscount, idleCoefficient)
 }
 
-func mergeVectors(cp cloud.Provider, costDatum *CostData, aggregation *Aggregation, rate string, discount float64, customDiscount float64, idleCoefficient float64) {
+func mergeVectors(cp models.Provider, costDatum *CostData, aggregation *Aggregation, rate string, discount float64, customDiscount float64, idleCoefficient float64) {
 	aggregation.CPUAllocationVectors = addVectors(costDatum.CPUAllocation, aggregation.CPUAllocationVectors)
 	aggregation.CPURequestedVectors = addVectors(costDatum.CPUReq, aggregation.CPURequestedVectors)
 	aggregation.CPUUsedVectors = addVectors(costDatum.CPUUsed, aggregation.CPUUsedVectors)
@@ -711,7 +712,7 @@ func getDiscounts(costDatum *CostData, cpuCost float64, ramCost float64, discoun
 	return blendedCPUDiscount, blendedRAMDiscount
 }
 
-func parseVectorPricing(cfg *cloud.CustomPricing, costDatum *CostData, cpuCostStr, ramCostStr, gpuCostStr, pvCostStr string) (float64, float64, float64, float64, bool) {
+func parseVectorPricing(cfg *models.CustomPricing, costDatum *CostData, cpuCostStr, ramCostStr, gpuCostStr, pvCostStr string) (float64, float64, float64, float64, bool) {
 	usesCustom := false
 	cpuCost, err := strconv.ParseFloat(cpuCostStr, 64)
 	if err != nil || math.IsNaN(cpuCost) || math.IsInf(cpuCost, 0) || cpuCost == 0 {
@@ -746,7 +747,7 @@ func parseVectorPricing(cfg *cloud.CustomPricing, costDatum *CostData, cpuCostSt
 	return cpuCost, ramCost, gpuCost, pvCost, usesCustom
 }
 
-func getPriceVectors(cp cloud.Provider, costDatum *CostData, rate string, discount float64, customDiscount float64, idleCoefficient float64) ([]*util.Vector, []*util.Vector, []*util.Vector, [][]*util.Vector, []*util.Vector) {
+func getPriceVectors(cp models.Provider, costDatum *CostData, rate string, discount float64, customDiscount float64, idleCoefficient float64) ([]*util.Vector, []*util.Vector, []*util.Vector, [][]*util.Vector, []*util.Vector) {
 
 	var cpuCost float64
 	var ramCost float64

pkg/costmodel/cluster.go

@@ -5,16 +5,16 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/opencost/opencost/pkg/kubecost"
-	"github.com/opencost/opencost/pkg/util/timeutil"
+	prometheus "github.com/prometheus/client_golang/api"
 	"golang.org/x/exp/slices"
 
 	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
+	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/prom"
-
-	prometheus "github.com/prometheus/client_golang/api"
+	"github.com/opencost/opencost/pkg/util/timeutil"
 )
 
 const (
@@ -141,7 +141,7 @@ type DiskIdentifier struct {
 	Name    string
 }
 
-func ClusterDisks(client prometheus.Client, provider cloud.Provider, start, end time.Time) (map[DiskIdentifier]*Disk, error) {
+func ClusterDisks(client prometheus.Client, provider models.Provider, start, end time.Time) (map[DiskIdentifier]*Disk, error) {
 	// Query for the duration between start and end
 	durStr := timeutil.DurationString(end.Sub(start))
 	if durStr == "" {
@@ -542,7 +542,7 @@ func costTimesMinute(activeDataMap map[NodeIdentifier]activeData, costMap map[No
 	}
 }
 
-func ClusterNodes(cp cloud.Provider, client prometheus.Client, start, end time.Time) (map[NodeIdentifier]*Node, error) {
+func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.Time) (map[NodeIdentifier]*Node, error) {
 	// Query for the duration between start and end
 	durStr := timeutil.DurationString(end.Sub(start))
 	if durStr == "" {
@@ -833,7 +833,7 @@ func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[L
 }
 
 // ComputeClusterCosts gives the cumulative and monthly-rate cluster costs over a window of time for all clusters.
-func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider cloud.Provider, window, offset time.Duration, withBreakdown bool) (map[string]*ClusterCosts, error) {
+func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models.Provider, window, offset time.Duration, withBreakdown bool) (map[string]*ClusterCosts, error) {
 	if window < 10*time.Minute {
 		return nil, fmt.Errorf("minimum window of 10m required; got %s", window)
 	}
@@ -1192,7 +1192,7 @@ func resultToTotals(qrs []*prom.QueryResult) ([][]string, error) {
 }
 
 // ClusterCostsOverTime gives the full cluster costs over time
-func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startString, endString string, window, offset time.Duration) (*Totals, error) {
+func ClusterCostsOverTime(cli prometheus.Client, provider models.Provider, startString, endString string, window, offset time.Duration) (*Totals, error) {
 	localStorageQuery := provider.GetLocalStorageQuery(window, offset, true, false)
 	if localStorageQuery != "" {
 		localStorageQuery = fmt.Sprintf("+ %s", localStorageQuery)
@@ -1298,7 +1298,7 @@ func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startS
 	}, nil
 }
 
-func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo []*prom.QueryResult, cp cloud.Provider) {
+func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo []*prom.QueryResult, cp models.Provider) {
 	for _, result := range resActiveMins {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {

pkg/costmodel/cluster_helpers.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/models"
 
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"
@@ -30,7 +31,7 @@ func mergeTypeMaps(clusterAndNameToType1, clusterAndNameToType2 map[nodeIdentifi
 
 func buildCPUCostMap(
 	resNodeCPUCost []*prom.QueryResult,
-	cp cloud.Provider,
+	cp models.Provider,
 	preemptible map[NodeIdentifier]bool,
 ) (
 	map[NodeIdentifier]float64,
@@ -104,7 +105,7 @@ func buildCPUCostMap(
 
 func buildRAMCostMap(
 	resNodeRAMCost []*prom.QueryResult,
-	cp cloud.Provider,
+	cp models.Provider,
 	preemptible map[NodeIdentifier]bool,
 ) (
 	map[NodeIdentifier]float64,
@@ -179,7 +180,7 @@ func buildRAMCostMap(
 func buildGPUCostMap(
 	resNodeGPUCost []*prom.QueryResult,
 	gpuCountMap map[NodeIdentifier]float64,
-	cp cloud.Provider,
+	cp models.Provider,
 	preemptible map[NodeIdentifier]bool,
 ) (
 	map[NodeIdentifier]float64,

pkg/costmodel/clusterinfo.go

@@ -3,7 +3,7 @@ package costmodel
 import (
 	"fmt"
 
-	cloudProvider "github.com/opencost/opencost/pkg/cloud"
+	cloudProvider "github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/config"
 	"github.com/opencost/opencost/pkg/costmodel/clusters"
 	"github.com/opencost/opencost/pkg/env"

pkg/costmodel/costmodel.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud"
+	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/costmodel/clusters"
 	"github.com/opencost/opencost/pkg/env"

pkg/costmodel/csv_export.go

@@ -3,6 +3,7 @@ package costmodel
 import (
 	"context"
 	"encoding/csv"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -23,10 +24,12 @@ type AllocationModel interface {
 
 var errNoData = errors.New("no data")
 
-func UpdateCSV(ctx context.Context, fileManager filemanager.FileManager, model AllocationModel) error {
+func UpdateCSV(ctx context.Context, fileManager filemanager.FileManager, model AllocationModel, labelsAll bool, labels []string) error {
 	exporter := &csvExporter{
 		FileManager: fileManager,
 		Model:       model,
+		LabelsAll:   labelsAll,
+		Labels:      labels,
 	}
 	return exporter.Update(ctx)
 }
@@ -34,6 +37,8 @@ func UpdateCSV(ctx context.Context, fileManager filemanager.FileManager, model A
 type csvExporter struct {
 	FileManager filemanager.FileManager
 	Model       AllocationModel
+	Labels      []string // If not empty, create a column for each label prefixed with "Label_"
+	LabelsAll   bool     // if true, export all labels to a "Labels" column in JSON format
 }
 
 // Update updates CSV file in cloud storage with new allocation data
@@ -153,35 +158,173 @@ func (e *csvExporter) writeCSVToWriter(ctx context.Context, w io.Writer, dates [
 	fmtFloat := func(f float64) string {
 		return strconv.FormatFloat(f, 'f', -1, 64)
 	}
+
+	type rowData struct {
+		date  time.Time
+		alloc *kubecost.Allocation
+	}
+
+	type columnDef struct {
+		column string
+		value  func(data rowData) string
+	}
+
+	csvDef := []columnDef{
+		{
+			column: "Date",
+			value: func(data rowData) string {
+				return data.date.Format("2006-01-02")
+			},
+		},
+		{
+			column: "Namespace",
+			value: func(data rowData) string {
+				return data.alloc.Properties.Namespace
+			},
+		},
+		{
+			column: "ControllerKind",
+			value: func(data rowData) string {
+				return data.alloc.Properties.ControllerKind
+			},
+		},
+		{
+			column: "ControllerName",
+			value: func(data rowData) string {
+				return data.alloc.Properties.Controller
+			},
+		},
+		{
+			column: "Pod",
+			value: func(data rowData) string {
+				return data.alloc.Properties.Pod
+			},
+		},
+		{
+			column: "Container",
+			value: func(data rowData) string {
+				return data.alloc.Properties.Container
+			},
+		},
+		{
+			column: "CPUCoreUsageAverage",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.CPUCoreUsageAverage)
+			},
+		},
+		{
+			column: "CPUCoreRequestAverage",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.CPUCoreRequestAverage)
+			},
+		},
+		{
+			column: "RAMBytesUsageAverage",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.RAMBytesUsageAverage)
+			},
+		},
+		{
+			column: "RAMBytesRequestAverage",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.RAMBytesRequestAverage)
+			},
+		},
+		{
+			column: "NetworkReceiveBytes",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.NetworkReceiveBytes)
+			},
+		},
+		{
+			column: "NetworkTransferBytes",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.NetworkTransferBytes)
+			},
+		},
+		{
+			column: "GPUs",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.GPUs())
+			},
+		},
+		{
+			column: "PVBytes",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.PVBytes())
+			},
+		},
+		{
+			column: "CPUCost",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.CPUTotalCost())
+			},
+		},
+		{
+			column: "RAMCost",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.RAMTotalCost())
+			},
+		},
+		{
+			column: "NetworkCost",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.NetworkTotalCost())
+			},
+		},
+		{
+			column: "PVCost",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.PVTotalCost())
+			},
+		},
+		{
+			column: "GPUCost",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.GPUTotalCost())
+			},
+		},
+		{
+			column: "TotalCost",
+			value: func(data rowData) string {
+				return fmtFloat(data.alloc.TotalCost())
+			},
+		},
+	}
+	if e.LabelsAll {
+		csvDef = append(csvDef, columnDef{
+			column: "Labels",
+			value: func(data rowData) string {
+				return fmtLabelsCSV(data.alloc.Properties.Labels)
+			},
+		})
+	}
+	for i := range e.Labels {
+		label := e.Labels[i] // it's important to copy the label name, otherwise all closures will reference the same label
+		csvDef = append(csvDef, columnDef{
+			column: "Label_" + label,
+			value: func(data rowData) string {
+				value, _ := data.alloc.Properties.Labels[label]
+				return value
+			},
+		})
+	}
+	csvDef = append(csvDef)
+
+	header := make([]string, 0, len(csvDef))
+	for _, def := range csvDef {
+		header = append(header, def.column)
+	}
+
 	csvWriter := csv.NewWriter(w)
-	err := csvWriter.Write([]string{
-		"Date",
-		"Namespace",
-		"ControllerKind",
-		"ControllerName",
-		"Pod",
-		"Container",
-
-		"CPUCoreUsageAverage",
-		"CPUCoreRequestAverage",
-		"RAMBytesUsageAverage",
-		"RAMBytesRequestAverage",
-		"NetworkReceiveBytes",
-		"NetworkTransferBytes",
-		"GPUs",
-		"PVBytes",
-
-		"CPUCost",
-		"RAMCost",
-		"NetworkCost",
-		"PVCost",
-		"GPUCost",
-		"TotalCost",
-	})
+	lines := 0
+	err := csvWriter.Write(header)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to write header: %w", err)
 	}
-	lines := 0
+
+	log.Infof("writing CSV with header: %v", header)
+
 	for _, date := range dates {
 		start := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC)
 		end := start.AddDate(0, 0, 1)
@@ -194,36 +337,15 @@ func (e *csvExporter) writeCSVToWriter(ctx context.Context, w io.Writer, dates [
 			if err := ctx.Err(); err != nil {
 				return err
 			}
-
-			log.Infof("%f", alloc.TotalCost())
-
-			err := csvWriter.Write([]string{
-				date.Format("2006-01-02"),
-				alloc.Properties.Namespace,
-				alloc.Properties.ControllerKind,
-				alloc.Properties.Controller,
-				alloc.Properties.Pod,
-				alloc.Properties.Container,
-
-				fmtFloat(alloc.CPUCoreUsageAverage),
-				fmtFloat(alloc.CPUCoreRequestAverage),
-				fmtFloat(alloc.RAMBytesUsageAverage),
-				fmtFloat(alloc.RAMBytesRequestAverage),
-				fmtFloat(alloc.NetworkReceiveBytes),
-				fmtFloat(alloc.NetworkTransferBytes),
-				fmtFloat(alloc.GPUs()),
-				fmtFloat(alloc.PVBytes()),
-
-				fmtFloat(alloc.CPUTotalCost()),
-				fmtFloat(alloc.RAMTotalCost()),
-				fmtFloat(alloc.NetworkTotalCost()),
-				fmtFloat(alloc.PVCost()),
-				fmtFloat(alloc.GPUCost),
-				fmtFloat(alloc.TotalCost()),
-			})
+			row := make([]string, 0, len(csvDef))
+			for _, def := range csvDef {
+				row = append(row, def.value(rowData{date: date, alloc: alloc}))
+			}
+			err := csvWriter.Write(row)
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to write csv row: %w", err)
 			}
+
 			lines++
 		}
 	}
@@ -240,6 +362,19 @@ func (e *csvExporter) writeCSVToWriter(ctx context.Context, w io.Writer, dates [
 	return nil
 }
 
+func fmtLabelsCSV(labels map[string]string) string {
+	if len(labels) == 0 {
+		return ""
+	}
+
+	data, err := json.Marshal(labels)
+	if err != nil {
+		log.Errorf("failed to marshal labels: %s", err)
+		return ""
+	}
+	return string(data)
+}
+
 // loadDate scans through CSV export file and extract all dates from "Date" column
 func (e *csvExporter) loadDates(csvFile *os.File) (map[time.Time]struct{}, error) {
 	_, err := csvFile.Seek(0, io.SeekStart)

pkg/costmodel/csv_export_test.go

@@ -60,7 +60,7 @@ func Test_UpdateCSV(t *testing.T) {
 				}, nil
 			},
 		}
-		err := UpdateCSV(context.TODO(), storage, model)
+		err := UpdateCSV(context.TODO(), storage, model, false, nil)
 		require.NoError(t, err)
 		// uploaded a single file with the data
 		assert.Len(t, model.ComputeAllocationCalls(), 1)
@@ -71,10 +71,45 @@ func Test_UpdateCSV(t *testing.T) {
 `, string(storage.Data))
 	})
 
+	t.Run("export labels", func(t *testing.T) {
+		storage := &filemanager.InMemoryFile{}
+		model := &AllocationModelMock{
+			DateRangeFunc: func() (time.Time, time.Time, error) {
+				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), nil
+			},
+			ComputeAllocationFunc: func(start time.Time, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
+				return &kubecost.AllocationSet{
+					Allocations: map[string]*kubecost.Allocation{
+						"test": {
+							Properties: &kubecost.AllocationProperties{
+								Namespace:      "test-namespace",
+								Controller:     "test-controller-name",
+								ControllerKind: "test-controller-kind",
+								Pod:            "test-pod",
+								Container:      "test-container",
+								Labels: map[string]string{
+									"test-label1": "test-value1",
+									"test-label2": "test-value2",
+								},
+							},
+						},
+					},
+				}, nil
+			},
+		}
+		err := UpdateCSV(context.TODO(), storage, model, true, []string{"test-label1", "test-label2"})
+		require.NoError(t, err)
+		// uploaded a single file with the data
+		assert.Len(t, model.ComputeAllocationCalls(), 1)
+		assert.Equal(t, `Date,Namespace,ControllerKind,ControllerName,Pod,Container,CPUCoreUsageAverage,CPUCoreRequestAverage,RAMBytesUsageAverage,RAMBytesRequestAverage,NetworkReceiveBytes,NetworkTransferBytes,GPUs,PVBytes,CPUCost,RAMCost,NetworkCost,PVCost,GPUCost,TotalCost,Labels,Label_test-label1,Label_test-label2
+2021-01-01,test-namespace,test-controller-kind,test-controller-name,test-pod,test-container,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"{""test-label1"":""test-value1"",""test-label2"":""test-value2""}",test-value1,test-value2
+`, string(storage.Data))
+	})
+
 	t.Run("merge new data with previous data (with different CSV structure)", func(t *testing.T) {
 		storage := &filemanager.InMemoryFile{
-			Data: []byte(`Date,Namespace,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
-2021-01-01,test-namespace,0.1,0.2,0.3,0.4,0.5,0.6
+			Data: []byte(`Date,Namespace,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost,Label_app
+2021-01-01,test-namespace,0.1,0.2,0.3,0.4,0.5,0.6,app1
 `),
 		}
 		model := &AllocationModelMock{
@@ -94,7 +129,7 @@ func Test_UpdateCSV(t *testing.T) {
 				}, nil
 			},
 		}
-		err := UpdateCSV(context.TODO(), storage, model)
+		err := UpdateCSV(context.TODO(), storage, model, false, nil)
 		require.NoError(t, err)
 		// uploaded a single file with the data
 		assert.Len(t, model.ComputeAllocationCalls(), 1)
@@ -102,9 +137,9 @@ func Test_UpdateCSV(t *testing.T) {
 		// 2021-01-01 is already in the export file, so we only compute for 2021-01-02
 		assert.Equal(t, time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), model.ComputeAllocationCalls()[0].Start)
 		assert.Equal(t, time.Date(2021, 1, 3, 0, 0, 0, 0, time.UTC), model.ComputeAllocationCalls()[0].End)
-		assert.Equal(t, `Date,Namespace,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost,ControllerKind,ControllerName,Pod,Container,NetworkReceiveBytes,NetworkTransferBytes,GPUs,PVBytes,NetworkCost,PVCost,GPUCost,TotalCost
-2021-01-01,test-namespace,0.1,0.2,0.3,0.4,0.5,0.6,,,,,,,,,,,,
-2021-01-02,test-namespace,0,0,1,0,0,0,,,,,0,0,0,0,0,0,0,1
+		assert.Equal(t, `Date,Namespace,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost,Label_app,ControllerKind,ControllerName,Pod,Container,NetworkReceiveBytes,NetworkTransferBytes,GPUs,PVBytes,NetworkCost,PVCost,GPUCost,TotalCost
+2021-01-01,test-namespace,0.1,0.2,0.3,0.4,0.5,0.6,app1,,,,,,,,,,,,
+2021-01-02,test-namespace,0,0,1,0,0,0,,,,,,0,0,0,0,0,0,0,1
 `, string(storage.Data))
 	})
 
@@ -120,7 +155,7 @@ func Test_UpdateCSV(t *testing.T) {
 				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), nil
 			},
 		}
-		err := UpdateCSV(context.TODO(), storage, model)
+		err := UpdateCSV(context.TODO(), storage, model, false, nil)
 		require.Equal(t, err, errNoData)
 		assert.Equal(t, string(storage.Data), data)
 		assert.Len(t, model.ComputeAllocationCalls(), 0)
@@ -138,7 +173,7 @@ func Test_UpdateCSV(t *testing.T) {
 			},
 		}
 		storage := &filemanager.InMemoryFile{}
-		err := UpdateCSV(context.TODO(), storage, model)
+		err := UpdateCSV(context.TODO(), storage, model, false, nil)
 		require.Equal(t, err, errNoData)
 	})
 }

pkg/costmodel/metrics.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/costmodel/clusters"
 	"github.com/opencost/opencost/pkg/env"
@@ -136,7 +136,7 @@ var (
 )
 
 // initCostModelMetrics uses a sync.Once to ensure that these metrics are only created once
-func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider cloud.Provider, clusterInfo clusters.ClusterInfoProvider, metricsConfig *metrics.MetricsConfig) {
+func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider models.Provider, clusterInfo clusters.ClusterInfoProvider, metricsConfig *metrics.MetricsConfig) {
 
 	disabledMetrics := metricsConfig.GetDisabledMetricsMap()
 	var toRegisterGV []*prometheus.GaugeVec
@@ -297,7 +297,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider cloud
 type CostModelMetricsEmitter struct {
 	PrometheusClient promclient.Client
 	KubeClusterCache clustercache.ClusterCache
-	CloudProvider    cloud.Provider
+	CloudProvider    models.Provider
 	Model            *CostModel
 
 	// Metrics
@@ -323,7 +323,7 @@ type CostModelMetricsEmitter struct {
 }
 
 // NewCostModelMetricsEmitter creates a new cost-model metrics emitter. Use Start() to begin metric emission.
-func NewCostModelMetricsEmitter(promClient promclient.Client, clusterCache clustercache.ClusterCache, provider cloud.Provider, clusterInfo clusters.ClusterInfoProvider, model *CostModel) *CostModelMetricsEmitter {
+func NewCostModelMetricsEmitter(promClient promclient.Client, clusterCache clustercache.ClusterCache, provider models.Provider, clusterInfo clusters.ClusterInfoProvider, model *CostModel) *CostModelMetricsEmitter {
 
 	// Get metric configurations, if any
 	metricsConfig, err := metrics.GetMetricsConfig()
@@ -652,7 +652,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				} else {
 					region = defaultRegion
 				}
-				cacPv := &cloud.PV{
+				cacPv := &models.PV{
 					Class:      pv.Spec.StorageClassName,
 					Region:     region,
 					Parameters: parameters,

pkg/costmodel/networkcosts.go

@@ -1,7 +1,7 @@
 package costmodel
 
 import (
-	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud"
+	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/prom"

pkg/costmodel/promparsers.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud"
+	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"

pkg/costmodel/router.go

@@ -33,6 +33,9 @@ import (
 	sentry "github.com/getsentry/sentry-go"
 
 	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/azure"
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/costmodel/clusters"
 	"github.com/opencost/opencost/pkg/env"
@@ -83,7 +86,7 @@ type Accesses struct {
 	KubeClientSet       kubernetes.Interface
 	ClusterCache        clustercache.ClusterCache
 	ClusterMap          clusters.ClusterMap
-	CloudProvider       cloud.Provider
+	CloudProvider       models.Provider
 	ConfigFileManager   *config.ConfigFileManager
 	ClusterInfoProvider clusters.ClusterInfoProvider
 	Model               *CostModel
@@ -622,7 +625,7 @@ func (a *Accesses) UpdateBigQueryInfoConfigs(w http.ResponseWriter, r *http.Requ
 func (a *Accesses) UpdateAzureStorageConfigs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
-	data, err := a.CloudProvider.UpdateConfig(r.Body, cloud.AzureStorageUpdateType)
+	data, err := a.CloudProvider.UpdateConfig(r.Body, azure.AzureStorageUpdateType)
 	if err != nil {
 		w.Write(WrapData(data, err))
 		return
@@ -1620,7 +1623,7 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 		if err != nil {
 			log.Infof("Error saving cluster id %s", err.Error())
 		}
-		_, _, err = cloud.GetOrCreateClusterMeta(info["id"], info["name"])
+		_, _, err = utils.GetOrCreateClusterMeta(info["id"], info["name"])
 		if err != nil {
 			log.Infof("Unable to set cluster id '%s' for cluster '%s', %s", info["id"], info["name"], err.Error())
 		}

pkg/costmodel/settings.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/patrickmn/go-cache"
 )
@@ -87,7 +87,7 @@ func (a *Accesses) customPricingHasChanged() bool {
 
 	// describe parameters by which we determine whether or not custom
 	// pricing settings have changed
-	encodeCustomPricing := func(cp *cloud.CustomPricing) string {
+	encodeCustomPricing := func(cp *models.CustomPricing) string {
 		return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", cp.CustomPricesEnabled, cp.CPU, cp.SpotCPU,
 			cp.RAM, cp.SpotRAM, cp.GPU, cp.Storage, cp.CurrencyCode, cp.SharedOverhead)
 	}
@@ -127,7 +127,7 @@ func (a *Accesses) discountHasChanged() bool {
 
 	// describe parameters by which we determine whether or not custom
 	// pricing settings have changed
-	encodeDiscount := func(cp *cloud.CustomPricing) string {
+	encodeDiscount := func(cp *models.CustomPricing) string {
 		return fmt.Sprintf("%s:%s", cp.Discount, cp.NegotiatedDiscount)
 	}
 

pkg/costmodel/sql.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud"
+	costAnalyzerCloud "github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/util"

pkg/env/costmodelenv.go

@@ -99,7 +99,9 @@ const (
 
 	regionOverrideList = "REGION_OVERRIDE_LIST"
 
-	ExportCSVFile = "EXPORT_CSV_FILE"
+	ExportCSVFile       = "EXPORT_CSV_FILE"
+	ExportCSVLabelsList = "EXPORT_CSV_LABELS_LIST"
+	ExportCSVLabelsAll  = "EXPORT_CSV_LABELS_ALL"
 )
 
 const DefaultConfigMountPath = "/var/configs"
@@ -110,6 +112,18 @@ func IsETLReadOnlyMode() bool {
 	return GetBool(ETLReadOnlyMode, false)
 }
 
+func GetExportCSVFile() string {
+	return Get(ExportCSVFile, "")
+}
+
+func GetExportCSVLabelsAll() bool {
+	return GetBool(ExportCSVLabelsAll, false)
+}
+
+func GetExportCSVLabelsList() []string {
+	return GetList(ExportCSVLabelsList, ",")
+}
+
 // GetKubecostConfigBucket returns a file location for a mounted bucket configuration which is used to store
 // a subset of kubecost configurations that require sharing via remote storage.
 func GetKubecostConfigBucket() string {

pkg/filemanager/filemanager.go

@@ -44,6 +44,8 @@ func NewFileManager(path string) (FileManager, error) {
 		return NewGCSStorageFile(path)
 	case strings.Contains(path, "blob.core.windows.net"):
 		return NewAzureBlobFile(path)
+	case path == "":
+		return nil, errors.New("empty path")
 	default:
 		return NewSystemFile(path), nil
 	}