Merge pull request #1956 from CampfireRemnants/open-source-azure-cloudcost-integration

Added Azure CUR integration to the Open Source.

pkg/cloud/azure/azurestorageintegration.go

@@ -0,0 +1,99 @@
+package azure
+
+import (
+	"strings"
+	"time"
+
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+type AzureStorageIntegration struct {
+	AzureStorageBillingParser
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (asi *AzureStorageIntegration) GetCloudCost(start, end time.Time) (*kubecost.CloudCostSetRange, error) {
+	ccsr, err := kubecost.NewCloudCostSetRange(start, end, timeutil.Day, asi.Key())
+	if err != nil {
+		return nil, err
+	}
+
+	status, err := asi.ParseBillingData(start, end, func(abv *BillingRowValues) error {
+		s := abv.Date
+		e := abv.Date.Add(timeutil.Day)
+		window := kubecost.NewWindow(&s, &e)
+
+		k8sPtc := 0.0
+		if AzureIsK8s(abv.Tags) {
+			k8sPtc = 1.0
+		}
+
+		// Create CloudCost
+		// Using the NetCost as a 'placeholder' for Invoiced and Amortized Net costs now,
+		// until we can revisit and spend the time to do the calculations correctly
+		cc := &kubecost.CloudCost{
+			Properties: &kubecost.CloudCostProperties{
+				ProviderID:      AzureSetProviderID(abv),
+				Provider:        kubecost.AzureProvider,
+				AccountID:       abv.SubscriptionID,
+				InvoiceEntityID: abv.InvoiceEntityID,
+				Service:         abv.Service,
+				Category:        SelectAzureCategory(abv.MeterCategory),
+				Labels:          abv.Tags,
+			},
+			Window: window,
+			AmortizedNetCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+			InvoicedCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+			ListCost: kubecost.CostMetric{
+				Cost:              abv.Cost,
+				KubernetesPercent: k8sPtc,
+			},
+			NetCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+			// NOTE: on Azure, there is no "AmortizedCost" per se, so we use
+			// AmortizedNetCost, or NetCost, instead.
+			AmortizedCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+		}
+
+		// Check if Item
+		if abv.IsCompute(cc.Properties.Category) {
+			// TODO: Will need to split VMSS for other features
+			ccsr.LoadCloudCost(cc)
+		}
+		return nil
+	})
+	if err != nil {
+		asi.ConnectionStatus = status
+		return nil, err
+	}
+	return ccsr, nil
+}
+
+// Check for the presence of k8s labels
+func AzureIsK8s(labels map[string]string) bool {
+	for key := range labels {
+		if strings.HasPrefix(key, "aks-managed-") {
+			return true
+		}
+		if strings.HasPrefix(key, "kubernetes.io-created-") {
+			return true
+		}
+		if strings.HasPrefix(key, "k8s-azure-created-") {
+			return true
+		}
+	}
+	return false
+}

pkg/cloud/azure/azurestorageintegration_test.go

@@ -0,0 +1,69 @@
+package azure
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/pkg/util/json"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+func GetCloudCost_Test(t *testing.T) {
+	azureConfigPath := os.Getenv("AZURE_CONFIGURATION")
+	if azureConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable AZURE_CONFIGURATION")
+	}
+	azureConfigBin, err := os.ReadFile(azureConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var azureConfig StorageConfiguration
+	err = json.Unmarshal(azureConfigBin, &azureConfig)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+	testCases := map[string]struct {
+		integration *AzureStorageIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+		// No CUR data is expected within 2 days of now
+		"too_recent_window": {
+			integration: &AzureStorageIntegration{
+				AzureStorageBillingParser: AzureStorageBillingParser{
+					StorageConnection: StorageConnection{
+						StorageConfiguration: azureConfig,
+					},
+				},
+			},
+			end:      time.Now(),
+			start:    time.Now().Add(-timeutil.Day),
+			expected: true,
+		},
+		// CUR data should be available
+		"last week window": {
+			integration: &AzureStorageIntegration{
+				AzureStorageBillingParser: AzureStorageBillingParser{
+					StorageConnection: StorageConnection{
+						StorageConfiguration: azureConfig,
+					},
+				},
+			},
+			end:      time.Now().Add(-7 * timeutil.Day),
+			start:    time.Now().Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}