package azure import ( "fmt" "regexp" "strconv" "strings" "time" "github.com/opencost/opencost/core/pkg/log" "github.com/opencost/opencost/core/pkg/opencost" "github.com/opencost/opencost/core/pkg/util/json" ) const azureDateLayout = "2006-01-02" const AzureEnterpriseDateLayout = "01/02/2006" var groupRegex = regexp.MustCompile("(/[^/]+)") // BillingRowValues holder for Azure Billing Values type BillingRowValues struct { Date time.Time MeterCategory string SubscriptionID string InvoiceEntityID string InstanceID string Service string Tags map[string]string AdditionalInfo map[string]any Cost float64 NetCost float64 } func (brv *BillingRowValues) IsCompute(category string) bool { if category == opencost.ComputeCategory { return true } if category == opencost.StorageCategory || category == opencost.NetworkCategory { if brv.Service == "Microsoft.Compute" { return true } } if category == opencost.NetworkCategory && brv.MeterCategory == "Virtual Network" { return true } if category == opencost.NetworkCategory && brv.MeterCategory == "Bandwidth" { return true } return false } // BillingExportParser holds indexes of relevent fields in Azure Billing CSV in addition to the correct data format type BillingExportParser struct { Date int MeterCategory int InvoiceEntityID int SubscriptionID int InstanceID int Service int Tags int AdditionalInfo int Cost int NetCost int DateFormat string } // match "SubscriptionGuid" in "Abonnement-GUID (SubscriptionGuid)" var getParenContentRegEx = regexp.MustCompile("\\((.*?)\\)") func NewBillingParseSchema(headers []string) (*BillingExportParser, error) { // clear BOM from headers if len(headers) != 0 { headers[0] = strings.TrimPrefix(headers[0], "\xEF\xBB\xBF") } headerIndexes := map[string]int{} for i, header := range headers { // Azure Headers in different regions will have english headers in parentheses match := getParenContentRegEx.FindStringSubmatch(header) if len(match) != 0 { header = match[len(match)-1] } headerIndexes[strings.ToLower(header)] = i } abp := &BillingExportParser{} // Set Date Column and Date Format if i, ok := headerIndexes["usagedatetime"]; ok { abp.Date = i abp.DateFormat = azureDateLayout } else if j, ok2 := headerIndexes["date"]; ok2 { abp.Date = j abp.DateFormat = AzureEnterpriseDateLayout } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Date field") } // set Subscription ID if i, ok := headerIndexes["subscriptionid"]; ok { abp.SubscriptionID = i } else if j, ok2 := headerIndexes["subscriptionguid"]; ok2 { abp.SubscriptionID = j } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Subscription ID field") } // Set Billing ID if i, ok := headerIndexes["billingaccountid"]; ok { abp.InvoiceEntityID = i } else if j, ok2 := headerIndexes["billingaccountname"]; ok2 { abp.InvoiceEntityID = j } else { // if no billing ID column is present use subscription ID abp.InvoiceEntityID = abp.SubscriptionID } // Set Instance ID if i, ok := headerIndexes["instanceid"]; ok { abp.InstanceID = i } else if j, ok2 := headerIndexes["instancename"]; ok2 { abp.InstanceID = j } else if k, ok3 := headerIndexes["resourceid"]; ok3 { abp.InstanceID = k } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Instance ID field") } // Set Meter Category if i, ok := headerIndexes["metercategory"]; ok { abp.MeterCategory = i } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Meter Category field") } // Set Tags if i, ok := headerIndexes["tags"]; ok { abp.Tags = i } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Tags field") } // Set Additional Info if i, ok := headerIndexes["additionalinfo"]; ok { abp.AdditionalInfo = i } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Additional Info field") } // Set Service if i, ok := headerIndexes["consumedservice"]; ok { abp.Service = i } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Service field") } // Set Net Cost if i, ok := headerIndexes["costinbillingcurrency"]; ok { abp.NetCost = i } else if j, ok2 := headerIndexes["pretaxcost"]; ok2 { abp.NetCost = j } else if k, ok3 := headerIndexes["cost"]; ok3 { abp.NetCost = k } else { return nil, fmt.Errorf("NewBillingParseSchema: failed to find Net Cost field") } // Set Cost if i, ok := headerIndexes["paygcostinbillingcurrency"]; ok { abp.Cost = i } else { // if no Cost column is present use Net Cost column abp.Cost = abp.NetCost } return abp, nil } func (bep *BillingExportParser) ParseRow(start, end time.Time, record []string) *BillingRowValues { usageDate, err := time.Parse(bep.DateFormat, record[bep.Date]) if err != nil { // try other format, and switch if successful if bep.DateFormat == azureDateLayout { bep.DateFormat = AzureEnterpriseDateLayout } else { bep.DateFormat = azureDateLayout } usageDate, err = time.Parse(bep.DateFormat, record[bep.Date]) // If parse still fails then return line if err != nil { log.Errorf("failed to parse usage date: '%s'", record[bep.Date]) return nil } } // skip if usage data isn't in subject window if usageDate.Before(start) || !usageDate.Before(end) { return nil } cost, err := strconv.ParseFloat(record[bep.Cost], 64) if err != nil { log.Errorf("failed to parse cost: '%s'", record[bep.Cost]) return nil } netCost, err := strconv.ParseFloat(record[bep.NetCost], 64) if err != nil { log.Errorf("failed to parse net cost: '%s'", record[bep.NetCost]) return nil } additionalInfo := make(map[string]any) additionalInfoJson := encloseInBrackets(record[bep.AdditionalInfo]) if additionalInfoJson != "" { err = json.Unmarshal([]byte(additionalInfoJson), &additionalInfo) if err != nil { log.Errorf("Could not parse additional information %s, with Error: %s", additionalInfoJson, err.Error()) } } tags := make(map[string]string) tagJson := encloseInBrackets(record[bep.Tags]) if tagJson != "" { tagsAny := make(map[string]any) err = json.Unmarshal([]byte(tagJson), &tagsAny) if err != nil { log.Errorf("Could not parse tags: %v, with Error: %s", tagJson, err.Error()) } for name, value := range tagsAny { if valueStr, ok := value.(string); ok && valueStr != "" { tags[name] = valueStr } } } return &BillingRowValues{ Date: usageDate, MeterCategory: record[bep.MeterCategory], SubscriptionID: record[bep.SubscriptionID], InvoiceEntityID: record[bep.InvoiceEntityID], InstanceID: record[bep.InstanceID], Service: record[bep.Service], Tags: tags, AdditionalInfo: additionalInfo, Cost: cost, NetCost: netCost, } } // enclose json strings in brackets if they are missing func encloseInBrackets(jsonString string) string { if jsonString == "" || (jsonString[0] == '{' && jsonString[len(jsonString)-1] == '}') { return jsonString } return fmt.Sprintf("{%s}", jsonString) } // isVMSSShared represents a bool that lets you know while setting providerID we were // able to get the actual VMName associated with a VM of a group of VMs in VMSS. func AzureSetProviderID(abv *BillingRowValues) (providerID string, isVMSSShared bool) { category := SelectAzureCategory(abv.MeterCategory) if value, ok := abv.AdditionalInfo["VMName"]; ok { return "azure://" + resourceGroupToLowerCase(abv.InstanceID) + getVMNumberForVMSS(fmt.Sprintf("%v", value)), false } else if value, ok := abv.AdditionalInfo["VmName"]; ok { return "azure://" + resourceGroupToLowerCase(abv.InstanceID) + getVMNumberForVMSS(fmt.Sprintf("%v", value)), false } else if value2, ook := abv.AdditionalInfo["IpAddress"]; ook && abv.MeterCategory == "Virtual Network" { return fmt.Sprintf("%v", value2), false } if category == opencost.StorageCategory || (category == opencost.NetworkCategory && abv.MeterCategory == "Bandwidth") { if value2, ok2 := abv.Tags["creationSource"]; ok2 { creationSource := fmt.Sprintf("%v", value2) return strings.TrimPrefix(creationSource, "aks-"), true } else if value2, ok2 := abv.Tags["aks-managed-creationSource"]; ok2 { creationSource := fmt.Sprintf("%v", value2) return strings.TrimPrefix(creationSource, "vmssclient-"), true } else { return getSubStringAfterFinalSlash(abv.InstanceID), true } } return "azure://" + resourceGroupToLowerCase(abv.InstanceID), true } func SelectAzureCategory(meterCategory string) string { if meterCategory == "Virtual Machines" || meterCategory == "Virtual Machines Licenses" { return opencost.ComputeCategory } else if meterCategory == "Storage" { return opencost.StorageCategory } else if meterCategory == "Load Balancer" || meterCategory == "Bandwidth" || meterCategory == "Virtual Network" { return opencost.NetworkCategory } else { return opencost.OtherCategory } } func resourceGroupToLowerCase(providerID string) string { var sb strings.Builder for matchNum, group := range groupRegex.FindAllString(providerID, -1) { if matchNum == 3 { sb.WriteString(strings.ToLower(group)) } else { sb.WriteString(group) } } return sb.String() } // Returns the substring after the final "/" in a string func getSubStringAfterFinalSlash(id string) string { index := strings.LastIndex(id, "/") if index == -1 { log.DedupedInfof(5, "azure.getSubStringAfterFinalSlash: failed to parse %s", id) return id } return id[index+1:] } func getVMNumberForVMSS(vmName string) string { vmNameSplit := strings.Split(vmName, "_") if len(vmNameSplit) > 1 { return "/virtualMachines/" + vmNameSplit[1] } return "" }