Merge pull request #2760 from nik-kc/nik/gcp_cc_resource_fallback

GCP Cloud Costs resource name fallback

pkg/cloud/gcp/bigqueryintegration.go

@@ -24,6 +24,7 @@ const (
 	SKUDescriptionColumnName     = "description"
 	LabelsColumnName             = "labels"
 	ResourceNameColumnName       = "resource"
+	ResourceGlobalNameColumnName = "global_resource"
 	CostColumnName               = "cost"
 	ListCostColumnName           = "list_cost"
 	CreditsColumnName            = "credits"
@@ -46,6 +47,7 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		fmt.Sprintf("service.description as %s", ServiceDescriptionColumnName),
 		fmt.Sprintf("sku.description as %s", SKUDescriptionColumnName),
 		fmt.Sprintf("resource.name as %s", ResourceNameColumnName),
+		fmt.Sprintf("resource.global_name as %s", ResourceGlobalNameColumnName),
 		fmt.Sprintf("TO_JSON_STRING(labels) as %s", LabelsColumnName),
 		fmt.Sprintf("SUM(cost) as %s", CostColumnName),
 		fmt.Sprintf("SUM(cost_at_list) as %s", ListCostColumnName),
@@ -60,6 +62,7 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		SKUDescriptionColumnName,
 		LabelsColumnName,
 		ResourceNameColumnName,
+		ResourceGlobalNameColumnName,
 	}
 
 	whereConjuncts := GetWhereConjuncts(start, end)

pkg/cloud/gcp/bigqueryintegration_types.go

@@ -113,6 +113,26 @@ func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema
 			}
 
 			properties.ProviderID = ParseProviderID(resource)
+		case ResourceGlobalNameColumnName:
+			// skip if we already got ProviderID from resource.name, as resource.global_name is a fallback for when
+			// resource.name is null
+			if len(properties.ProviderID) > 0 {
+				continue
+			}
+
+			resourceGlobalNameValue := values[i]
+			if resourceGlobalNameValue == nil {
+				properties.ProviderID = ""
+				continue
+			}
+			resourceGlobalName, ok := resourceGlobalNameValue.(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ResourceGlobalNameColumnName, values[i])
+				properties.ProviderID = ""
+				continue
+			}
+
+			properties.ProviderID = ParseProviderID(resourceGlobalName)
 		case CostColumnName:
 			costValue, ok := values[i].(float64)
 			if !ok {

pkg/cloud/gcp/bigqueryintegration_types_test.go

@@ -0,0 +1,75 @@
+package gcp
+
+import (
+	"testing"
+	"time"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/opencost/opencost/core/pkg/opencost"
+)
+
+func Test_Load_ResourceFallback(t *testing.T) {
+	schema := bigquery.Schema{
+		&bigquery.FieldSchema{
+			Name: UsageDateColumnName,
+		},
+		&bigquery.FieldSchema{
+			Name: ResourceNameColumnName,
+		},
+		&bigquery.FieldSchema{
+			Name: ResourceGlobalNameColumnName,
+		},
+	}
+
+	testCases := map[string]struct {
+		values             []bigquery.Value
+		expectedProviderID string
+	}{
+		"no data": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value(nil),
+				bigquery.Value(nil),
+			},
+			expectedProviderID: "",
+		},
+		"resource name only": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value("resource_name"),
+				bigquery.Value(nil),
+			},
+			expectedProviderID: "resource_name",
+		},
+		"resource global name only": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value(nil),
+				bigquery.Value("resource_global_name"),
+			},
+			expectedProviderID: "resource_global_name",
+		},
+		"resource name and global name": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value("resource_name"),
+				bigquery.Value("resource_global_name"),
+			},
+			expectedProviderID: "resource_name",
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			ccl := CloudCostLoader{
+				CloudCost: &opencost.CloudCost{},
+			}
+
+			err := ccl.Load(testCase.values, schema)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if ccl.CloudCost.Properties.ProviderID != testCase.expectedProviderID {
+				t.Errorf("Incorrect result, actual ProviderID: %s, expected: %s", ccl.CloudCost.Properties.ProviderID, testCase.expectedProviderID)
+			}
+		})
+	}
+}