Merge branch 'develop' into AjayTripathy-alibaba-improvements

.github/workflows/build-and-publish-release.yml

@@ -65,6 +65,13 @@ jobs:
           repository: 'opencost/opencost'
           ref: '${{ steps.branch.outputs.BRANCH_NAME }}'
           path: ./opencost
+      
+      - name: Checkout UI Repo
+        uses: actions/checkout@v4
+        with:
+          repository: 'opencost/opencost-ui'
+          ref: '${{ steps.branch.outputs.BRANCH_NAME }}'
+          path: ./opencost-ui
 
       - name: Set SHA
         id: sha
@@ -103,7 +110,7 @@ jobs:
           go-version: 'stable'
 
       - name: Set up just
-        uses: extractions/setup-just@v1
+        uses: extractions/setup-just@v2
 
       - name: Install crane
         uses: imjasonh/setup-crane@v0.3
@@ -135,3 +142,13 @@ jobs:
         #  crane copy '${{ steps.tags.outputs.IMAGE_TAG }}' '${steps.tags.outputs.IMAGE_TAG_QUAY}'
         #  crane copy '${{ steps.tags.outputs.IMAGE_TAG }}' '${steps.tags.outputs.IMAGE_TAG_LATEST_QUAY}'
         #  crane copy '${{ steps.tags.outputs.IMAGE_TAG }}' '${steps.tags.outputs.IMAGE_TAG_VERSION_QUAY}'
+
+      - name: Build and push (multiarch) OpenCost UI
+        working-directory: ./opencost-ui
+        run: |
+          just build '${{ steps.tags.outputs.IMAGE_TAG_UI }}' '${{ steps.version_number.outputs.RELEASE_VERSION }}'
+          crane copy '${{ steps.tags.outputs.IMAGE_TAG_UI }}' '${{ steps.tags.outputs.IMAGE_TAG_UI_LATEST }}'
+          crane copy '${{ steps.tags.outputs.IMAGE_TAG_UI }}' '${{ steps.tags.outputs.IMAGE_TAG_UI_VERSION }}'
+        #  crane copy '${steps.tags.outputs.IMAGE_TAG_UI}' '${steps.tags.outputs.IMAGE_TAG_UI_QUAY}'
+        #  crane copy '${steps.tags.outputs.IMAGE_TAG_UI}' '${steps.tags.outputs.IMAGE_TAG_UI_LATEST_QUAY}'
+        #  crane copy '${steps.tags.outputs.IMAGE_TAG_UI}' '${steps.tags.outputs.IMAGE_TAG_UI_VERSION_QUAY}'

.github/workflows/build-test.yaml

@@ -29,7 +29,7 @@ jobs:
           version: '25.3'
       -
         name: Install just
-        uses: extractions/setup-just@v1
+        uses: extractions/setup-just@v2
 
       - name: install protobuf-go
         run: |
@@ -49,7 +49,7 @@ jobs:
 
       -
         name: Install just
-        uses: extractions/setup-just@v1
+        uses: extractions/setup-just@v2
 
       -
         name: Install Go

pkg/cloud/aws/provider.go

@@ -1382,8 +1382,8 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 	}
 	// Throw error if public price is not found
 	if !publicPricingFound {
-		log.Errorf("Could not fetch data for \"%s\"", k.ID())
-		return nil, meta, fmt.Errorf("Could not fetch data for \"%s\"", k.ID())
+		log.Errorf("For node \"%s\", cannot find the following key in OnDemand pricing data \"%s\"", k.ID(), k.Features())
+		return nil, meta, fmt.Errorf("for node \"%s\", cannot find the following key in OnDemand pricing data \"%s\"", k.ID(), k.Features())
 	}
 
 	return &models.Node{

pkg/cloud/azure/storageconnection.go

@@ -42,6 +42,9 @@ func (sc *StorageConnection) getBlobURLTemplate() string {
 	// Use gov cloud blob url if gov is detected in AzureCloud
 	if strings.Contains(strings.ToLower(sc.Cloud), "gov") {
 		return "https://%s.blob.core.usgovcloudapi.net/%s"
+	} else if strings.Contains(strings.ToLower(sc.Cloud), "china") {
+		// Use China cloud blob url if china is detected in AzureCloud
+		return "https://%s.blob.core.chinacloudapi.cn/%s"
 	}
 	// default to Public Cloud template
 	return "https://%s.blob.core.windows.net/%s"

pkg/cloud/gcp/bigqueryintegration.go

@@ -24,6 +24,7 @@ const (
 	SKUDescriptionColumnName     = "description"
 	LabelsColumnName             = "labels"
 	ResourceNameColumnName       = "resource"
+	ResourceGlobalNameColumnName = "global_resource"
 	CostColumnName               = "cost"
 	ListCostColumnName           = "list_cost"
 	CreditsColumnName            = "credits"
@@ -46,6 +47,7 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		fmt.Sprintf("service.description as %s", ServiceDescriptionColumnName),
 		fmt.Sprintf("sku.description as %s", SKUDescriptionColumnName),
 		fmt.Sprintf("resource.name as %s", ResourceNameColumnName),
+		fmt.Sprintf("resource.global_name as %s", ResourceGlobalNameColumnName),
 		fmt.Sprintf("TO_JSON_STRING(labels) as %s", LabelsColumnName),
 		fmt.Sprintf("SUM(cost) as %s", CostColumnName),
 		fmt.Sprintf("SUM(cost_at_list) as %s", ListCostColumnName),
@@ -60,6 +62,7 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		SKUDescriptionColumnName,
 		LabelsColumnName,
 		ResourceNameColumnName,
+		ResourceGlobalNameColumnName,
 	}
 
 	whereConjuncts := GetWhereConjuncts(start, end)

pkg/cloud/gcp/bigqueryintegration_types.go

@@ -113,6 +113,26 @@ func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema
 			}
 
 			properties.ProviderID = ParseProviderID(resource)
+		case ResourceGlobalNameColumnName:
+			// skip if we already got ProviderID from resource.name, as resource.global_name is a fallback for when
+			// resource.name is null
+			if len(properties.ProviderID) > 0 {
+				continue
+			}
+
+			resourceGlobalNameValue := values[i]
+			if resourceGlobalNameValue == nil {
+				properties.ProviderID = ""
+				continue
+			}
+			resourceGlobalName, ok := resourceGlobalNameValue.(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ResourceGlobalNameColumnName, values[i])
+				properties.ProviderID = ""
+				continue
+			}
+
+			properties.ProviderID = ParseProviderID(resourceGlobalName)
 		case CostColumnName:
 			costValue, ok := values[i].(float64)
 			if !ok {

pkg/cloud/gcp/bigqueryintegration_types_test.go

@@ -0,0 +1,75 @@
+package gcp
+
+import (
+	"testing"
+	"time"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/opencost/opencost/core/pkg/opencost"
+)
+
+func Test_Load_ResourceFallback(t *testing.T) {
+	schema := bigquery.Schema{
+		&bigquery.FieldSchema{
+			Name: UsageDateColumnName,
+		},
+		&bigquery.FieldSchema{
+			Name: ResourceNameColumnName,
+		},
+		&bigquery.FieldSchema{
+			Name: ResourceGlobalNameColumnName,
+		},
+	}
+
+	testCases := map[string]struct {
+		values             []bigquery.Value
+		expectedProviderID string
+	}{
+		"no data": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value(nil),
+				bigquery.Value(nil),
+			},
+			expectedProviderID: "",
+		},
+		"resource name only": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value("resource_name"),
+				bigquery.Value(nil),
+			},
+			expectedProviderID: "resource_name",
+		},
+		"resource global name only": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value(nil),
+				bigquery.Value("resource_global_name"),
+			},
+			expectedProviderID: "resource_global_name",
+		},
+		"resource name and global name": {
+			values: []bigquery.Value{
+				bigquery.Value(time.Now()),
+				bigquery.Value("resource_name"),
+				bigquery.Value("resource_global_name"),
+			},
+			expectedProviderID: "resource_name",
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			ccl := CloudCostLoader{
+				CloudCost: &opencost.CloudCost{},
+			}
+
+			err := ccl.Load(testCase.values, schema)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if ccl.CloudCost.Properties.ProviderID != testCase.expectedProviderID {
+				t.Errorf("Incorrect result, actual ProviderID: %s, expected: %s", ccl.CloudCost.Properties.ProviderID, testCase.expectedProviderID)
+			}
+		})
+	}
+}

pkg/costmodel/costmodel.go

@@ -1168,8 +1168,11 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			// was the big thing to investigate. All the funky ratio math
 			// we were doing was messing with their default pricing. for SUSE Rancher.
 
-			// We couldn't find a gpu cost, so fix cpu and ram, then accordingly
-			log.Infof("GPU without cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
+			// We reach this when a GPU is detected on a node, but no cost for
+			// the GPU is defined in the OnDemand pricing. Calculate ratios of
+			// CPU to RAM and GPU to RAM costs, then distribute the total node
+			// cost among the CPU, RAM, and GPU.
+			log.Tracef("GPU without cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
 
 			defaultCPU, err := strconv.ParseFloat(cfg.CPU, 64)
 			if err != nil {
@@ -1261,8 +1264,10 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			newCnode.RAMBytes = fmt.Sprintf("%f", ram)
 			newCnode.GPUCost = fmt.Sprintf("%f", gpuPrice)
 		} else if newCnode.RAMCost == "" {
-			// We couldn't find a ramcost, so fix cpu and allocate ram accordingly
-			log.Debugf("No RAM cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
+			// We reach this when no RAM cost is defined in the OnDemand
+			// pricing. It calculates a cpuToRAMRatio and ramMultiple to
+			// distrubte the total node cost among CPU and RAM costs.
+			log.Tracef("No RAM cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
 
 			defaultCPU, err := strconv.ParseFloat(cfg.CPU, 64)
 			if err != nil {
@@ -1352,7 +1357,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			}
 			newCnode.RAMBytes = fmt.Sprintf("%f", ram)
 
-			log.Debugf("Computed \"%s\" RAM Cost := %v", name, newCnode.RAMCost)
+			log.Tracef("Computed \"%s\" RAM Cost := %v", name, newCnode.RAMCost)
 		}
 
 		nodes[name] = &newCnode