Merge remote-tracking branch 'upstream/develop' into nick/orphaned-resources-gcp-aws

pkg/costmodel/allocation.go

@@ -58,6 +58,13 @@ const (
 	queryFmtLBActiveMins             = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]`
 )
 
+// Constants for Network Cost Subtype
+const (
+	networkCrossZoneCost   = "NetworkCrossZoneCost"
+	networkCrossRegionCost = "NetworkCrossRegionCost"
+	networkInternetCost    = "NetworkInternetCost"
+)
+
 // CanCompute should return true if CostModel can act as a valid source for the
 // given time range. In the case of CostModel we want to attempt to compute as
 // long as the range starts in the past. If the CostModel ends up not having
@@ -493,9 +500,9 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	applyRAMBytesUsedMax(podMap, resRAMUsageMax, podUIDKeyMap)
 	applyGPUsAllocated(podMap, resGPUsRequested, resGPUsAllocated, podUIDKeyMap)
 	applyNetworkTotals(podMap, resNetTransferBytes, resNetReceiveBytes, podUIDKeyMap)
-	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB, podUIDKeyMap)
-	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB, podUIDKeyMap)
-	applyNetworkAllocation(podMap, resNetInternetGiB, resNetInternetCostPerGiB, podUIDKeyMap)
+	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB, podUIDKeyMap, networkCrossZoneCost)
+	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB, podUIDKeyMap, networkCrossRegionCost)
+	applyNetworkAllocation(podMap, resNetInternetGiB, resNetInternetCostPerGiB, podUIDKeyMap, networkInternetCost)
 
 	// In the case that a two pods with the same name had different containers,
 	// we will double-count the containers. There is no way to associate each

pkg/costmodel/allocation_helpers.go

@@ -709,7 +709,7 @@ func applyNetworkTotals(podMap map[podKey]*pod, resNetworkTransferBytes []*prom.
 	}
 }
 
-func applyNetworkAllocation(podMap map[podKey]*pod, resNetworkGiB []*prom.QueryResult, resNetworkCostPerGiB []*prom.QueryResult, podUIDKeyMap map[podKey][]podKey) {
+func applyNetworkAllocation(podMap map[podKey]*pod, resNetworkGiB []*prom.QueryResult, resNetworkCostPerGiB []*prom.QueryResult, podUIDKeyMap map[podKey][]podKey, networkCostSubType string) {
 	costPerGiBByCluster := map[string]float64{}
 
 	for _, res := range resNetworkCostPerGiB {
@@ -749,7 +749,18 @@ func applyNetworkAllocation(podMap map[podKey]*pod, resNetworkGiB []*prom.QueryR
 			for _, alloc := range thisPod.Allocations {
 				gib := res.Values[0].Value / float64(len(thisPod.Allocations))
 				costPerGiB := costPerGiBByCluster[podKey.Cluster]
-				alloc.NetworkCost = gib * costPerGiB / float64(len(pods))
+				currentNetworkSubCost := gib * costPerGiB / float64(len(pods))
+				switch networkCostSubType {
+				case networkCrossZoneCost:
+					alloc.NetworkCrossZoneCost = currentNetworkSubCost
+				case networkCrossRegionCost:
+					alloc.NetworkCrossRegionCost = currentNetworkSubCost
+				case networkInternetCost:
+					alloc.NetworkInternetCost = currentNetworkSubCost
+				default:
+					log.Warnf("CostModel.applyNetworkAllocation: unknown network subtype passed to the function: %s", networkCostSubType)
+				}
+				alloc.NetworkCost += currentNetworkSubCost
 			}
 		}
 	}

pkg/costmodel/router.go

@@ -1398,6 +1398,47 @@ func (a *Accesses) Status(w http.ResponseWriter, r *http.Request, _ httprouter.P
 	}
 }
 
+type LogLevelRequestResponse struct {
+	Level string `json:"level"`
+}
+
+func (a *Accesses) GetLogLevel(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	level := log.GetLogLevel()
+	llrr := LogLevelRequestResponse{
+		Level: level,
+	}
+
+	body, err := json.Marshal(llrr)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("unable to retrive log level"), http.StatusInternalServerError)
+		return
+	}
+	_, err = w.Write(body)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("unable to write response: %s", body), http.StatusInternalServerError)
+		return
+	}
+}
+
+func (a *Accesses) SetLogLevel(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	params := LogLevelRequestResponse{}
+	err := json.NewDecoder(r.Body).Decode(&params)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("unable to decode request body, error: %s", err), http.StatusBadRequest)
+		return
+	}
+
+	err = log.SetLogLevel(params.Level)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("level must be a valid log level according to zerolog; level given: %s, error: %s", params.Level, err), http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
 // captures the panic event in sentry
 func capturePanicEvent(err string, stack string) {
 	msg := fmt.Sprintf("Panic: %s\nStackTrace: %s\n", err, stack)
@@ -1749,6 +1790,9 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 	a.Router.GET("/diagnostics/requestQueue", a.GetPrometheusQueueState)
 	a.Router.GET("/diagnostics/prometheusMetrics", a.GetPrometheusMetrics)
 
+	a.Router.GET("/logs/level", a.GetLogLevel)
+	a.Router.POST("/logs/level", a.SetLogLevel)
+
 	a.httpServices.RegisterAll(a.Router)
 
 	return a

pkg/kubecost/allocation.go

@@ -63,6 +63,9 @@ type Allocation struct {
 	NetworkTransferBytes       float64               `json:"networkTransferBytes"`
 	NetworkReceiveBytes        float64               `json:"networkReceiveBytes"`
 	NetworkCost                float64               `json:"networkCost"`
+	NetworkCrossZoneCost       float64               `json:"networkCrossZoneCost"`   // @bingen:field[version=16]
+	NetworkCrossRegionCost     float64               `json:"networkCrossRegionCost"` // @bingen:field[version=16]
+	NetworkInternetCost        float64               `json:"networkInternetCost"`    // @bingen:field[version=16]
 	NetworkCostAdjustment      float64               `json:"networkCostAdjustment"`
 	LoadBalancerCost           float64               `json:"loadBalancerCost"`
 	LoadBalancerCostAdjustment float64               `json:"loadBalancerCostAdjustment"`
@@ -288,6 +291,9 @@ func (a *Allocation) Clone() *Allocation {
 		NetworkTransferBytes:       a.NetworkTransferBytes,
 		NetworkReceiveBytes:        a.NetworkReceiveBytes,
 		NetworkCost:                a.NetworkCost,
+		NetworkCrossZoneCost:       a.NetworkCrossZoneCost,
+		NetworkCrossRegionCost:     a.NetworkCrossRegionCost,
+		NetworkInternetCost:        a.NetworkInternetCost,
 		NetworkCostAdjustment:      a.NetworkCostAdjustment,
 		LoadBalancerCost:           a.LoadBalancerCost,
 		LoadBalancerCostAdjustment: a.LoadBalancerCostAdjustment,
@@ -355,6 +361,15 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.NetworkCost, that.NetworkCost) {
 		return false
 	}
+	if !util.IsApproximately(a.NetworkCrossZoneCost, that.NetworkCrossZoneCost) {
+		return false
+	}
+	if !util.IsApproximately(a.NetworkCrossRegionCost, that.NetworkCrossRegionCost) {
+		return false
+	}
+	if !util.IsApproximately(a.NetworkInternetCost, that.NetworkInternetCost) {
+		return false
+	}
 	if !util.IsApproximately(a.NetworkCostAdjustment, that.NetworkCostAdjustment) {
 		return false
 	}
@@ -771,6 +786,9 @@ func (a *Allocation) add(that *Allocation) {
 	a.GPUCost += that.GPUCost
 	a.RAMCost += that.RAMCost
 	a.NetworkCost += that.NetworkCost
+	a.NetworkCrossZoneCost += that.NetworkCrossZoneCost
+	a.NetworkCrossRegionCost += that.NetworkCrossRegionCost
+	a.NetworkInternetCost += that.NetworkInternetCost
 	a.LoadBalancerCost += that.LoadBalancerCost
 	a.SharedCost += that.SharedCost
 	a.ExternalCost += that.ExternalCost

pkg/kubecost/allocation_json.go

@@ -29,6 +29,9 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "networkTransferBytes", a.NetworkTransferBytes, ",")
 	jsonEncodeFloat64(buffer, "networkReceiveBytes", a.NetworkReceiveBytes, ",")
 	jsonEncodeFloat64(buffer, "networkCost", a.NetworkCost, ",")
+	jsonEncodeFloat64(buffer, "networkCrossZoneCost", a.NetworkCrossZoneCost, ",")
+	jsonEncodeFloat64(buffer, "networkCrossRegionCost", a.NetworkCrossRegionCost, ",")
+	jsonEncodeFloat64(buffer, "networkInternetCost", a.NetworkInternetCost, ",")
 	jsonEncodeFloat64(buffer, "networkCostAdjustment", a.NetworkCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "loadBalancerCost", a.LoadBalancerCost, ",")
 	jsonEncodeFloat64(buffer, "loadBalancerCostAdjustment", a.LoadBalancerCostAdjustment, ",")

pkg/kubecost/bingen.go

@@ -45,7 +45,7 @@ package kubecost
 // @bingen:end
 
 // Allocation Version Set: Includes Allocation pipeline specific resources
-// @bingen:set[name=Allocation,version=15]
+// @bingen:set[name=Allocation,version=16]
 // @bingen:generate:Allocation
 // @bingen:generate[stringtable]:AllocationSet
 // @bingen:generate:AllocationSetRange

pkg/kubecost/kubecost_codecs.go

@@ -40,7 +40,7 @@ const (
 	AssetsCodecVersion uint8 = 18
 
 	// AllocationCodecVersion is used for any resources listed in the Allocation version set
-	AllocationCodecVersion uint8 = 15
+	AllocationCodecVersion uint8 = 16
 
 	// AuditCodecVersion is used for any resources listed in the Audit version set
 	AuditCodecVersion uint8 = 1
@@ -700,6 +700,9 @@ func (target *Allocation) MarshalBinaryWithContext(ctx *EncodingContext) (err er
 	buff.WriteFloat64(target.NetworkTransferBytes)       // write float64
 	buff.WriteFloat64(target.NetworkReceiveBytes)        // write float64
 	buff.WriteFloat64(target.NetworkCost)                // write float64
+	buff.WriteFloat64(target.NetworkCrossZoneCost)       // write float64
+	buff.WriteFloat64(target.NetworkCrossRegionCost)     // write float64
+	buff.WriteFloat64(target.NetworkInternetCost)        // write float64
 	buff.WriteFloat64(target.NetworkCostAdjustment)      // write float64
 	buff.WriteFloat64(target.LoadBalancerCost)           // write float64
 	buff.WriteFloat64(target.LoadBalancerCostAdjustment) // write float64
@@ -908,32 +911,59 @@ func (target *Allocation) UnmarshalBinaryWithContext(ctx *DecodingContext) (err
 	y := buff.ReadFloat64() // read float64
 	target.NetworkCost = y
 
-	aa := buff.ReadFloat64() // read float64
-	target.NetworkCostAdjustment = aa
+	// field version check
+	if uint8(16) <= version {
+		aa := buff.ReadFloat64() // read float64
+		target.NetworkCrossZoneCost = aa
 
-	bb := buff.ReadFloat64() // read float64
-	target.LoadBalancerCost = bb
+	} else {
+		target.NetworkCrossZoneCost = float64(0) // default
+	}
+
+	// field version check
+	if uint8(16) <= version {
+		bb := buff.ReadFloat64() // read float64
+		target.NetworkCrossRegionCost = bb
+
+	} else {
+		target.NetworkCrossRegionCost = float64(0) // default
+	}
+
+	// field version check
+	if uint8(16) <= version {
+		cc := buff.ReadFloat64() // read float64
+		target.NetworkInternetCost = cc
 
-	cc := buff.ReadFloat64() // read float64
-	target.LoadBalancerCostAdjustment = cc
+	} else {
+		target.NetworkInternetCost = float64(0) // default
+	}
+
+	dd := buff.ReadFloat64() // read float64
+	target.NetworkCostAdjustment = dd
+
+	ee := buff.ReadFloat64() // read float64
+	target.LoadBalancerCost = ee
+
+	ff := buff.ReadFloat64() // read float64
+	target.LoadBalancerCostAdjustment = ff
 
 	// --- [begin][read][alias](PVAllocations) ---
-	var dd map[PVKey]*PVAllocation
+	var gg map[PVKey]*PVAllocation
 	if buff.ReadUInt8() == uint8(0) {
-		dd = nil
+		gg = nil
 	} else {
 		// --- [begin][read][map](map[PVKey]*PVAllocation) ---
-		ff := buff.ReadInt() // map len
-		ee := make(map[PVKey]*PVAllocation, ff)
-		for i := 0; i < ff; i++ {
+		kk := buff.ReadInt() // map len
+		hh := make(map[PVKey]*PVAllocation, kk)
+		for i := 0; i < kk; i++ {
 			// --- [begin][read][struct](PVKey) ---
-			gg := &PVKey{}
+			ll := &PVKey{}
 			buff.ReadInt() // [compatibility, unused]
-			errE := gg.UnmarshalBinaryWithContext(ctx)
+			errE := ll.UnmarshalBinaryWithContext(ctx)
 			if errE != nil {
 				return errE
 			}
-			v := *gg
+			v := *ll
 			// --- [end][read][struct](PVKey) ---
 
 			var z *PVAllocation
@@ -941,60 +971,60 @@ func (target *Allocation) UnmarshalBinaryWithContext(ctx *DecodingContext) (err
 				z = nil
 			} else {
 				// --- [begin][read][struct](PVAllocation) ---
-				hh := &PVAllocation{}
+				mm := &PVAllocation{}
 				buff.ReadInt() // [compatibility, unused]
-				errF := hh.UnmarshalBinaryWithContext(ctx)
+				errF := mm.UnmarshalBinaryWithContext(ctx)
 				if errF != nil {
 					return errF
 				}
-				z = hh
+				z = mm
 				// --- [end][read][struct](PVAllocation) ---
 
 			}
-			ee[v] = z
+			hh[v] = z
 		}
-		dd = ee
+		gg = hh
 		// --- [end][read][map](map[PVKey]*PVAllocation) ---
 
 	}
-	target.PVs = PVAllocations(dd)
+	target.PVs = PVAllocations(gg)
 	// --- [end][read][alias](PVAllocations) ---
 
-	kk := buff.ReadFloat64() // read float64
-	target.PVCostAdjustment = kk
-
-	ll := buff.ReadFloat64() // read float64
-	target.RAMByteHours = ll
-
-	mm := buff.ReadFloat64() // read float64
-	target.RAMBytesRequestAverage = mm
-
 	nn := buff.ReadFloat64() // read float64
-	target.RAMBytesUsageAverage = nn
+	target.PVCostAdjustment = nn
 
 	oo := buff.ReadFloat64() // read float64
-	target.RAMCost = oo
+	target.RAMByteHours = oo
 
 	pp := buff.ReadFloat64() // read float64
-	target.RAMCostAdjustment = pp
+	target.RAMBytesRequestAverage = pp
 
 	qq := buff.ReadFloat64() // read float64
-	target.SharedCost = qq
+	target.RAMBytesUsageAverage = qq
 
 	rr := buff.ReadFloat64() // read float64
-	target.ExternalCost = rr
+	target.RAMCost = rr
+
+	ss := buff.ReadFloat64() // read float64
+	target.RAMCostAdjustment = ss
+
+	tt := buff.ReadFloat64() // read float64
+	target.SharedCost = tt
+
+	uu := buff.ReadFloat64() // read float64
+	target.ExternalCost = uu
 
 	if buff.ReadUInt8() == uint8(0) {
 		target.RawAllocationOnly = nil
 	} else {
 		// --- [begin][read][struct](RawAllocationOnlyData) ---
-		ss := &RawAllocationOnlyData{}
+		ww := &RawAllocationOnlyData{}
 		buff.ReadInt() // [compatibility, unused]
-		errG := ss.UnmarshalBinaryWithContext(ctx)
+		errG := ww.UnmarshalBinaryWithContext(ctx)
 		if errG != nil {
 			return errG
 		}
-		target.RawAllocationOnly = ss
+		target.RawAllocationOnly = ww
 		// --- [end][read][struct](RawAllocationOnlyData) ---
 
 	}

pkg/log/log.go

@@ -49,6 +49,22 @@ func InitLogging(showLogLevelSetMessage bool) {
 
 }
 
+func GetLogLevel() string {
+	return zerolog.GlobalLevel().String()
+}
+
+func SetLogLevel(l string) error {
+
+	level, err := zerolog.ParseLevel(l)
+	if err != nil {
+		return err
+	}
+
+	zerolog.SetGlobalLevel(level)
+	log.Info().Msg(fmt.Sprintf("log level set to %s.", l))
+	return nil
+}
+
 func Errorf(format string, a ...interface{}) {
 	log.Error().Msgf(format, a...)
 }

ui/.babelrc

@@ -1,4 +1,3 @@
 {
-  "plugins": ["@babel/plugin-transform-runtime", "@babel/plugin-proposal-class-properties"],
-  "presets": ["@babel/preset-react"]
+  "plugins": ["@babel/plugin-transform-runtime", "@babel/plugin-proposal-class-properties"]
 }

ui/.nvmrc

@@ -0,0 +1 @@
+18.3.0

ui/README.md

@@ -5,8 +5,8 @@ To manually run the OpenCost UI, follow the steps below.
 
 ## Requirements
 
-* `nodejs >= 12.0.0`
-* `npm >= 7.0.0`
+* `nodejs >= 18.3.0`
+* `npm >= 8.11.0`
 
 ## Installation & Running
 To run the UI, open a terminal to the `opencost/ui/` directory (where this README is located) and run

ui/package.json

@@ -26,7 +26,7 @@
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
     "react-router-dom": "^5.2.0",
-    "recharts": "^2.0.8"
+    "recharts": "^2.2.0"
   },
   "devDependencies": {
     "@babel/core": "^7.13.10",