Merge pull request #1052 from kubecost/sean/cloudETL

Sean/cloud etl

pkg/kubecost/asset.go

@@ -2893,6 +2893,63 @@ func (asr *AssetSetRange) Length() int {
 	return len(asr.assets)
 }
 
+// InsertRange merges the given AssetSetRange into the receiving one by
+// lining up sets with matching windows, then inserting each asset from
+// the given ASR into the respective set in the receiving ASR. If the given
+// ASR contains an AssetSetRange from a window that does not exist in the
+// receiving ASR, then an error is returned. However, the given ASR does not
+// need to cover the full range of the receiver.
+func (asr *AssetSetRange) InsertRange(that *AssetSetRange) error {
+	if asr == nil {
+		return fmt.Errorf("cannot insert range into nil AssetSetRange")
+	}
+
+	// keys maps window to index in asr
+	keys := map[string]int{}
+	asr.Each(func(i int, as *AssetSet) {
+		if as == nil {
+			return
+		}
+		keys[as.Window.String()] = i
+	})
+
+	// Nothing to merge, so simply return
+	if len(keys) == 0 {
+		return nil
+	}
+
+	var err error
+	that.Each(func(j int, thatAS *AssetSet) {
+		if thatAS == nil || err != nil {
+			return
+		}
+
+		// Find matching AssetSet in asr
+		i, ok := keys[thatAS.Window.String()]
+		if !ok {
+			err = fmt.Errorf("cannot merge AssetSet into window that does not exist: %s", thatAS.Window.String())
+			return
+		}
+		as, err := asr.Get(i)
+		if err != nil {
+			err = fmt.Errorf("AssetSetRange index does not exist: %d", i)
+			return
+		}
+
+		// Insert each Asset from the given set
+		thatAS.Each(func(k string, asset Asset) {
+			err = as.Insert(asset)
+			if err != nil {
+				err = fmt.Errorf("error inserting asset: %s", err)
+				return
+			}
+		})
+	})
+
+	// err might be nil
+	return err
+}
+
 func (asr *AssetSetRange) MarshalJSON() ([]byte, error) {
 	asr.RLock()
 	defer asr.RUnlock()

pkg/kubecost/cloudusage.go

@@ -0,0 +1,16 @@
+package kubecost
+
+// CloudUsage is temporarily aliased as the Cloud Asset type until further infrastructure and pages can be built to support its usage
+type CloudUsage = Cloud
+
+// CloudUsageSet is temporarily aliased as the AssetSet until further infrastructure and pages can be built to support its usage
+type CloudUsageSet = AssetSet
+
+// CloudUsageSetRange is temporarily aliased as the AssetSetRange until further infrastructure and pages can be built to support its usage
+type CloudUsageSetRange = AssetSetRange
+
+// CloudUsageAggregationOptions is temporarily aliased as the AssetAggregationOptions until further infrastructure and pages can be built to support its usage
+type CloudUsageAggregationOptions = AssetAggregationOptions
+
+// CloudUsageMatchFunc is temporarily aliased as the AssetMatchFunc until further infrastructure and pages can be built to support its usage
+type CloudUsageMatchFunc = AssetMatchFunc

pkg/kubecost/status.go

@@ -38,6 +38,7 @@ type FileStatus struct {
 // CloudStatus describes CloudStore metadata
 type CloudStatus struct {
 	CloudConnectionStatus string                `json:"cloudConnectionStatus"`
+	CloudUsage            *CloudAssetStatus     `json:"cloudUsage,omitempty"`
 	CloudAssets           *CloudAssetStatus     `json:"cloudAssets,omitempty"`
 	Reconciliation        *ReconciliationStatus `json:"reconciliation,omitempty"`
 }

pkg/util/stringutil/stringutil.go

@@ -76,3 +76,34 @@ func FormatUTCOffset(dur time.Duration) string {
 
 	return fmt.Sprintf("%s%02d:%02d", utcOffSig, utcOffHrs, utcOffMin)
 }
+
+// StringSlicesEqual checks if two string slices with arbitrary order have the same elements
+func StringSlicesEqual(left, right []string) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	// Build maps for each slice that counts each unique instance
+	leftMap := make(map[string]int, len(left))
+	for _, str := range left {
+		count, ok := leftMap[str]; if ok {
+			leftMap[str] = count + 1
+		} else {
+			leftMap[str] = 1
+		}
+	}
+	rightMap := make(map[string]int, len(right))
+	for _, str := range right {
+		count, ok := rightMap[str]; if ok {
+			rightMap[str] = count + 1
+		} else {
+			rightMap[str] = 1
+		}
+	}
+	// check that each unique key has the same count in each slice
+	for key, count := range leftMap {
+		if rightMap[key] != count {
+			return false
+		}
+	}
+	return true
+}