handle aws generated tags (#2896)

* Add support for aws generated tags for s3 and athena integrations

Signed-off-by: Sean Holcomb <seanholcomb@gmail.com>

* add consts

Signed-off-by: Sean Holcomb <seanholcomb@gmail.com>

---------

Signed-off-by: Sean Holcomb <seanholcomb@gmail.com>

pkg/cloud/aws/athenaintegration.go

@@ -14,6 +14,8 @@ import (
 )
 
 const LabelColumnPrefix = "resource_tags_user_"
+const AWSLabelColumnPrefix = "resource_tags_aws_"
+const AthenaResourceTagPrefix = "resource_tags_"
 
 // athenaDateLayout is the default AWS date format
 const AthenaDateLayout = "2006-01-02 15:04:05.000"
@@ -52,6 +54,7 @@ type AthenaQueryIndexes struct {
 	Query                  string
 	ColumnIndexes          map[string]int
 	TagColumns             []string
+	AWSTagColumns          []string
 	ListCostColumn         string
 	NetCostColumn          string
 	AmortizedNetCostColumn string
@@ -100,6 +103,10 @@ func (ai *AthenaIntegration) GetCloudCost(start, end time.Time) (*opencost.Cloud
 			groupByColumns = append(groupByColumns, quotedTag)
 			aqi.TagColumns = append(aqi.TagColumns, quotedTag)
 		}
+		if strings.HasPrefix(column, AWSLabelColumnPrefix) {
+			groupByColumns = append(groupByColumns, column)
+			aqi.AWSTagColumns = append(aqi.AWSTagColumns, column)
+		}
 	}
 	var selectColumns []string
 
@@ -347,6 +354,15 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 		}
 	}
 
+	for _, awsColumnName := range aqi.AWSTagColumns {
+		// partially remove prefix leaving "aws_"
+		labelName := strings.TrimPrefix(awsColumnName, AthenaResourceTagPrefix)
+		value := GetAthenaRowValue(row, aqi.ColumnIndexes, awsColumnName)
+		if value != "" {
+			labels[labelName] = value
+		}
+	}
+
 	invoiceEntityID := GetAthenaRowValue(row, aqi.ColumnIndexes, "bill_payer_account_id")
 	accountID := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_usage_account_id")
 	startStr := GetAthenaRowValue(row, aqi.ColumnIndexes, AthenaDateTruncColumn)

pkg/cloud/aws/s3selectintegration.go

@@ -34,6 +34,8 @@ const S3SelectNetRICost = `s."reservation/NetEffectiveCost"`
 const S3SelectNetSPCost = `s."savingsPlan/NetSavingsPlanEffectiveCost"`
 
 const S3SelectUserLabelPrefix = "resourceTags/user:"
+const S3SelectAWSLabelPrefix = "resourceTags/aws:"
+const S3SelectResourceTagsPrefix = "resourceTags/"
 
 type S3SelectIntegration struct {
 	S3SelectQuerier
@@ -122,12 +124,18 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 	// Determine which columns are user-defined tags and add those to the list
 	// of columns to query.
 	labelColumns := []string{}
+	awsLabelColumns := []string{}
 	for column := range allColumns {
 		if strings.HasPrefix(column, S3SelectUserLabelPrefix) {
 			quotedTag := fmt.Sprintf(`s."%s"`, column)
 			selectColumns = append(selectColumns, quotedTag)
 			labelColumns = append(labelColumns, quotedTag)
 		}
+		if strings.HasPrefix(column, S3SelectAWSLabelPrefix) {
+			quotedTag := fmt.Sprintf(`s."%s"`, column)
+			selectColumns = append(selectColumns, quotedTag)
+			awsLabelColumns = append(awsLabelColumns, quotedTag)
+		}
 	}
 
 	// Build map of query columns to use for parsing query
@@ -178,6 +186,17 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 					labels[labelName] = value
 				}
 			}
+			for _, awsLabelColumnName := range awsLabelColumns {
+				// remove quotes
+				labelName := strings.TrimPrefix(awsLabelColumnName, `s."`)
+				labelName = strings.TrimSuffix(labelName, `"`)
+				// partially remove prefix leaving "aws:"
+				labelName = strings.TrimPrefix(labelName, S3SelectResourceTagsPrefix)
+				value := GetCSVRowValue(row, columnIndexes, awsLabelColumnName)
+				if value != "" {
+					labels[labelName] = value
+				}
+			}
 
 			isKubernetes := 0.0
 			if itemProductCode == "AmazonEKS" || hasK8sLabel(labels) {
@@ -311,21 +330,33 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 	return ccsr, nil
 }
 
+const (
+	TagAWSEKSClusterName     = "aws:eks:cluster-name"
+	TagEKSClusterName        = "eks:cluster-name"
+	TagEKSCtlClusterName     = "alpha.eksctl.io/cluster-name"
+	TagKubernetesServiceName = "kubernetes.io/service-name"
+	TagKubernetesPVCName     = "kubernetes.io/created-for/pvc/name"
+	TagKubernetesPVName      = "kubernetes.io/created-for/pv/name"
+)
+
 // hsK8sLabel checks if the labels contain a k8s label
 func hasK8sLabel(labels opencost.CloudCostLabels) bool {
-	if _, ok := labels["eks:cluster-name"]; ok {
+	if _, ok := labels[TagAWSEKSClusterName]; ok {
+		return true
+	}
+	if _, ok := labels[TagEKSClusterName]; ok {
 		return true
 	}
-	if _, ok := labels["alpha.eksctl.io/cluster-name"]; ok {
+	if _, ok := labels[TagEKSCtlClusterName]; ok {
 		return true
 	}
-	if _, ok := labels["kubernetes.io/service-name"]; ok {
+	if _, ok := labels[TagKubernetesServiceName]; ok {
 		return true
 	}
-	if _, ok := labels["kubernetes.io/created-for/pvc/name"]; ok {
+	if _, ok := labels[TagKubernetesPVCName]; ok {
 		return true
 	}
-	if _, ok := labels["kubernetes.io/created-for/pv/name"]; ok {
+	if _, ok := labels[TagKubernetesPVName]; ok {
 		return true
 	}
 	return false