Merge pull request #471 from kubecost/AjayTripathy-dedupe-logs

[WIP] initial proposal for log limiting

go.sum

@@ -490,6 +490,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK8
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU=

pkg/cloud/csvprovider.go

@@ -13,6 +13,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/kubecost/cost-model/pkg/log"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
 
@@ -138,7 +139,7 @@ func (c *CSVProvider) DownloadPricingData() error {
 		c.Pricing = pricing
 		c.PricingPV = pvpricing
 	} else {
-		klog.Infof("[WARNING] No data received from csv")
+		log.DedupedWarningf(5, "No data received from csv at %s", c.CSVLocation)
 	}
 	time.AfterFunc(refreshMinutes*time.Minute, func() { c.DownloadPricingData() })
 	return nil

pkg/costmodel/costmodel.go

@@ -1140,7 +1140,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 
 		cnode, err := cp.NodePricing(cp.GetKey(nodeLabels, n))
 		if err != nil {
-			klog.V(1).Infof("[Warning] Error getting node pricing. Error: " + err.Error())
+			log.DedupedWarningf(10, "Error getting node pricing. Error: %s", err.Error())
 			if cnode != nil {
 				nodes[name] = cnode
 				continue

pkg/log/log.go

@@ -7,18 +7,68 @@ import (
 	"k8s.io/klog"
 )
 
+var seen = make(map[string]int)
+
 func Errorf(format string, a ...interface{}) {
 	klog.Errorf(fmt.Sprintf("[Error] %s", format), a...)
 }
 
+func DedupedErrorf(logTypeLimit int, format string, a ...interface{}) {
+	timesLogged, ok := seen[format]
+	if !ok {
+		seen[format] = 1
+	} else if timesLogged == logTypeLimit {
+		seen[format]++
+		f := fmt.Sprintf("[Error] %s", format)
+		klog.Errorf("%s seen %d times, suppressing future logs", f, logTypeLimit)
+	} else if timesLogged > logTypeLimit {
+		seen[format]++
+	} else {
+		seen[format]++
+		klog.Errorf(fmt.Sprintf("[Error] %s", format), a...)
+	}
+}
+
 func Warningf(format string, a ...interface{}) {
 	klog.V(2).Infof(fmt.Sprintf("[Warning] %s", format), a...)
 }
 
+func DedupedWarningf(logTypeLimit int, format string, a ...interface{}) {
+	timesLogged, ok := seen[format]
+	if !ok {
+		seen[format] = 1
+	} else if timesLogged == logTypeLimit {
+		seen[format]++
+		f := fmt.Sprintf("[Warning] %s", format)
+		klog.Errorf("%s seen %d times, suppressing future logs", f, logTypeLimit)
+	} else if timesLogged > logTypeLimit {
+		seen[format]++
+	} else {
+		seen[format]++
+		klog.V(2).Infof(fmt.Sprintf("[Warning] %s", format), a...)
+	}
+}
+
 func Infof(format string, a ...interface{}) {
 	klog.V(3).Infof(fmt.Sprintf("[Info] %s", format), a...)
 }
 
+func DedupedInfof(logTypeLimit int, format string, a ...interface{}) {
+	timesLogged, ok := seen[format]
+	if !ok {
+		seen[format] = 1
+	} else if timesLogged == logTypeLimit {
+		seen[format]++
+		f := fmt.Sprintf("[Info] %s", format)
+		klog.Errorf("%s seen %d times, suppressing future logs", f, logTypeLimit)
+	} else if timesLogged > logTypeLimit {
+		seen[format]++
+	} else {
+		seen[format]++
+		klog.V(3).Infof(fmt.Sprintf("[Info] %s", format), a...)
+	}
+}
+
 func Profilef(format string, a ...interface{}) {
 	klog.V(3).Infof(fmt.Sprintf("[Profiler] %s", format), a...)
 }