Rate-limit WAL failure logs and extend redaction

Azure storage errors include a per-request id, so logging whenever the
error text changed logged on every scrape during an outage. Failures now
log at error level when writes start failing and at most every 10 minutes
while they continue, and the recorded error is truncated to 512 bytes.

RedactURLs also covers header and JSON forms (key: value, "key":"value")
and password, client_secret and refresh_token values. The tail gap help
text notes that it includes normal restart downtime and covers the whole
retention window when nothing was restored.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Signed-off-by: peatey <warwick@automatic.systems>

core/pkg/util/stringutil/redact.go

@@ -14,7 +14,8 @@ var userInfoPattern = regexp.MustCompile(`^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/]*@`)
 
 // secretParamPattern matches the values of well known signature and credential parameters used by
 // cloud storage presigned URLs and connection strings, wherever they appear.
-var secretParamPattern = regexp.MustCompile(`(?i)\b(sig|signature|x-amz-signature|x-amz-credential|x-amz-security-token|x-goog-signature|x-goog-credential|accountkey|sharedaccesssignature|access_token|token)=[^&;\s"']+`)
+// Values may follow = (query strings, connection strings), : (headers) or ":" (JSON).
+var secretParamPattern = regexp.MustCompile(`(?i)\b(sig|signature|x-amz-signature|x-amz-credential|x-amz-security-token|x-goog-signature|x-goog-credential|accountkey|sharedaccesssignature|access_token|refresh_token|client_secret|password|token)("?\s*[:=]\s*"?)[^&;,\s"']+`)
 
 // RedactURLs removes query strings, fragments and user info from any URLs contained in s, and the
 // values of well known signature and credential parameters anywhere in s, so that error messages
@@ -28,5 +29,5 @@ func RedactURLs(s string) string {
 		}
 		return u
 	})
-	return secretParamPattern.ReplaceAllString(s, "${1}=REDACTED")
+	return secretParamPattern.ReplaceAllString(s, "${1}${2}REDACTED")
 }

modules/collector-source/pkg/metric/walinator.go

@@ -37,10 +37,19 @@ type Walinator struct {
 	limitResolution *util.Resolution
 	updater         Updater
 
-	statusLock sync.Mutex
-	status     source.WALStatus
+	statusLock     sync.Mutex
+	status         source.WALStatus
+	lastFailureLog time.Time
 }
 
+const (
+	// failureLogInterval is the minimum interval between error logs while wal writes keep failing
+	failureLogInterval = 10 * time.Minute
+
+	// maxStatusErrorLength truncates errors recorded in the wal status
+	maxStatusErrorLength = 512
+)
+
 func NewWalinator(
 	clusterID string,
 	applicationName string,
@@ -261,9 +270,15 @@ func (w *Walinator) recordExport(err error) {
 	}
 
 	msg := stringutil.RedactURLs(err.Error())
-	// log at error level when writes start failing or the cause changes, not on every scrape
-	if w.status.ConsecutiveExportFailures == 0 || msg != w.status.LastExportError {
-		log.Errorf("failed to export update results: %s", msg)
+	if len(msg) > maxStatusErrorLength {
+		msg = msg[:maxStatusErrorLength] + "..."
+	}
+
+	// log at error level when writes start failing and periodically while they keep failing, not on
+	// every scrape
+	if w.status.ConsecutiveExportFailures == 0 || now.Sub(w.lastFailureLog) >= failureLogInterval {
+		log.Errorf("failed to export update results (%d consecutive failures): %s", w.status.ConsecutiveExportFailures+1, msg)
+		w.lastFailureLog = now
 	} else {
 		log.Debugf("failed to export update results: %s", msg)
 	}

pkg/metrics/walmetrics.go

@@ -59,7 +59,7 @@ var walMetrics = []walMetric{
 		"Unix time of the newest collector WAL object applied during the startup restore, 0 if none.", prometheus.GaugeValue,
 		func(s source.WALStatus) float64 { return unixOrZero(s.RestoreNewest.Unix(), s.RestoreNewest.IsZero()) }),
 	newWALMetric("opencost_wal_restore_tail_gap_seconds",
-		"Interval between the newest restored collector WAL object and the start of the restore: history not persisted before the restart.", prometheus.GaugeValue,
+		"Interval between the newest restored collector WAL object and the start of the restore: history not persisted before the restart. Includes the downtime of a normal restart; covers the whole retention window when nothing was restored (e.g. a first install).", prometheus.GaugeValue,
 		func(s source.WALStatus) float64 { return s.RestoreTailGap.Seconds() }),
 	newWALMetric("opencost_wal_restore_largest_gap_seconds",
 		"Largest interval between consecutive restored collector WAL objects. Values well above the scrape interval indicate history that was never persisted or could not be restored.", prometheus.GaugeValue,

pkg/metrics/walmetrics_test.go

@@ -54,7 +54,7 @@ opencost_wal_restore_largest_gap_seconds 600
 # HELP opencost_wal_restore_newest_timestamp_seconds Unix time of the newest collector WAL object applied during the startup restore, 0 if none.
 # TYPE opencost_wal_restore_newest_timestamp_seconds gauge
 opencost_wal_restore_newest_timestamp_seconds 1.699999e+09
-# HELP opencost_wal_restore_tail_gap_seconds Interval between the newest restored collector WAL object and the start of the restore: history not persisted before the restart.
+# HELP opencost_wal_restore_tail_gap_seconds Interval between the newest restored collector WAL object and the start of the restore: history not persisted before the restart. Includes the downtime of a normal restart; covers the whole retention window when nothing was restored (e.g. a first install).
 # TYPE opencost_wal_restore_tail_gap_seconds gauge
 opencost_wal_restore_tail_gap_seconds 1800
 # HELP opencost_wal_restore_list_failed 1 if the collector WAL startup restore could not list stored objects, meaning nothing was restored.