Add WALStatus type and URL redaction helper

Define source.WALStatus and the optional source.WALStatusProvider interface
so data sources that persist through a write-ahead log can report whether
writes are succeeding and how complete their startup restore was. Existing
OpenCostDataSource implementations are unaffected.

Add stringutil.RedactURLs to strip query strings and user info from URLs
in error messages before they are exposed through status or diagnostics,
since storage client errors can embed presigned URLs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Signed-off-by: peatey <warwick@automatic.systems>

core/pkg/source/walstatus.go

@@ -0,0 +1,49 @@
+package source
+
+import "time"
+
+// WALStatus reports the health of a data source's write-ahead log: whether updates are being
+// persisted, and how complete the most recent restore was.
+type WALStatus struct {
+	// Enabled is false when the data source has no WAL configured; all other fields are zero.
+	Enabled bool `json:"enabled"`
+
+	// LastExportSuccess is the time of the most recent successful WAL write.
+	LastExportSuccess time.Time `json:"lastExportSuccess"`
+	// LastExportError is the most recent WAL write error, with any URL query strings and
+	// credentials removed.
+	LastExportError string `json:"lastExportError,omitempty"`
+	// LastExportErrorAt is the time of the most recent WAL write error.
+	LastExportErrorAt time.Time `json:"lastExportErrorAt"`
+	// ConsecutiveExportFailures is the number of WAL writes that have failed since the last success.
+	ConsecutiveExportFailures int `json:"consecutiveExportFailures"`
+	// ExportFailuresTotal is the number of WAL writes that have failed since start.
+	ExportFailuresTotal uint64 `json:"exportFailuresTotal"`
+
+	// RestoreCompleted is true once the startup restore has finished, whether or not it had errors.
+	RestoreCompleted bool `json:"restoreCompleted"`
+	// RestoreListError is set when the WAL objects could not be listed, meaning nothing was restored.
+	RestoreListError string `json:"restoreListError,omitempty"`
+	// RestoreObjectsSeen is the number of WAL objects inside the retention window.
+	RestoreObjectsSeen int `json:"restoreObjectsSeen"`
+	// RestoreObjectsApplied is the number of WAL objects successfully read, decoded and applied.
+	RestoreObjectsApplied int `json:"restoreObjectsApplied"`
+	// RestoreErrors is the number of WAL objects that could not be read or decoded.
+	RestoreErrors int `json:"restoreErrors"`
+	// RestoreDuration is how long the startup restore took.
+	RestoreDuration time.Duration `json:"restoreDuration"`
+	// RestoreOldest and RestoreNewest are the timestamps of the oldest and newest applied objects.
+	RestoreOldest time.Time `json:"restoreOldest"`
+	RestoreNewest time.Time `json:"restoreNewest"`
+	// RestoreLargestGap is the largest interval between consecutive applied objects, starting at
+	// RestoreLargestGapStart. A gap much larger than the scrape interval means history in that range
+	// was never persisted or could not be restored.
+	RestoreLargestGap      time.Duration `json:"restoreLargestGap"`
+	RestoreLargestGapStart time.Time     `json:"restoreLargestGapStart"`
+}
+
+// WALStatusProvider is optionally implemented by an OpenCostDataSource that persists its state
+// through a write-ahead log.
+type WALStatusProvider interface {
+	WALStatus() WALStatus
+}

core/pkg/util/stringutil/redact.go

@@ -0,0 +1,26 @@
+package stringutil
+
+import (
+	"regexp"
+)
+
+// urlPattern matches scheme://... URLs up to the next whitespace or quote.
+var urlPattern = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"']+`)
+
+// userInfoPattern matches the user:password@ portion following a URL scheme.
+var userInfoPattern = regexp.MustCompile(`^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/@]*@`)
+
+// RedactURLs removes query strings, fragments and user info from any URLs contained in s, so
+// that error messages from storage clients (which may embed presigned URLs or credentials) are
+// safe to expose through status endpoints and diagnostics.
+func RedactURLs(s string) string {
+	return urlPattern.ReplaceAllStringFunc(s, func(u string) string {
+		u = userInfoPattern.ReplaceAllString(u, "${1}REDACTED@")
+		for i, r := range u {
+			if r == '?' || r == '#' {
+				return u[:i] + "?REDACTED"
+			}
+		}
+		return u
+	})
+}

core/pkg/util/stringutil/redact_test.go

@@ -0,0 +1,18 @@
+package stringutil
+
+import "testing"
+
+func TestRedactURLs(t *testing.T) {
+	cases := map[string]string{
+		"plain error": "plain error",
+		`Put "https://b.s3.amazonaws.com/k?X-Amz-Signature=abc&X-Amz-Credential=AKIA": EOF`: `Put "https://b.s3.amazonaws.com/k?REDACTED": EOF`,
+		"dial https://user:secret@host/path failed":                                         "dial https://REDACTED@host/path failed",
+		"see https://host/a#frag and s3://b/k?sig=1":                                        "see https://host/a?REDACTED and s3://b/k?REDACTED",
+		"no query https://host/path/object.gz":                                              "no query https://host/path/object.gz",
+	}
+	for in, want := range cases {
+		if got := RedactURLs(in); got != want {
+			t.Errorf("RedactURLs(%q) = %q, want %q", in, got, want)
+		}
+	}
+}