add project, cluster, and user-id to all request endpoint spans automatically (#3095)

api/server/router/middleware/hydrate_trace.go

@@ -0,0 +1,21 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/porter-dev/porter/internal/telemetry"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// HydrateTraces pulls related IDs from requests, and puts them into a span which already exists.
+// If no span already exists, these attibutes will not be populated. This should not be used as a replacement for creating your own spans.
+// This should be added as the last middleware in the chain, so that it can pull IDs from the request context.
+func HydrateTraces(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		span := trace.SpanFromContext(ctx)
+		telemetry.AddKnownContextVariablesToSpan(ctx, span)
+		r = r.Clone(ctx)
+		next.ServeHTTP(w, r)
+	})
+}

api/server/router/router.go

@@ -22,11 +22,6 @@ import (
 
 func NewAPIRouter(config *config.Config) *chi.Mux {
 	r := chi.NewRouter()
-	r.Use(otelchi.Middleware(
-		"porter-server-middleware",
-		otelchi.WithRequestMethodInSpanName(true),
-		otelchi.WithChiRoutes(r),
-	))
 
 	endpointFactory := shared.NewAPIObjectEndpointFactory(config)
 
@@ -67,11 +62,11 @@ func NewAPIRouter(config *config.Config) *chi.Mux {
 	}
 
 	r.Route("/api", func(r chi.Router) {
-		// set panic middleware for all API endpoints to catch panics
-		r.Use(panicMW.Middleware)
-
-		// set the content type for all API endpoints and log all request info)
-		r.Use(middleware.ContentTypeJSON)
+		r.Use(
+			otelchi.Middleware("porter-server-middleware", otelchi.WithRequestMethodInSpanName(true), otelchi.WithChiRoutes(r)),
+			panicMW.Middleware,
+			middleware.ContentTypeJSON,
+		)
 
 		baseRoutes := baseRegisterer.GetRoutes(
 			r,
@@ -116,11 +111,11 @@ func NewAPIRouter(config *config.Config) *chi.Mux {
 	})
 
 	r.Route("/api/v1", func(r chi.Router) {
-		// set panic middleware for all API endpoints to catch panics
-		r.Use(panicMW.Middleware)
-
-		// set the content type for all API endpoints and log all request info
-		r.Use(middleware.ContentTypeJSON)
+		r.Use(
+			otelchi.Middleware("porter-server-middleware", otelchi.WithRequestMethodInSpanName(true), otelchi.WithChiRoutes(r)),
+			panicMW.Middleware,
+			middleware.ContentTypeJSON,
+		)
 
 		var allRoutes []*router.Route
 
@@ -300,10 +295,11 @@ func registerRoutes(config *config.Config, routes []*router.Route) {
 
 		if route.Endpoint.Metadata.CheckUsage && config.ServerConf.UsageTrackingEnabled {
 			usageMW := middleware.NewUsageMiddleware(config, route.Endpoint.Metadata.UsageMetric)
-
 			atomicGroup.Use(usageMW.Middleware)
 		}
 
+		atomicGroup.Use(middleware.HydrateTraces)
+
 		atomicGroup.Method(
 			string(route.Endpoint.Metadata.Method),
 			route.Endpoint.Metadata.Path.RelativePath,

internal/telemetry/span.go

@@ -10,6 +10,8 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/internal/models"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
@@ -19,33 +21,29 @@ import (
 // AttributeKey helps enforce consistent naming conventions for attribute key
 type AttributeKey string
 
-const (
-	// AttributeKeyUser refers to a Porter user
-	AttributeKeyUser AttributeKey = "user"
-	// AttributeKeyProject refers to a Porter project
-	AttributeKeyProject AttributeKey = "project"
-	// AttributeKeyCluster refers to a Porter cluster
-	AttributeKeyCluster AttributeKey = "cluster"
-)
-
 // NewSpan is a convenience function for creating a new span, with a Porter-namespaced name to avoid conflicts.
 // Any commonly used variables in the context, will be added to the span such as clusterID, projectID.
 // When using this function, make sure to call `defer span.End()` immediately after, to avoid lost spans.
 func NewSpan(ctx context.Context, name string) (context.Context, trace.Span) {
 	ctx, span := otel.Tracer("").Start(ctx, prefixSpanKey(name))
+	AddKnownContextVariablesToSpan(ctx, span)
+	return ctx, span
+}
 
-	// if user, ok := UserFromContext(ctx); ok {
-	// 	WithAttributes(span, AttributeKV{Key: AttributeKeyUser, Value: user.ID})
-	// }
+func AddKnownContextVariablesToSpan(ctx context.Context, span trace.Span) {
+	user, ok := ctx.Value(types.UserScope).(*models.User)
+	if ok {
+		WithAttributes(span, AttributeKV{Key: "user-id", Value: user.ID})
+	}
 
-	// WithAttributes(
-	// 	span,
-	// 	// TODO: find out where these context keys are actually stored. I believe that these are scopes, not context keys
-	// 	AttributeKV{Key: AttributeKeyCluster, Value: stringFromContext(ctx, types.ClusterScope)},
-	// 	AttributeKV{Key: AttributeKeyProject, Value: stringFromContext(ctx, types.ProjectScope)},
-	// )
+	cluster, ok := ctx.Value(types.ClusterScope).(*models.Cluster)
+	if ok {
+		WithAttributes(span, AttributeKV{Key: "cluster-id", Value: cluster.ID})
+	}
 
-	return ctx, span
+	if project, ok := ctx.Value(types.ProjectScope).(*models.Project); ok {
+		WithAttributes(span, AttributeKV{Key: "project-id", Value: project.ID})
+	}
 }
 
 // AttributeKV is a wrapper for otel attributes KV