Context Propagation (#3164)

.github/golangci-lint.yaml

@@ -0,0 +1,67 @@
+---
+run:
+  timeout: 5m
+  issues-exit-code: 1
+  build-tags:
+    - codeanalysis
+
+# enable exported entity commenting lint rule
+issues:
+  exclude:
+    - EXC0012
+  exclude-use-default: false
+linters-settings:
+  revive:
+    rules:
+      - name: exported
+        severity: error
+  # gocyclo:
+  #   min-complexity: 15
+  gomoddirectives:
+    replace-local: false
+  gosec:
+    excludes:
+    - G307
+
+linters:
+  # disable all default-enabled linters so nothing is mysterious
+  disable-all: true
+  # all enabled linters found at https://golangci-lint.run/usage/linters/
+  enable:
+    - errcheck	
+    - gosimple
+    - govet	
+    - ineffassign	
+    - staticcheck
+    - typecheck
+    - unused
+    - whitespace
+    - unparam
+    - unconvert
+    - goconst
+    - misspell
+    - revive
+    - gofumpt
+    - gocyclo
+    - gomoddirectives
+    - gosec
+
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions
+  # default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+  # make issues output unique by line, default is true
+  uniq-by-line: true
+
+  # add a prefix to the output file references; default is no prefix
+  path-prefix: ""
+
+  # sorts results by: filepath, line and column
+  sort-results: false

.github/workflows/internal_tooling_stack_porter-ui.yml

@@ -25,9 +25,9 @@ jobs:
       - name: Download Go Modules
         run: go mod download
       - name: Build Server Binary
-        run: go build -ldflags="-w -s -X 'main.Version=production'" -o ./bin/app ./cmd/app
+        run: go build -ldflags="-w -s -X 'main.Version=production'" -tags ee -o ./bin/app ./cmd/app
       - name: Build Migration Binary
-        run: go build -ldflags '-w -s' -o ./bin/migrate ./cmd/migrate
+        run: go build -ldflags '-w -s' -tags ee -o ./bin/migrate ./cmd/migrate
       - name: Compress binaries
         run: |
           upx bin/* --best --lzma

.github/workflows/production.yml

@@ -25,14 +25,18 @@ jobs:
       - name: Download Go Modules
         run: go mod download
       - name: Build Server Binary
-        run: go build -ldflags="-w -s -X 'main.Version=production'" -o ./bin/app ./cmd/app
+        run: go build -ldflags="-w -s -X 'main.Version=production'" -tags ee -o ./bin/app ./cmd/app
       - name: Build Migration Binary
-        run: go build -ldflags '-w -s' -o ./bin/migrate ./cmd/migrate
+        run: go build -ldflags '-w -s' -tags ee -o ./bin/migrate ./cmd/migrate
+      - name: Compress binaries
+        run: |
+          upx bin/* --best --lzma
       - name: Store Binaries
         uses: actions/upload-artifact@v3
         with:
           name: go-binaries
           path: bin/
+          retention-days: 1
   build-npm:
     runs-on: ubuntu-latest
     steps:
@@ -55,6 +59,7 @@ jobs:
         with:
           name: npm-static-files
           path: dashboard/build/
+          retention-days: 1
   porter-deploy:
     runs-on: ubuntu-latest
     needs: [build-go, build-npm]

api/server/authz/cluster.go

@@ -18,10 +18,12 @@ import (
 	"k8s.io/client-go/dynamic"
 )
 
+type ContextKey string
+
 const (
-	KubernetesAgentCtxKey         string = "k8s-agent"
-	KubernetesDynamicClientCtxKey string = "k8s-dyn-client"
-	HelmAgentCtxKey               string = "helm-agent"
+	KubernetesAgentCtxKey         ContextKey = "k8s-agent"
+	KubernetesDynamicClientCtxKey ContextKey = "k8s-dyn-client"
+	HelmAgentCtxKey               ContextKey = "helm-agent"
 )
 
 type ClusterScopedFactory struct {
@@ -98,9 +100,18 @@ func (d *OutOfClusterAgentGetter) GetOutOfClusterConfig(cluster *models.Cluster)
 }
 
 func (d *OutOfClusterAgentGetter) GetAgent(r *http.Request, cluster *models.Cluster, namespace string) (*kubernetes.Agent, error) {
+	ctx, span := telemetry.NewSpan(r.Context(), "get-k8s-agent")
+	defer span.End()
+
+	telemetry.WithAttributes(span,
+		telemetry.AttributeKV{Key: "cluster-id", Value: cluster.ID},
+		telemetry.AttributeKV{Key: "project-id", Value: cluster.ProjectID},
+		telemetry.AttributeKV{Key: "namespace", Value: namespace},
+	)
+
 	// look for the agent in context if cluster isnt a capi cluster
 	if cluster.ProvisionedBy != "CAPI" {
-		ctxAgentVal := r.Context().Value(KubernetesAgentCtxKey)
+		ctxAgentVal := ctx.Value(KubernetesAgentCtxKey)
 
 		if ctxAgentVal != nil {
 			if agent, ok := ctxAgentVal.(*kubernetes.Agent); ok {
@@ -117,15 +128,18 @@ func (d *OutOfClusterAgentGetter) GetAgent(r *http.Request, cluster *models.Clus
 	} else {
 		ooc.DefaultNamespace = namespace
 	}
+	telemetry.WithAttributes(span,
+		telemetry.AttributeKV{Key: "default-namespace", Value: namespace},
+	)
 
-	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
+	agent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, ooc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get agent: %s", err.Error())
 	}
 
-	newCtx := context.WithValue(r.Context(), KubernetesAgentCtxKey, agent)
+	newCtx := context.WithValue(ctx, KubernetesAgentCtxKey, agent)
 
-	r = r.WithContext(newCtx)
+	r = r.Clone(newCtx)
 
 	return agent, nil
 }
@@ -139,8 +153,10 @@ func (d *OutOfClusterAgentGetter) GetHelmAgent(ctx context.Context, r *http.Requ
 		telemetry.AttributeKV{Key: "project-id", Value: cluster.ProjectID},
 	)
 
+	r = r.Clone(ctx)
+
 	// look for the agent in context
-	ctxAgentVal := r.Context().Value(HelmAgentCtxKey)
+	ctxAgentVal := ctx.Value(HelmAgentCtxKey)
 
 	if ctxAgentVal != nil {
 		telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "agent-from-context", Value: true})

api/server/handlers/cluster/get_kubeconfig.go

@@ -1,10 +1,7 @@
 package cluster
 
 import (
-	"context"
 	"encoding/base64"
-	"errors"
-	"fmt"
 	"net/http"
 
 	"github.com/bufbuild/connect-go"
@@ -16,6 +13,7 @@ import (
 	"github.com/porter-dev/porter/api/server/shared/config"
 	"github.com/porter-dev/porter/api/types"
 	"github.com/porter-dev/porter/internal/models"
+	"github.com/porter-dev/porter/internal/telemetry"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -35,36 +33,40 @@ func NewGetTemporaryKubeconfigHandler(
 }
 
 func (c *GetTemporaryKubeconfigHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx, span := telemetry.NewSpan(r.Context(), "serve-get-temporary-kubeconfig")
+	defer span.End()
+
 	if c.Config().ServerConf.DisableTemporaryKubeconfig {
-		c.HandleAPIError(w, r, apierrors.NewErrNotFound(
-			errors.New("temporary kubeconfig generation is disabled on this instance"),
-		))
+		e := telemetry.Error(ctx, span, nil, "temporary kubeconfig generation is disabled on this instance")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusNotFound))
 		return
 	}
-	ctx := r.Context()
 
 	cluster, _ := ctx.Value(types.ClusterScope).(*models.Cluster)
 
 	outOfClusterConfig := c.GetOutOfClusterConfig(cluster)
 
 	if cluster.ProvisionedBy == "CAPI" {
-		kubeconfigResp, err := c.Config().ClusterControlPlaneClient.KubeConfigForCluster(context.Background(), connect.NewRequest(
+		kubeconfigResp, err := c.Config().ClusterControlPlaneClient.KubeConfigForCluster(ctx, connect.NewRequest(
 			&porterv1.KubeConfigForClusterRequest{
 				ProjectId: int64(cluster.ProjectID),
 				ClusterId: int64(cluster.ID),
 			},
 		))
 		if err != nil {
-			c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error getting temporary capi config: %w", err)))
+			e := telemetry.Error(ctx, span, err, "error getting temporary capi config")
+			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
 			return
 		}
 		if kubeconfigResp.Msg == nil {
-			c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error reading temporary capi config: %w", err)))
+			e := telemetry.Error(ctx, span, err, "error reading temporary capi config")
+			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
 			return
 		}
 		b64, err := base64.StdEncoding.DecodeString(kubeconfigResp.Msg.KubeConfig)
 		if err != nil {
-			c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("unable to decode base64 kubeconfig: %w", err)))
+			e := telemetry.Error(ctx, span, err, "unable to decode base64 kubeconfig")
+			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
 			return
 		}
 		res := &types.GetTemporaryKubeconfigResponse{
@@ -74,7 +76,7 @@ func (c *GetTemporaryKubeconfigHandler) ServeHTTP(w http.ResponseWriter, r *http
 		return
 	}
 
-	kubeconfig, err := outOfClusterConfig.CreateRawConfigFromCluster()
+	kubeconfig, err := outOfClusterConfig.CreateRawConfigFromCluster(ctx)
 	if err != nil {
 		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
 		return

api/server/handlers/registry/get_token.go

@@ -18,6 +18,7 @@ import (
 	"github.com/porter-dev/porter/internal/models"
 	"github.com/porter-dev/porter/internal/oauth"
 	"github.com/porter-dev/porter/internal/registry"
+	"github.com/porter-dev/porter/internal/telemetry"
 
 	"github.com/aws/aws-sdk-go/aws/arn"
 )
@@ -151,7 +152,10 @@ func NewRegistryGetGCRTokenHandler(
 }
 
 func (c *RegistryGetGCRTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	proj, _ := r.Context().Value(types.ProjectScope).(*models.Project)
+	ctx, span := telemetry.NewSpan(r.Context(), "serve-registry-get-gcr-token")
+	defer span.End()
+
+	proj, _ := ctx.Value(types.ProjectScope).(*models.Project)
 
 	request := &types.GetRegistryGCRTokenRequest{}
 
@@ -162,7 +166,8 @@ func (c *RegistryGetGCRTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 	// list registries and find one that matches the region
 	regs, err := c.Repo().Registry().ListRegistriesByProjectID(proj.ID)
 	if err != nil {
-		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		e := telemetry.Error(ctx, span, err, "error listing registries by project id")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
 		return
 	}
 
@@ -173,15 +178,16 @@ func (c *RegistryGetGCRTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 		if reg.GCPIntegrationID != 0 && strings.Contains(reg.URL, request.ServerURL) {
 			_reg := registry.Registry(*reg)
 
-			oauthTok, err := _reg.GetGCRToken(c.Repo())
-
-			// if the oauth token is not nil, but the error is not nil, we still return the token
-			// but log an error
-			if oauthTok != nil && err != nil {
-				c.HandleAPIErrorNoWrite(w, r, apierrors.NewErrInternal(err))
-			} else if err != nil {
-				c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
-				return
+			oauthTok, err := _reg.GetGCRToken(ctx, c.Repo())
+			if err != nil {
+				// if the oauth token is not nil, we still return the token but log an error
+				if oauthTok == nil {
+					e := telemetry.Error(ctx, span, err, "error getting gcr token")
+					c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
+					return
+				}
+				e := telemetry.Error(ctx, span, err, "error getting gcr token, but token was returned")
+				c.HandleAPIErrorNoWrite(w, r, apierrors.NewErrInternal(e))
 			}
 
 			token = oauthTok.AccessToken
@@ -213,7 +219,10 @@ func NewRegistryGetGARTokenHandler(
 }
 
 func (c *RegistryGetGARTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	proj, _ := r.Context().Value(types.ProjectScope).(*models.Project)
+	ctx, span := telemetry.NewSpan(r.Context(), "serve-registry-get-gar-token")
+	defer span.End()
+
+	proj, _ := ctx.Value(types.ProjectScope).(*models.Project)
 
 	request := &types.GetRegistryGCRTokenRequest{}
 
@@ -224,7 +233,8 @@ func (c *RegistryGetGARTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 	// list registries and find one that matches the region
 	regs, err := c.Repo().Registry().ListRegistriesByProjectID(proj.ID)
 	if err != nil {
-		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		e := telemetry.Error(ctx, span, err, "error listing registries by project id")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
 		return
 	}
 
@@ -235,15 +245,16 @@ func (c *RegistryGetGARTokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 		if reg.GCPIntegrationID != 0 && strings.Contains(reg.URL, request.ServerURL) {
 			_reg := registry.Registry(*reg)
 
-			oauthTok, err := _reg.GetGARToken(c.Repo())
-
-			// if the oauth token is not nil, but the error is not nil, we still return the token
-			// but log an error
-			if oauthTok != nil && err != nil {
-				c.HandleAPIErrorNoWrite(w, r, apierrors.NewErrInternal(err))
-			} else if err != nil {
-				c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
-				return
+			oauthTok, err := _reg.GetGARToken(ctx, c.Repo())
+			if err != nil {
+				// if the oauth token is not nil, we still return the token but log an error
+				if oauthTok == nil {
+					e := telemetry.Error(ctx, span, err, "error getting gar token")
+					c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(e, http.StatusInternalServerError))
+					return
+				}
+				e := telemetry.Error(ctx, span, err, "error getting gar token, but token was returned")
+				c.HandleAPIErrorNoWrite(w, r, apierrors.NewErrInternal(e))
 			}
 
 			token = oauthTok.AccessToken

api/server/shared/config/env/envconfs.go

@@ -154,6 +154,8 @@ type DBConf struct {
 	SQLLite     bool   `env:"SQL_LITE,default=false"`
 	SQLLitePath string `env:"SQL_LITE_PATH,default=/porter/porter.db"`
 
+	// VaultEnabled is used to denote if Porter should use Vault for secrets management. This was previously set by 'ee' build tags
+	VaultEnabled   bool   `env:"VAULT_ENABLED,default=false"`
 	VaultPrefix    string `env:"VAULT_PREFIX,default=production"`
 	VaultAPIKey    string `env:"VAULT_API_KEY"`
 	VaultServerURL string `env:"VAULT_SERVER_URL"`

api/server/shared/config/loader/init_ee.go

@@ -38,12 +38,4 @@ func init() {
 	} else {
 		InstanceBillingManager = &billing.NoopBillingManager{}
 	}
-
-	if InstanceEnvConf.DBConf.VaultAPIKey != "" && InstanceEnvConf.DBConf.VaultServerURL != "" && InstanceEnvConf.DBConf.VaultPrefix != "" {
-		InstanceCredentialBackend = vault.NewClient(
-			InstanceEnvConf.DBConf.VaultServerURL,
-			InstanceEnvConf.DBConf.VaultAPIKey,
-			InstanceEnvConf.DBConf.VaultPrefix,
-		)
-	}
 }

api/server/shared/config/loader/loader.go

@@ -10,8 +10,6 @@ import (
 	"path/filepath"
 	"strconv"
 
-	"github.com/porter-dev/porter/internal/telemetry"
-
 	gorillaws "github.com/gorilla/websocket"
 	"github.com/porter-dev/api-contracts/generated/go/porter/v1/porterv1connect"
 	"github.com/porter-dev/porter/api/server/shared/apierrors/alerter"
@@ -19,6 +17,7 @@ import (
 	"github.com/porter-dev/porter/api/server/shared/config/env"
 	"github.com/porter-dev/porter/api/server/shared/config/envloader"
 	"github.com/porter-dev/porter/api/server/shared/websocket"
+	"github.com/porter-dev/porter/ee/integrations/vault"
 	"github.com/porter-dev/porter/internal/adapter"
 	"github.com/porter-dev/porter/internal/analytics"
 	"github.com/porter-dev/porter/internal/auth/sessionstore"
@@ -31,18 +30,16 @@ import (
 	"github.com/porter-dev/porter/internal/oauth"
 	"github.com/porter-dev/porter/internal/repository/credentials"
 	"github.com/porter-dev/porter/internal/repository/gorm"
-	"github.com/porter-dev/porter/provisioner/client"
-
+	"github.com/porter-dev/porter/internal/telemetry"
 	lr "github.com/porter-dev/porter/pkg/logger"
-
+	"github.com/porter-dev/porter/provisioner/client"
 	pgorm "gorm.io/gorm"
 )
 
 var (
-	InstanceBillingManager    billing.BillingManager
-	InstanceEnvConf           *envloader.EnvConf
-	InstanceDB                *pgorm.DB
-	InstanceCredentialBackend credentials.CredentialStorage
+	InstanceBillingManager billing.BillingManager
+	InstanceEnvConf        *envloader.EnvConf
+	InstanceDB             *pgorm.DB
 )
 
 type EnvConfigLoader struct {
@@ -72,27 +69,36 @@ func (e *EnvConfigLoader) LoadConfig() (res *config.Config, err error) {
 	envConf := InstanceEnvConf
 	sc := envConf.ServerConf
 
+	if envConf == nil {
+		return nil, errors.New("nil environment config passed to loader")
+	}
+
+	var instanceCredentialBackend credentials.CredentialStorage
+	if envConf.DBConf.VaultEnabled {
+		if envConf.DBConf.VaultAPIKey == "" || envConf.DBConf.VaultServerURL != "" || envConf.DBConf.VaultPrefix != "" {
+			return nil, errors.New("Vault is enabled but missing required environment variables [VAULT_API_KEY,VAULT_SERVER_URL,VAULT_PREFIX]")
+		}
+
+		instanceCredentialBackend = vault.NewClient(
+			envConf.DBConf.VaultServerURL,
+			envConf.DBConf.VaultAPIKey,
+			envConf.DBConf.VaultPrefix,
+		)
+	}
+
 	res = &config.Config{
 		Logger:            lr.NewConsole(sc.Debug),
 		ServerConf:        sc,
 		DBConf:            envConf.DBConf,
 		RedisConf:         envConf.RedisConf,
 		BillingManager:    InstanceBillingManager,
-		CredentialBackend: InstanceCredentialBackend,
+		CredentialBackend: instanceCredentialBackend,
 	}
 	res.Logger.Info().Msg("Loading MetadataFromConf")
 	res.Metadata = config.MetadataFromConf(envConf.ServerConf, e.version)
 	res.Logger.Info().Msg("Loaded MetadataFromConf")
 	res.DB = InstanceDB
 
-	// res.Logger.Info().Msg("Starting gorm automigrate")
-	// err = gorm.AutoMigrate(InstanceDB, sc.Debug)
-	//
-	// if err != nil {
-	//	return nil, err
-	// }
-	// res.Logger.Info().Msg("Completed gorm automigrate")
-
 	var key [32]byte
 
 	for i, b := range []byte(envConf.DBConf.EncryptionKey) {
@@ -100,7 +106,7 @@ func (e *EnvConfigLoader) LoadConfig() (res *config.Config, err error) {
 	}
 
 	res.Logger.Info().Msg("Creating new gorm repository")
-	res.Repo = gorm.NewRepository(InstanceDB, &key, InstanceCredentialBackend)
+	res.Repo = gorm.NewRepository(InstanceDB, &key, instanceCredentialBackend)
 	res.Logger.Info().Msg("Created new gorm repository")
 
 	res.Logger.Info().Msg("Creating new session store")

ee/integrations/vault/types.go

@@ -1,6 +1,3 @@
-//go:build ee
-// +build ee
-
 package vault
 
 import "github.com/porter-dev/porter/internal/repository/credentials"

ee/integrations/vault/vault.go

@@ -1,6 +1,3 @@
-//go:build ee
-// +build ee
-
 package vault
 
 import (

internal/helm/config.go

@@ -1,6 +1,7 @@
 package helm
 
 import (
+	"context"
 	"errors"
 	"io/ioutil"
 	"time"
@@ -32,7 +33,7 @@ type Form struct {
 
 // GetAgentOutOfClusterConfig creates a new Agent from outside the cluster using
 // the underlying kubernetes.GetAgentOutOfClusterConfig method
-func GetAgentOutOfClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
+func GetAgentOutOfClusterConfig(ctx context.Context, form *Form, l *logger.Logger) (*Agent, error) {
 	// create a kubernetes agent
 	conf := &kubernetes.OutOfClusterConfig{
 		Cluster:                   form.Cluster,
@@ -43,7 +44,7 @@ func GetAgentOutOfClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
 		Timeout:                   form.Timeout,
 	}
 
-	k8sAgent, err := kubernetes.GetAgentOutOfClusterConfig(conf)
+	k8sAgent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, conf)
 	if err != nil {
 		return nil, err
 	}
@@ -81,9 +82,9 @@ func GetAgentFromK8sAgent(stg string, ns string, l *logger.Logger, k8sAgent *kub
 
 // GetAgentInClusterConfig creates a new Agent from inside the cluster using
 // the underlying kubernetes.GetAgentInClusterConfig method
-func GetAgentInClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
+func GetAgentInClusterConfig(ctx context.Context, form *Form, l *logger.Logger) (*Agent, error) {
 	// create a kubernetes agent
-	k8sAgent, err := kubernetes.GetAgentInClusterConfig(form.Namespace)
+	k8sAgent, err := kubernetes.GetAgentInClusterConfig(ctx, form.Namespace)
 	if err != nil {
 		return nil, err
 	}

internal/kubernetes/agent.go

@@ -52,10 +52,13 @@ import (
 )
 
 // Agent is a Kubernetes agent for performing operations that interact with the
-// api server
+// api server. Do not create this struct directly, use NewKubernetesAgent instead.
 type Agent struct {
 	RESTClientGetter genericclioptions.RESTClientGetter
 	Clientset        kubernetes.Interface
+
+	// context is used here as a workaround since RESTClientGetter and kubernetes.Interface do not support contexts
+	context context.Context
 }
 
 type Message struct {
@@ -74,6 +77,15 @@ func (e *AuthError) Error() string {
 	return "Unauthorized error"
 }
 
+// NewKubernetesAgent creates a new agent for accessing kubernetes on a cluster
+func NewKubernetesAgent(ctx context.Context, restClientGetter genericclioptions.RESTClientGetter, clientset kubernetes.Interface) Agent {
+	return Agent{
+		RESTClientGetter: restClientGetter,
+		Clientset:        clientset,
+		context:          ctx,
+	}
+}
+
 // UpdateClientset updates the Agent's Clientset (this refreshes auth tokens)
 func (a *Agent) UpdateClientset() error {
 	restConf, err := a.RESTClientGetter.ToRESTConfig()

internal/kubernetes/config.go

@@ -3,7 +3,6 @@ package kubernetes
 import (
 	"context"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -64,18 +63,18 @@ func GetDynamicClientOutOfClusterConfig(conf *OutOfClusterConfig) (dynamic.Inter
 }
 
 // GetAgentOutOfClusterConfig creates a new Agent using the OutOfClusterConfig
-func GetAgentOutOfClusterConfig(conf *OutOfClusterConfig) (*Agent, error) {
-	ctx, span := telemetry.NewSpan(context.Background(), "get-agent-out-of-cluster-config")
+func GetAgentOutOfClusterConfig(ctx context.Context, conf *OutOfClusterConfig) (*Agent, error) {
+	ctx, span := telemetry.NewSpan(ctx, "get-agent-out-of-cluster-config")
 	defer span.End()
 
 	if conf.AllowInClusterConnections && conf.Cluster.AuthMechanism == models.InCluster {
-		return GetAgentInClusterConfig(conf.DefaultNamespace)
+		return GetAgentInClusterConfig(ctx, conf.DefaultNamespace)
 	}
 
 	var restConf *rest.Config
 
 	if conf.Cluster.ProvisionedBy == "CAPI" {
-		telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "capi-provisioned", Value: true})
+		telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "provisioner", Value: conf.Cluster.ProvisionedBy})
 
 		rc, err := restConfigForCAPICluster(ctx, conf.CAPIManagementClusterClient, *conf.Cluster)
 		if err != nil {
@@ -83,6 +82,8 @@ func GetAgentOutOfClusterConfig(conf *OutOfClusterConfig) (*Agent, error) {
 		}
 		restConf = rc
 	} else {
+		telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "provisioner", Value: "non-capi"})
+
 		rc, err := conf.ToRESTConfig()
 		if err != nil {
 			return nil, telemetry.Error(ctx, span, err, "error getting rest config")
@@ -99,7 +100,9 @@ func GetAgentOutOfClusterConfig(conf *OutOfClusterConfig) (*Agent, error) {
 		return nil, telemetry.Error(ctx, span, err, "error getting new clientset for config")
 	}
 
-	return &Agent{conf, clientset}, nil
+	agent := NewKubernetesAgent(ctx, conf, clientset)
+
+	return &agent, nil
 }
 
 // restConfigForCAPICluster gets the kubernetes rest API client for a CAPI cluster
@@ -196,21 +199,29 @@ func IsInCluster() bool {
 
 // GetAgentInClusterConfig uses the service account that kubernetes
 // gives to pods to connect
-func GetAgentInClusterConfig(namespace string) (*Agent, error) {
+func GetAgentInClusterConfig(ctx context.Context, namespace string) (*Agent, error) {
 	conf, err := rest.InClusterConfig()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error getting in cluster config: %w", err)
 	}
 
 	restClientGetter := NewRESTClientGetterFromInClusterConfig(conf, namespace)
 	clientset, err := kubernetes.NewForConfig(conf)
+	if err != nil {
+		return nil, fmt.Errorf("error getting new clientset for config: %w", err)
+	}
 
-	return &Agent{restClientGetter, clientset}, nil
+	agent := NewKubernetesAgent(ctx, restClientGetter, clientset)
+
+	return &agent, nil
 }
 
 // GetAgentTesting creates a new Agent using an optional existing storage class
+// TODO: this should be in a test package, not here.
 func GetAgentTesting(objects ...runtime.Object) *Agent {
-	return &Agent{&fakeRESTClientGetter{}, fake.NewSimpleClientset(objects...)}
+	agent := NewKubernetesAgent(context.Background(), &fakeRESTClientGetter{}, fake.NewSimpleClientset(objects...))
+
+	return &agent
 }
 
 // OutOfClusterConfig is the set of parameters required for an out-of-cluster connection.
@@ -230,11 +241,18 @@ type OutOfClusterConfig struct {
 
 // ToRESTConfig creates a kubernetes REST client factory -- it calls ClientConfig on
 // the result of ToRawKubeConfigLoader, and also adds a custom http transport layer
-// if necessary (required for GCP auth)
+// if necessary (required for GCP auth).
+// TODO: this should be split out from OutOfClusterConfig, and implemented separately in order to wrap the kubernetes RESTGetter interface.
+// Until then, we lose context propagation on all these calls
 func (conf *OutOfClusterConfig) ToRESTConfig() (*rest.Config, error) {
 	ctx, span := telemetry.NewSpan(context.Background(), "ooc-to-rest-config")
 	defer span.End()
 
+	telemetry.WithAttributes(span,
+		telemetry.AttributeKV{Key: "cluster-id", Value: conf.Cluster.ID},
+		telemetry.AttributeKV{Key: "project-id", Value: conf.Cluster.ProjectID},
+	)
+
 	if conf.Cluster.ProvisionedBy == "CAPI" {
 		telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "capi-provisioned", Value: true})
 
@@ -360,7 +378,7 @@ func (conf *OutOfClusterConfig) GetClientConfigFromCluster(ctx context.Context)
 		return clientcmd.NewClientConfigFromBytes(kubeAuth.Kubeconfig)
 	}
 
-	apiConfig, err := conf.CreateRawConfigFromCluster()
+	apiConfig, err := conf.CreateRawConfigFromCluster(ctx)
 	if err != nil {
 		return nil, telemetry.Error(ctx, span, err, "error creating raw config from cluster")
 	}
@@ -379,7 +397,10 @@ func (conf *OutOfClusterConfig) GetClientConfigFromCluster(ctx context.Context)
 	return config, nil
 }
 
-func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error) {
+func (conf *OutOfClusterConfig) CreateRawConfigFromCluster(ctx context.Context) (*api.Config, error) {
+	ctx, span := telemetry.NewSpan(ctx, "ooc-create-raw-config-from-cluster")
+	defer span.End()
+
 	cluster := conf.Cluster
 
 	apiConfig := &api.Config{}
@@ -408,6 +429,11 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 		authInfoMap[authInfoName].ImpersonateGroups = groups
 	}
 
+	telemetry.WithAttributes(span,
+		telemetry.AttributeKV{Key: "auth-mechanism", Value: cluster.AuthMechanism},
+		telemetry.AttributeKV{Key: "server", Value: cluster.Server},
+	)
+
 	switch cluster.AuthMechanism {
 	case models.X509:
 		kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
@@ -415,9 +441,13 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.KubeIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading kube integration")
 		}
 
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.KubeIntegrationID},
+		)
+
 		authInfoMap[authInfoName].ClientCertificateData = kubeAuth.ClientCertificateData
 		authInfoMap[authInfoName].ClientKeyData = kubeAuth.ClientKeyData
 	case models.Basic:
@@ -426,8 +456,11 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.KubeIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading kube integration")
 		}
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.KubeIntegrationID},
+		)
 
 		authInfoMap[authInfoName].Username = string(kubeAuth.Username)
 		authInfoMap[authInfoName].Password = string(kubeAuth.Password)
@@ -437,7 +470,7 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.KubeIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading kube integration")
 		}
 
 		authInfoMap[authInfoName].Token = string(kubeAuth.Token)
@@ -447,8 +480,11 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.OIDCIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading oidc integration")
 		}
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.OIDCIntegrationID},
+		)
 
 		authInfoMap[authInfoName].AuthProvider = &api.AuthProviderConfig{
 			Name: "oidc",
@@ -467,17 +503,24 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.GCPIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading gcp integration")
 		}
 
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.GCPIntegrationID},
+		)
+
 		tok, err := gcpAuth.GetBearerToken(
+			ctx,
 			conf.getTokenCache,
 			conf.setTokenCache,
 			"https://www.googleapis.com/auth/cloud-platform",
 		)
-
-		if tok == nil && err != nil {
-			return nil, err
+		if err != nil {
+			return nil, telemetry.Error(ctx, span, err, "error getting gcp token")
+		}
+		if tok == nil {
+			return nil, telemetry.Error(ctx, span, nil, "unable to get gcp token")
 		}
 
 		// add this as a bearer token
@@ -488,9 +531,13 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.AWSIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
 		}
 
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.AWSIntegrationID},
+		)
+
 		awsClusterID := cluster.Name
 		shouldOverride := false
 
@@ -499,9 +546,9 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			shouldOverride = true
 		}
 
-		tok, err := awsAuth.GetBearerToken(conf.getTokenCache, conf.setTokenCache, awsClusterID, shouldOverride)
+		tok, err := awsAuth.GetBearerToken(ctx, conf.getTokenCache, conf.setTokenCache, awsClusterID, shouldOverride)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "unable to get AWS bearer token")
 		}
 
 		// add this as a bearer token
@@ -512,12 +559,15 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.DOIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading oauth integration")
 		}
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.DOIntegrationID},
+		)
 
 		tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, conf.DigitalOceanOAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, conf.Repo))
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "unable to get oauth access token for Digital Ocean")
 		}
 
 		// add this as a bearer token
@@ -528,12 +578,15 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 			cluster.AzureIntegrationID,
 		)
 		if err != nil {
-			return nil, err
+			return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
 		}
+		telemetry.WithAttributes(span,
+			telemetry.AttributeKV{Key: "integration-id", Value: cluster.AzureIntegrationID},
+		)
 
 		authInfoMap[authInfoName].Token = string(azInt.AKSPassword)
 	default:
-		return nil, errors.New("not a supported auth mechanism")
+		return nil, telemetry.Error(ctx, span, nil, "auth mechanism not supported")
 	}
 
 	// create a context of the cluster name
@@ -553,11 +606,11 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 	return apiConfig, nil
 }
 
-func (conf *OutOfClusterConfig) getTokenCache() (tok *ints.TokenCache, err error) {
+func (conf *OutOfClusterConfig) getTokenCache(ctx context.Context) (tok *ints.TokenCache, err error) {
 	return &conf.Cluster.TokenCache.TokenCache, nil
 }
 
-func (conf *OutOfClusterConfig) setTokenCache(token string, expiry time.Time) error {
+func (conf *OutOfClusterConfig) setTokenCache(ctx context.Context, token string, expiry time.Time) error {
 	_, err := conf.Repo.Cluster().UpdateClusterTokenCache(
 		&ints.ClusterTokenCache{
 			ClusterID: conf.Cluster.ID,

internal/models/integrations/aws.go

@@ -1,6 +1,9 @@
 package integrations
 
 import (
+	"context"
+	"fmt"
+
 	"gorm.io/gorm"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -103,12 +106,13 @@ func (a *AWSIntegration) PopulateAWSArn() error {
 
 // GetBearerToken retrieves a bearer token for an AWS account
 func (a *AWSIntegration) GetBearerToken(
+	ctx context.Context,
 	getTokenCache GetTokenCacheFunc,
 	setTokenCache SetTokenCacheFunc,
 	clusterID string,
 	shouldClusterIdOverride bool,
 ) (string, error) {
-	cache, err := getTokenCache()
+	cache, err := getTokenCache(ctx)
 
 	// check the token cache for a non-expired token
 	if cache != nil {
@@ -119,12 +123,12 @@ func (a *AWSIntegration) GetBearerToken(
 
 	generator, err := token.NewGenerator(false, false)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("error creating token generator: %w", err)
 	}
 
 	sess, err := a.GetSession()
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("error getting session: %w", err)
 	}
 
 	var validClusterId string
@@ -145,10 +149,13 @@ func (a *AWSIntegration) GetBearerToken(
 		ClusterID:     validClusterId,
 	})
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("error generating token: %w", err)
 	}
 
-	setTokenCache(tok.Token, tok.Expiration)
+	err = setTokenCache(ctx, tok.Token, tok.Expiration)
+	if err != nil {
+		return "", fmt.Errorf("non-fatal error setting token cache: %w", err)
+	}
 
 	return tok.Token, nil
 }

internal/models/integrations/gcp.go

@@ -3,6 +3,7 @@ package integrations
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/porter-dev/porter/api/types"
 
@@ -55,11 +56,13 @@ func (g *GCPIntegration) ToGCPIntegrationType() *types.GCPIntegration {
 
 // GetBearerToken retrieves a bearer token for a GCP account
 func (g *GCPIntegration) GetBearerToken(
+	ctx context.Context,
+
 	getTokenCache GetTokenCacheFunc,
 	setTokenCache SetTokenCacheFunc,
 	scopes ...string,
 ) (*oauth2.Token, error) {
-	cache, err := getTokenCache()
+	cache, err := getTokenCache(ctx)
 
 	// check the token cache for a non-expired token
 	if cache != nil {
@@ -72,23 +75,26 @@ func (g *GCPIntegration) GetBearerToken(
 	}
 
 	creds, err := google.CredentialsFromJSON(
-		context.Background(),
+		ctx,
 		g.GCPKeyData,
 		scopes...,
 	)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get credentials from json: %w", err)
 	}
 
 	tok, err := creds.TokenSource.Token()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get token from credentials: %w", err)
 	}
 
 	// update the token cache
-	err = setTokenCache(tok.AccessToken, tok.Expiry)
+	err = setTokenCache(ctx, tok.AccessToken, tok.Expiry)
+	if err != nil {
+		return nil, fmt.Errorf("non-fatal error setting token cache: %w", err)
+	}
 
-	return tok, err
+	return tok, nil
 }
 
 // credentialsFile is the unmarshalled representation of a GCP credentials file.

internal/models/integrations/token_cache.go

@@ -1,6 +1,7 @@
 package integrations
 
 import (
+	"context"
 	"time"
 
 	"gorm.io/gorm"
@@ -23,11 +24,11 @@ type TokenCache struct {
 
 // GetTokenCacheFunc is a function that retrieves the token and expiry
 // time from the db
-type GetTokenCacheFunc func() (tok *TokenCache, err error)
+type GetTokenCacheFunc func(ctx context.Context) (tok *TokenCache, err error)
 
 // SetTokenCacheFunc is a function that updates the token cache
 // with a new token and expiry time
-type SetTokenCacheFunc func(token string, expiry time.Time) error
+type SetTokenCacheFunc func(ctx context.Context, token string, expiry time.Time) error
 
 // IsExpired returns true if a token is expired, false otherwise
 func (t *TokenCache) IsExpired() bool {

internal/registry/registry.go

@@ -105,7 +105,7 @@ func (r *Registry) ListRepositories(
 	if r.GCPIntegrationID != 0 {
 		telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
 		if strings.Contains(r.URL, "pkg.dev") {
-			return r.listGARRepositories(repo)
+			return r.listGARRepositories(ctx, repo)
 		}
 
 		repos, err := r.listGCRRepositories(repo)
@@ -224,8 +224,8 @@ type gcrRepositoryResp struct {
 	Errors       []gcrErr `json:"errors"`
 }
 
-func (r *Registry) GetGCRToken(repo repository.Repository) (*oauth2.Token, error) {
-	getTokenCache := r.getTokenCacheFunc(repo)
+func (r *Registry) GetGCRToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
+	getTokenCache := r.getTokenCacheFunc(ctx, repo)
 
 	gcp, err := repo.GCPIntegration().ReadGCPIntegration(
 		r.ProjectID,
@@ -237,8 +237,9 @@ func (r *Registry) GetGCRToken(repo repository.Repository) (*oauth2.Token, error
 
 	// get oauth2 access token
 	return gcp.GetBearerToken(
+		ctx,
 		getTokenCache,
-		r.setTokenCacheFunc(repo),
+		r.setTokenCacheFunc(ctx, repo),
 		"https://www.googleapis.com/auth/devstorage.read_write",
 	)
 }
@@ -319,8 +320,8 @@ func (r *Registry) listGCRRepositories(
 	return res, nil
 }
 
-func (r *Registry) GetGARToken(repo repository.Repository) (*oauth2.Token, error) {
-	getTokenCache := r.getTokenCacheFunc(repo)
+func (r *Registry) GetGARToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
+	getTokenCache := r.getTokenCacheFunc(ctx, repo)
 
 	gcp, err := repo.GCPIntegration().ReadGCPIntegration(
 		r.ProjectID,
@@ -332,25 +333,29 @@ func (r *Registry) GetGARToken(repo repository.Repository) (*oauth2.Token, error
 
 	// get oauth2 access token
 	return gcp.GetBearerToken(
+		ctx,
 		getTokenCache,
-		r.setTokenCacheFunc(repo),
+		r.setTokenCacheFunc(ctx, repo),
 		"https://www.googleapis.com/auth/cloud-platform",
 	)
 }
 
 type garTokenSource struct {
+	// ctx is only passed in here as the oauth2.Token() doesnt support contexts
+	ctx  context.Context
 	reg  *Registry
 	repo repository.Repository
 }
 
 func (source *garTokenSource) Token() (*oauth2.Token, error) {
-	return source.reg.GetGARToken(source.repo)
+	return source.reg.GetGARToken(source.ctx, source.repo)
 }
 
 // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
 // where a repository is a single image. This function returns the list of fully qualified names
 // of GAR images including their repository names.
 func (r *Registry) listGARRepositories(
+	ctx context.Context,
 	repo repository.Repository,
 ) ([]*ptypes.RegistryRepository, error) {
 	gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
@@ -361,9 +366,10 @@ func (r *Registry) listGARRepositories(
 		return nil, err
 	}
 
-	client, err := artifactregistry.NewClient(context.Background(), option.WithTokenSource(&garTokenSource{
+	client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
 		reg:  r,
 		repo: repo,
+		ctx:  ctx,
 	}), option.WithScopes("roles/artifactregistry.reader"))
 	if err != nil {
 		return nil, err
@@ -410,9 +416,10 @@ func (r *Registry) listGARRepositories(
 		nextToken = it.PageInfo().Token
 	}
 
-	svc, err := v1artifactregistry.NewService(context.Background(), option.WithTokenSource(&garTokenSource{
+	svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
 		reg:  r,
 		repo: repo,
+		ctx:  ctx,
 	}), option.WithScopes("roles/artifactregistry.reader"))
 	if err != nil {
 		return nil, err
@@ -809,9 +816,10 @@ func (r *Registry) listPrivateRegistryRepositories(
 }
 
 func (r *Registry) getTokenCacheFunc(
+	ctx context.Context,
 	repo repository.Repository,
 ) ints.GetTokenCacheFunc {
-	return func() (tok *ints.TokenCache, err error) {
+	return func(ctx context.Context) (tok *ints.TokenCache, err error) {
 		reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
 		if err != nil {
 			return nil, err
@@ -822,9 +830,10 @@ func (r *Registry) getTokenCacheFunc(
 }
 
 func (r *Registry) setTokenCacheFunc(
+	ctx context.Context,
 	repo repository.Repository,
 ) ints.SetTokenCacheFunc {
-	return func(token string, expiry time.Time) error {
+	return func(ctx context.Context, token string, expiry time.Time) error {
 		_, err := repo.Registry().UpdateRegistryTokenCache(
 			&ints.RegTokenCache{
 				TokenCache: ints.TokenCache{
@@ -857,7 +866,7 @@ func (r *Registry) CreateRepository(
 		}
 		return r.createECRRepository(aws, name)
 	} else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
-		return r.createGARRepository(conf.Repo, name)
+		return r.createGARRepository(ctx, conf.Repo, name)
 	}
 
 	project, err := conf.Repo.Project().ReadProject(r.ProjectID)
@@ -927,6 +936,7 @@ func (r *Registry) createECRRepository(
 }
 
 func (r *Registry) createGARRepository(
+	ctx context.Context,
 	repo repository.Repository,
 	name string,
 ) error {
@@ -938,9 +948,10 @@ func (r *Registry) createGARRepository(
 		return err
 	}
 
-	client, err := artifactregistry.NewClient(context.Background(), option.WithTokenSource(&garTokenSource{
+	client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
 		reg:  r,
 		repo: repo,
+		ctx:  ctx,
 	}), option.WithScopes("roles/artifactregistry.admin"))
 	if err != nil {
 		return err
@@ -1003,7 +1014,7 @@ func (r *Registry) ListImages(
 
 	if r.GCPIntegrationID != 0 {
 		if strings.Contains(r.URL, "pkg.dev") {
-			return r.listGARImages(repoName, repo)
+			return r.listGARImages(ctx, repoName, repo)
 		}
 
 		return r.listGCRImages(repoName, repo)
@@ -1369,7 +1380,7 @@ func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([
 	return res, nil
 }
 
-func (r *Registry) listGARImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
+func (r *Registry) listGARImages(ctx context.Context, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
 	repoImageSlice := strings.Split(repoName, "/")
 
 	if len(repoImageSlice) != 2 {
@@ -1384,9 +1395,10 @@ func (r *Registry) listGARImages(repoName string, repo repository.Repository) ([
 		return nil, err
 	}
 
-	svc, err := v1artifactregistry.NewService(context.Background(), option.WithTokenSource(&garTokenSource{
+	svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
 		reg:  r,
 		repo: repo,
+		ctx:  ctx,
 	}), option.WithScopes("roles/artifactregistry.reader"))
 	if err != nil {
 		return nil, err

internal/telemetry/span.go

@@ -30,6 +30,7 @@ func NewSpan(ctx context.Context, name string) (context.Context, trace.Span) {
 	return ctx, span
 }
 
+// AddKnownContextVariablesToSpan adds known commonly read context variables to a span
 func AddKnownContextVariablesToSpan(ctx context.Context, span trace.Span) {
 	user, ok := ctx.Value(types.UserScope).(*models.User)
 	if ok {

internal/usage/usage.go

@@ -1,6 +1,7 @@
 package usage
 
 import (
+	"context"
 	"errors"
 	"time"
 
@@ -132,6 +133,7 @@ func isUsageChanged(oldUsageCache, currUsageCache *models.ProjectUsageCache) boo
 
 // gets the total resource usage across all nodes in all clusters
 func getResourceUsage(opts *GetUsageOpts, clusters []*models.Cluster) (uint, uint, error) {
+	ctx := context.Background()
 	var totCPU, totMem uint = 0, 0
 
 	for _, cluster := range clusters {
@@ -143,7 +145,7 @@ func getResourceUsage(opts *GetUsageOpts, clusters []*models.Cluster) (uint, uin
 			CAPIManagementClusterClient: opts.ClusterControlPlaneServiceClient,
 		}
 
-		agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
+		agent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, ooc)
 		if err != nil {
 			continue
 		}

provisioner/server/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"time"
@@ -140,6 +141,8 @@ func FromEnv() (*EnvConf, error) {
 }
 
 func GetConfig(envConf *EnvConf) (*Config, error) {
+	ctx := context.Background()
+
 	res := &Config{
 		ProvisionerConf: envConf.ProvisionerConf,
 		DBConf:          envConf.DBConf,
@@ -206,7 +209,7 @@ func GetConfig(envConf *EnvConf) (*Config, error) {
 			LocalTerraformDirectory: envConf.LocalTerraformDirectory,
 		})
 	} else if envConf.ProvisionerMethod == "kubernetes" {
-		provAgent, err := getProvisionerAgent(envConf.ProvisionerConf)
+		provAgent, err := getProvisionerAgent(ctx, envConf.ProvisionerConf)
 		if err != nil {
 			return nil, err
 		}
@@ -234,7 +237,7 @@ func GetConfig(envConf *EnvConf) (*Config, error) {
 	return res, nil
 }
 
-func getProvisionerAgent(conf *ProvisionerConf) (*kubernetes.Agent, error) {
+func getProvisionerAgent(ctx context.Context, conf *ProvisionerConf) (*kubernetes.Agent, error) {
 	if conf.ProvisionerCluster == "kubeconfig" && conf.SelfKubeconfig != "" {
 		agent, err := klocal.GetSelfAgentFromFileConfig(conf.SelfKubeconfig)
 		if err != nil {
@@ -246,7 +249,7 @@ func getProvisionerAgent(conf *ProvisionerConf) (*kubernetes.Agent, error) {
 		return nil, fmt.Errorf(`"kubeconfig" cluster option requires path to kubeconfig`)
 	}
 
-	agent, _ := kubernetes.GetAgentInClusterConfig(conf.ProvisionerJobNamespace)
+	agent, _ := kubernetes.GetAgentInClusterConfig(ctx, conf.ProvisionerJobNamespace)
 
 	return agent, nil
 }

provisioner/server/handlers/state/create_resource.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -17,6 +18,7 @@ import (
 	"github.com/porter-dev/porter/internal/kubernetes"
 	"github.com/porter-dev/porter/internal/kubernetes/envgroup"
 	"github.com/porter-dev/porter/internal/models"
+	"github.com/porter-dev/porter/internal/telemetry"
 	"github.com/porter-dev/porter/provisioner/integrations/redis_stream"
 	"github.com/porter-dev/porter/provisioner/server/config"
 	ptypes "github.com/porter-dev/porter/provisioner/types"
@@ -38,9 +40,11 @@ func NewCreateResourceHandler(
 }
 
 func (c *CreateResourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx, span := telemetry.NewSpan(r.Context(), "serve-create-provisioner-resource")
+	defer span.End()
 	// read the infra from the attached scope
-	infra, _ := r.Context().Value(types.InfraScope).(*models.Infra)
-	operation, _ := r.Context().Value(types.OperationScope).(*models.Operation)
+	infra, _ := ctx.Value(types.InfraScope).(*models.Infra)
+	operation, _ := ctx.Value(types.OperationScope).(*models.Operation)
 	req := &ptypes.CreateResourceRequest{}
 	if ok := c.decoderValidator.DecodeAndValidate(w, r, req); !ok {
 		return
@@ -88,9 +92,9 @@ func (c *CreateResourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
 	case string(types.InfraECR):
 		_, err = createECRRegistry(c.Config, infra, operation, req.Output)
 	case string(types.InfraRDS):
-		_, err = createRDSDatabase(c.Config, infra, operation, req.Output)
+		_, err = createRDSDatabase(ctx, c.Config, infra, operation, req.Output)
 	case string(types.InfraS3):
-		err = createS3Bucket(c.Config, infra, operation, req.Output)
+		err = createS3Bucket(ctx, c.Config, infra, operation, req.Output)
 	case string(types.InfraDOCR):
 		_, err = createDOCRRegistry(c.Config, infra, operation, req.Output)
 	case string(types.InfraGCR):
@@ -138,7 +142,7 @@ func createECRRegistry(config *config.Config, infra *models.Infra, operation *mo
 	return reg, nil
 }
 
-func createRDSDatabase(config *config.Config, infra *models.Infra, operation *models.Operation, output map[string]interface{}) (*models.Database, error) {
+func createRDSDatabase(ctx context.Context, config *config.Config, infra *models.Infra, operation *models.Operation, output map[string]interface{}) (*models.Database, error) {
 	// check for infra id being 0 as a safeguard so that all non-provisioned
 	// clusters are not matched by read
 	if infra.ID == 0 {
@@ -177,20 +181,20 @@ func createRDSDatabase(config *config.Config, infra *models.Infra, operation *mo
 	if err != nil {
 		return nil, err
 	}
-	err = createRDSEnvGroup(config, infra, database, lastApplied)
+	err = createRDSEnvGroup(ctx, config, infra, database, lastApplied)
 	if err != nil {
 		return nil, err
 	}
 	return database, nil
 }
 
-func createS3Bucket(config *config.Config, infra *models.Infra, operation *models.Operation, output map[string]interface{}) error {
+func createS3Bucket(ctx context.Context, config *config.Config, infra *models.Infra, operation *models.Operation, output map[string]interface{}) error {
 	lastApplied := make(map[string]interface{})
 	err := json.Unmarshal(operation.LastApplied, &lastApplied)
 	if err != nil {
 		return err
 	}
-	return createS3EnvGroup(config, infra, lastApplied, output)
+	return createS3EnvGroup(ctx, config, infra, lastApplied, output)
 }
 
 func createCluster(config *config.Config, infra *models.Infra, operation *models.Operation, output map[string]interface{}) (*models.Cluster, error) {
@@ -327,7 +331,7 @@ func createACRRegistry(config *config.Config, infra *models.Infra, operation *mo
 	return config.Repo.Registry().CreateRegistry(reg)
 }
 
-func createRDSEnvGroup(config *config.Config, infra *models.Infra, database *models.Database, lastApplied map[string]interface{}) error {
+func createRDSEnvGroup(ctx context.Context, config *config.Config, infra *models.Infra, database *models.Database, lastApplied map[string]interface{}) error {
 	cluster, err := config.Repo.Cluster().ReadCluster(infra.ProjectID, infra.ParentClusterID)
 	if err != nil {
 		return err
@@ -337,7 +341,7 @@ func createRDSEnvGroup(config *config.Config, infra *models.Infra, database *mod
 		DigitalOceanOAuth: config.DOConf,
 		Cluster:           cluster,
 	}
-	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
+	agent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, ooc)
 	if err != nil {
 		return fmt.Errorf("failed to get agent: %s", err.Error())
 	}
@@ -365,7 +369,7 @@ func createRDSEnvGroup(config *config.Config, infra *models.Infra, database *mod
 	return nil
 }
 
-func deleteRDSEnvGroup(config *config.Config, infra *models.Infra, lastApplied map[string]interface{}) error {
+func deleteRDSEnvGroup(ctx context.Context, config *config.Config, infra *models.Infra, lastApplied map[string]interface{}) error {
 	cluster, err := config.Repo.Cluster().ReadCluster(infra.ProjectID, infra.ParentClusterID)
 	if err != nil {
 		return err
@@ -375,7 +379,7 @@ func deleteRDSEnvGroup(config *config.Config, infra *models.Infra, lastApplied m
 		DigitalOceanOAuth: config.DOConf,
 		Cluster:           cluster,
 	}
-	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
+	agent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, ooc)
 	if err != nil {
 		return fmt.Errorf("failed to get agent: %s", err.Error())
 	}
@@ -386,7 +390,7 @@ func deleteRDSEnvGroup(config *config.Config, infra *models.Infra, lastApplied m
 	return nil
 }
 
-func createS3EnvGroup(config *config.Config, infra *models.Infra, lastApplied map[string]interface{}, output map[string]interface{}) error {
+func createS3EnvGroup(ctx context.Context, config *config.Config, infra *models.Infra, lastApplied map[string]interface{}, output map[string]interface{}) error {
 	cluster, err := config.Repo.Cluster().ReadCluster(infra.ProjectID, infra.ParentClusterID)
 	if err != nil {
 		return err
@@ -396,7 +400,7 @@ func createS3EnvGroup(config *config.Config, infra *models.Infra, lastApplied ma
 		DigitalOceanOAuth: config.DOConf,
 		Cluster:           cluster,
 	}
-	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
+	agent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, ooc)
 	if err != nil {
 		return fmt.Errorf("failed to get agent: %s", err.Error())
 	}
@@ -417,7 +421,7 @@ func createS3EnvGroup(config *config.Config, infra *models.Infra, lastApplied ma
 	return nil
 }
 
-func deleteS3EnvGroup(config *config.Config, infra *models.Infra, lastApplied map[string]interface{}) error {
+func deleteS3EnvGroup(ctx context.Context, config *config.Config, infra *models.Infra, lastApplied map[string]interface{}) error {
 	cluster, err := config.Repo.Cluster().ReadCluster(infra.ProjectID, infra.ParentClusterID)
 	if err != nil {
 		return err
@@ -427,7 +431,7 @@ func deleteS3EnvGroup(config *config.Config, infra *models.Infra, lastApplied ma
 		DigitalOceanOAuth: config.DOConf,
 		Cluster:           cluster,
 	}
-	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
+	agent, err := kubernetes.GetAgentOutOfClusterConfig(ctx, ooc)
 	if err != nil {
 		return fmt.Errorf("failed to get agent: %s", err.Error())
 	}

provisioner/server/handlers/state/delete_resource.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 
@@ -8,6 +9,7 @@ import (
 	"github.com/porter-dev/porter/api/server/shared/apierrors"
 	"github.com/porter-dev/porter/api/types"
 	"github.com/porter-dev/porter/internal/models"
+	"github.com/porter-dev/porter/internal/telemetry"
 	"github.com/porter-dev/porter/provisioner/integrations/redis_stream"
 	"github.com/porter-dev/porter/provisioner/server/config"
 )
@@ -27,9 +29,12 @@ func NewDeleteResourceHandler(
 }
 
 func (c *DeleteResourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx, span := telemetry.NewSpan(r.Context(), "serve-delete-resource")
+	defer span.End()
+
 	// read the infra from the attached scope
-	infra, _ := r.Context().Value(types.InfraScope).(*models.Infra)
-	operation, _ := r.Context().Value(types.OperationScope).(*models.Operation)
+	infra, _ := ctx.Value(types.InfraScope).(*models.Infra)
+	operation, _ := ctx.Value(types.OperationScope).(*models.Operation)
 
 	// update the operation to indicate completion
 	operation.Status = "completed"
@@ -75,7 +80,7 @@ func (c *DeleteResourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
 	case types.InfraRDS:
 		_, err = deleteDatabase(c.Config, infra, operation)
 	case types.InfraS3:
-		err = deleteS3Bucket(c.Config, infra, operation)
+		err = deleteS3Bucket(ctx, c.Config, infra, operation)
 	}
 
 	if err != nil {
@@ -131,7 +136,7 @@ func deleteDatabase(config *config.Config, infra *models.Infra, operation *model
 	return database, nil
 }
 
-func deleteS3Bucket(config *config.Config, infra *models.Infra, operation *models.Operation) error {
+func deleteS3Bucket(ctx context.Context, config *config.Config, infra *models.Infra, operation *models.Operation) error {
 	lastApplied := make(map[string]interface{})
 
 	err := json.Unmarshal(operation.LastApplied, &lastApplied)
@@ -139,5 +144,5 @@ func deleteS3Bucket(config *config.Config, infra *models.Infra, operation *model
 		return err
 	}
 
-	return deleteS3EnvGroup(config, infra, lastApplied)
+	return deleteS3EnvGroup(ctx, config, infra, lastApplied)
 }