unify kubeconfig by only checking for CAPI in the RawKubeConfig method, inserting the correct CA data and token (#3011)

Co-authored-by: David Townley <davidtownley@Davids-MacBook-Air.local>

api/server/authz/cluster.go

@@ -118,7 +118,7 @@ func (d *OutOfClusterAgentGetter) GetAgent(r *http.Request, cluster *models.Clus
 
 	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get agent: %s", err.Error())
+		return nil, fmt.Errorf("failed to get agent: %w", err)
 	}
 
 	newCtx := context.WithValue(r.Context(), KubernetesAgentCtxKey, agent)

api/server/handlers/cluster/get_kubeconfig.go

@@ -1,14 +1,9 @@
 package cluster
 
 import (
-	"context"
-	"encoding/base64"
 	"errors"
-	"fmt"
 	"net/http"
 
-	"github.com/bufbuild/connect-go"
-	porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
 	"github.com/porter-dev/porter/api/server/authz"
 	"github.com/porter-dev/porter/api/server/handlers"
 	"github.com/porter-dev/porter/api/server/shared"
@@ -46,33 +41,33 @@ func (c *GetTemporaryKubeconfigHandler) ServeHTTP(w http.ResponseWriter, r *http
 	cluster, _ := ctx.Value(types.ClusterScope).(*models.Cluster)
 
 	outOfClusterConfig := c.GetOutOfClusterConfig(cluster)
-
-	if cluster.ProvisionedBy == "CAPI" {
-		kubeconfigResp, err := c.Config().ClusterControlPlaneClient.KubeConfigForCluster(context.Background(), connect.NewRequest(
-			&porterv1.KubeConfigForClusterRequest{
-				ProjectId: int64(cluster.ProjectID),
-				ClusterId: int64(cluster.ID),
-			},
-		))
-		if err != nil {
-			c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error getting temporary capi config: %w", err)))
-			return
-		}
-		if kubeconfigResp.Msg == nil {
-			c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error reading temporary capi config: %w", err)))
-			return
-		}
-		b64, err := base64.StdEncoding.DecodeString(kubeconfigResp.Msg.KubeConfig)
-		if err != nil {
-			c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("unable to decode base64 kubeconfig: %w", err)))
-			return
-		}
-		res := &types.GetTemporaryKubeconfigResponse{
-			Kubeconfig: b64,
-		}
-		c.WriteResult(w, r, res)
-		return
-	}
+	//
+	//if cluster.ProvisionedBy == "CAPI" {
+	//	kubeconfigResp, err := c.Config().ClusterControlPlaneClient.KubeConfigForCluster(context.Background(), connect.NewRequest(
+	//		&porterv1.KubeConfigForClusterRequest{
+	//			ProjectId: int64(cluster.ProjectID),
+	//			ClusterId: int64(cluster.ID),
+	//		},
+	//	))
+	//	if err != nil {
+	//		c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error getting temporary capi config: %w", err)))
+	//		return
+	//	}
+	//	if kubeconfigResp.Msg == nil {
+	//		c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("error reading temporary capi config: %w", err)))
+	//		return
+	//	}
+	//	b64, err := base64.StdEncoding.DecodeString(kubeconfigResp.Msg.KubeConfig)
+	//	if err != nil {
+	//		c.HandleAPIError(w, r, apierrors.NewErrInternal(fmt.Errorf("unable to decode base64 kubeconfig: %w", err)))
+	//		return
+	//	}
+	//	res := &types.GetTemporaryKubeconfigResponse{
+	//		Kubeconfig: b64,
+	//	}
+	//	c.WriteResult(w, r, res)
+	//	return
+	//}
 
 	kubeconfig, err := outOfClusterConfig.CreateRawConfigFromCluster()
 	if err != nil {

go.mod

@@ -74,7 +74,7 @@ require (
 	github.com/glebarez/sqlite v1.6.0
 	github.com/nats-io/nats.go v1.24.0
 	github.com/open-policy-agent/opa v0.44.0
-	github.com/porter-dev/api-contracts v0.0.60
+	github.com/porter-dev/api-contracts v0.0.61
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.1
 	github.com/stefanmcshane/helm v0.0.0-20221213002717-88a4a2c6e77d
 	github.com/xanzy/go-gitlab v0.68.0

go.sum

@@ -1466,8 +1466,8 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
-github.com/porter-dev/api-contracts v0.0.60 h1:wqsN9XkqcQLIiOK2wdTyknIG8ZCw9y80SX9lpnri0is=
-github.com/porter-dev/api-contracts v0.0.60/go.mod h1:qr2L58mJLr5DUGV5OPw3REiSrQvJq6TgkKyEWP95dyU=
+github.com/porter-dev/api-contracts v0.0.61 h1:s/0A3YoPIPvcWwjkN3XfmN/Q4+iR0QN+HBTVcLg+DrY=
+github.com/porter-dev/api-contracts v0.0.61/go.mod h1:qr2L58mJLr5DUGV5OPw3REiSrQvJq6TgkKyEWP95dyU=
 github.com/porter-dev/switchboard v0.0.0-20221019155755-67ff2bf04935 h1:hfb3nt3AJXIBbevu6ARTg9SdOkMP6WLbKBiG5hT5rcc=
 github.com/porter-dev/switchboard v0.0.0-20221019155755-67ff2bf04935/go.mod h1:xSPzqSFMQ6OSbp42fhCi4AbGbQbsm6nRvOkrblFeXU4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=

internal/kubernetes/config.go

@@ -69,19 +69,19 @@ func GetAgentOutOfClusterConfig(conf *OutOfClusterConfig) (*Agent, error) {
 
 	var restConf *rest.Config
 
-	if conf.Cluster.ProvisionedBy == "CAPI" {
-		rc, err := restConfigForCAPICluster(context.Background(), conf.CAPIManagementClusterClient, *conf.Cluster)
-		if err != nil {
-			return nil, err
-		}
-		restConf = rc
-	} else {
-		rc, err := conf.ToRESTConfig()
-		if err != nil {
-			return nil, err
-		}
-		restConf = rc
+	//if conf.Cluster.ProvisionedBy == "CAPI" {
+	//	rc, err := restConfigForCAPICluster(context.Background(), conf.CAPIManagementClusterClient, *conf.Cluster)
+	//	if err != nil {
+	//		return nil, err
+	//	}
+	//	restConf = rc
+	//} else {
+	rc, err := conf.ToRESTConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert ooc config to rest config: %w", err)
 	}
+	restConf = rc
+	//}
 
 	if restConf == nil {
 		return nil, fmt.Errorf("error getting rest config for cluster %s", conf.Cluster.ProvisionedBy)
@@ -89,7 +89,7 @@ func GetAgentOutOfClusterConfig(conf *OutOfClusterConfig) (*Agent, error) {
 
 	clientset, err := kubernetes.NewForConfig(restConf)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get new clientset from rest config: %w", err)
 	}
 
 	return &Agent{conf, clientset}, nil
@@ -207,22 +207,22 @@ type OutOfClusterConfig struct {
 // the result of ToRawKubeConfigLoader, and also adds a custom http transport layer
 // if necessary (required for GCP auth)
 func (conf *OutOfClusterConfig) ToRESTConfig() (*rest.Config, error) {
-	if conf.Cluster.ProvisionedBy == "CAPI" {
-		rc, err := restConfigForCAPICluster(context.Background(), conf.CAPIManagementClusterClient, *conf.Cluster)
-		if err != nil {
-			return nil, err
-		}
-		return rc, nil
-	}
+	//if conf.Cluster.ProvisionedBy == "CAPI" {
+	//	rc, err := restConfigForCAPICluster(context.Background(), conf.CAPIManagementClusterClient, *conf.Cluster)
+	//	if err != nil {
+	//		return nil, err
+	//	}
+	//	return rc, nil
+	//}
 
 	cmdConf, err := conf.GetClientConfigFromCluster()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get cmdConf from cluster: %w", err)
 	}
 
 	restConf, err := cmdConf.ClientConfig()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get client config from cmdConf: %w", err)
 	}
 
 	restConf.Timeout = conf.Timeout
@@ -282,28 +282,28 @@ func (conf *OutOfClusterConfig) GetClientConfigFromCluster() (clientcmd.ClientCo
 		return nil, fmt.Errorf("cluster cannot be nil")
 	}
 
-	if conf.Cluster.ProvisionedBy == "CAPI" {
-		rc, err := kubeConfigForCAPICluster(context.Background(), conf.CAPIManagementClusterClient, *conf.Cluster)
-		if err != nil {
-			return nil, err
-		}
-		clientConfig, err := clientcmd.NewClientConfigFromBytes([]byte(rc))
-		if err != nil {
-			return nil, err
-		}
-		rawConfig, err := clientConfig.RawConfig()
-		if err != nil {
-			return nil, err
-		}
-
-		overrides := &clientcmd.ConfigOverrides{}
-
-		overrides.Context = api.Context{
-			Namespace: conf.DefaultNamespace,
-		}
-
-		return clientcmd.NewDefaultClientConfig(rawConfig, overrides), nil
-	}
+	//if conf.Cluster.ProvisionedBy == "CAPI" {
+	//	rc, err := kubeConfigForCAPICluster(context.Background(), conf.CAPIManagementClusterClient, *conf.Cluster)
+	//	if err != nil {
+	//		return nil, err
+	//	}
+	//	clientConfig, err := clientcmd.NewClientConfigFromBytes([]byte(rc))
+	//	if err != nil {
+	//		return nil, err
+	//	}
+	//	rawConfig, err := clientConfig.RawConfig()
+	//	if err != nil {
+	//		return nil, err
+	//	}
+	//
+	//	overrides := &clientcmd.ConfigOverrides{}
+	//
+	//	overrides.Context = api.Context{
+	//		Namespace: conf.DefaultNamespace,
+	//	}
+	//
+	//	return clientcmd.NewDefaultClientConfig(rawConfig, overrides), nil
+	//}
 
 	if conf.Cluster.AuthMechanism == models.Local {
 		kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
@@ -319,7 +319,7 @@ func (conf *OutOfClusterConfig) GetClientConfigFromCluster() (clientcmd.ClientCo
 
 	apiConfig, err := conf.CreateRawConfigFromCluster()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create raw config from cluster: %w", err)
 	}
 
 	overrides := &clientcmd.ConfigOverrides{}
@@ -364,87 +364,27 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 		authInfoMap[authInfoName].ImpersonateGroups = groups
 	}
 
-	switch cluster.AuthMechanism {
-	case models.X509:
-		kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
-			cluster.ProjectID,
-			cluster.KubeIntegrationID,
-		)
+	if conf.Cluster.ProvisionedBy == "CAPI" {
+		decodedCert, err := capiCertAuthData(conf.CAPIManagementClusterClient, int(cluster.ID), int(cluster.ProjectID))
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error retrieving capi certificate authority data: %w", err)
 		}
 
-		authInfoMap[authInfoName].ClientCertificateData = kubeAuth.ClientCertificateData
-		authInfoMap[authInfoName].ClientKeyData = kubeAuth.ClientKeyData
-	case models.Basic:
-		kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
-			cluster.ProjectID,
-			cluster.KubeIntegrationID,
-		)
-		if err != nil {
-			return nil, err
-		}
+		clusterMap[cluster.Name].CertificateAuthorityData = decodedCert
 
-		authInfoMap[authInfoName].Username = string(kubeAuth.Username)
-		authInfoMap[authInfoName].Password = string(kubeAuth.Password)
-	case models.Bearer:
-		kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
-			cluster.ProjectID,
-			cluster.KubeIntegrationID,
-		)
-		if err != nil {
-			return nil, err
-		}
-
-		authInfoMap[authInfoName].Token = string(kubeAuth.Token)
-	case models.OIDC:
-		oidcAuth, err := conf.Repo.OIDCIntegration().ReadOIDCIntegration(
-			cluster.ProjectID,
-			cluster.OIDCIntegrationID,
-		)
-		if err != nil {
-			return nil, err
-		}
+		req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
+			ProjectId: int64(cluster.ProjectID),
+		})
 
-		authInfoMap[authInfoName].AuthProvider = &api.AuthProviderConfig{
-			Name: "oidc",
-			Config: map[string]string{
-				"idp-issuer-url":                 string(oidcAuth.IssuerURL),
-				"client-id":                      string(oidcAuth.ClientID),
-				"client-secret":                  string(oidcAuth.ClientSecret),
-				"idp-certificate-authority-data": string(oidcAuth.CertificateAuthorityData),
-				"id-token":                       string(oidcAuth.IDToken),
-				"refresh-token":                  string(oidcAuth.RefreshToken),
-			},
-		}
-	case models.GCP:
-		gcpAuth, err := conf.Repo.GCPIntegration().ReadGCPIntegration(
-			cluster.ProjectID,
-			cluster.GCPIntegrationID,
-		)
+		creds, err := conf.CAPIManagementClusterClient.AssumeRoleCredentials(context.Background(), req)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
 		}
 
-		tok, err := gcpAuth.GetBearerToken(
-			conf.getTokenCache,
-			conf.setTokenCache,
-			"https://www.googleapis.com/auth/cloud-platform",
-		)
-
-		if tok == nil && err != nil {
-			return nil, err
-		}
-
-		// add this as a bearer token
-		authInfoMap[authInfoName].Token = tok.AccessToken
-	case models.AWS:
-		awsAuth, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
-			cluster.ProjectID,
-			cluster.AWSIntegrationID,
-		)
-		if err != nil {
-			return nil, err
+		awsAuth := &ints.AWSIntegration{
+			AWSAccessKeyID:     []byte(creds.Msg.AwsAccessId),
+			AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
+			AWSSessionToken:    []byte(creds.Msg.AwsSessionToken),
 		}
 
 		awsClusterID := cluster.Name
@@ -457,39 +397,141 @@ func (conf *OutOfClusterConfig) CreateRawConfigFromCluster() (*api.Config, error
 
 		tok, err := awsAuth.GetBearerToken(conf.getTokenCache, conf.setTokenCache, awsClusterID, shouldOverride)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error getting bearer token for repository: %w", err)
 		}
 
-		// add this as a bearer token
 		authInfoMap[authInfoName].Token = tok
-	case models.DO:
-		oauthInt, err := conf.Repo.OAuthIntegration().ReadOAuthIntegration(
-			cluster.ProjectID,
-			cluster.DOIntegrationID,
-		)
-		if err != nil {
-			return nil, err
-		}
 
-		tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, conf.DigitalOceanOAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, conf.Repo))
-		if err != nil {
-			return nil, err
-		}
+		fmt.Printf("I successfully ran!")
 
-		// add this as a bearer token
-		authInfoMap[authInfoName].Token = tok
-	case models.Azure:
-		azInt, err := conf.Repo.AzureIntegration().ReadAzureIntegration(
-			cluster.ProjectID,
-			cluster.AzureIntegrationID,
-		)
-		if err != nil {
-			return nil, err
+	} else {
+		switch cluster.AuthMechanism {
+		case models.X509:
+			kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
+				cluster.ProjectID,
+				cluster.KubeIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfoMap[authInfoName].ClientCertificateData = kubeAuth.ClientCertificateData
+			authInfoMap[authInfoName].ClientKeyData = kubeAuth.ClientKeyData
+		case models.Basic:
+			kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
+				cluster.ProjectID,
+				cluster.KubeIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfoMap[authInfoName].Username = string(kubeAuth.Username)
+			authInfoMap[authInfoName].Password = string(kubeAuth.Password)
+		case models.Bearer:
+			kubeAuth, err := conf.Repo.KubeIntegration().ReadKubeIntegration(
+				cluster.ProjectID,
+				cluster.KubeIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfoMap[authInfoName].Token = string(kubeAuth.Token)
+		case models.OIDC:
+			oidcAuth, err := conf.Repo.OIDCIntegration().ReadOIDCIntegration(
+				cluster.ProjectID,
+				cluster.OIDCIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfoMap[authInfoName].AuthProvider = &api.AuthProviderConfig{
+				Name: "oidc",
+				Config: map[string]string{
+					"idp-issuer-url":                 string(oidcAuth.IssuerURL),
+					"client-id":                      string(oidcAuth.ClientID),
+					"client-secret":                  string(oidcAuth.ClientSecret),
+					"idp-certificate-authority-data": string(oidcAuth.CertificateAuthorityData),
+					"id-token":                       string(oidcAuth.IDToken),
+					"refresh-token":                  string(oidcAuth.RefreshToken),
+				},
+			}
+		case models.GCP:
+			gcpAuth, err := conf.Repo.GCPIntegration().ReadGCPIntegration(
+				cluster.ProjectID,
+				cluster.GCPIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			tok, err := gcpAuth.GetBearerToken(
+				conf.getTokenCache,
+				conf.setTokenCache,
+				"https://www.googleapis.com/auth/cloud-platform",
+			)
+
+			if tok == nil && err != nil {
+				return nil, err
+			}
+
+			// add this as a bearer token
+			authInfoMap[authInfoName].Token = tok.AccessToken
+		case models.AWS:
+			awsAuth, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
+				cluster.ProjectID,
+				cluster.AWSIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			awsClusterID := cluster.Name
+			shouldOverride := false
+
+			if cluster.AWSClusterID != "" {
+				awsClusterID = cluster.AWSClusterID
+				shouldOverride = true
+			}
+
+			tok, err := awsAuth.GetBearerToken(conf.getTokenCache, conf.setTokenCache, awsClusterID, shouldOverride)
+			if err != nil {
+				return nil, err
+			}
+
+			// add this as a bearer token
+			authInfoMap[authInfoName].Token = tok
+		case models.DO:
+			oauthInt, err := conf.Repo.OAuthIntegration().ReadOAuthIntegration(
+				cluster.ProjectID,
+				cluster.DOIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, conf.DigitalOceanOAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, conf.Repo))
+			if err != nil {
+				return nil, err
+			}
+
+			// add this as a bearer token
+			authInfoMap[authInfoName].Token = tok
+		case models.Azure:
+			azInt, err := conf.Repo.AzureIntegration().ReadAzureIntegration(
+				cluster.ProjectID,
+				cluster.AzureIntegrationID,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfoMap[authInfoName].Token = string(azInt.AKSPassword)
+		default:
+			return nil, errors.New("not a supported auth mechanism")
 		}
-
-		authInfoMap[authInfoName].Token = string(azInt.AKSPassword)
-	default:
-		return nil, errors.New("not a supported auth mechanism")
 	}
 
 	// create a context of the cluster name
@@ -527,6 +569,24 @@ func (conf *OutOfClusterConfig) setTokenCache(token string, expiry time.Time) er
 	return err
 }
 
+func capiCertAuthData(ccpClient porterv1connect.ClusterControlPlaneServiceClient, clusterId, projectId int) ([]byte, error) {
+	req := connect.NewRequest(&porterv1.CertificateAuthorityDataRequest{
+		ProjectId: int64(projectId),
+		ClusterId: int64(clusterId),
+	})
+	cert, err := ccpClient.CertificateAuthorityData(context.Background(), req)
+	if err != nil {
+		return []byte(""), fmt.Errorf("error getting certificate authority data: %w", err)
+	}
+
+	decodedCert, err := b64.DecodeString(cert.Msg.CertificateAuthorityData)
+	if err != nil {
+		return []byte(""), fmt.Errorf("error decoding certificate authority data: %w", err)
+	}
+
+	return decodedCert, nil
+}
+
 // NewRESTClientGetterFromInClusterConfig returns a RESTClientGetter using
 // default values set from the *rest.Config
 func NewRESTClientGetterFromInClusterConfig(conf *rest.Config, namespace string) genericclioptions.RESTClientGetter {