add helm repo endpoints for deploying from custom helm regs

api/server/authz/cluster.go

@@ -86,9 +86,10 @@ func NewOutOfClusterAgentGetter(config *config.Config) KubernetesAgentGetter {
 
 func (d *OutOfClusterAgentGetter) GetOutOfClusterConfig(cluster *models.Cluster) *kubernetes.OutOfClusterConfig {
 	return &kubernetes.OutOfClusterConfig{
-		Repo:              d.config.Repo,
-		DigitalOceanOAuth: d.config.DOConf,
-		Cluster:           cluster,
+		Repo:                      d.config.Repo,
+		DigitalOceanOAuth:         d.config.DOConf,
+		Cluster:                   cluster,
+		AllowInClusterConnections: d.config.ServerConf.InitInCluster,
 	}
 }
 

api/server/handlers/helmrepo/create.go

@@ -0,0 +1,76 @@
+package helmrepo
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/porter-dev/porter/api/server/handlers"
+	"github.com/porter-dev/porter/api/server/shared"
+	"github.com/porter-dev/porter/api/server/shared/apierrors"
+	"github.com/porter-dev/porter/api/server/shared/config"
+	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/internal/models"
+	"gorm.io/gorm"
+)
+
+type HelmRepoCreateHandler struct {
+	handlers.PorterHandlerReadWriter
+}
+
+func NewHelmRepoCreateHandler(
+	config *config.Config,
+	decoderValidator shared.RequestDecoderValidator,
+	writer shared.ResultWriter,
+) *HelmRepoCreateHandler {
+	return &HelmRepoCreateHandler{
+		PorterHandlerReadWriter: handlers.NewDefaultPorterHandler(config, decoderValidator, writer),
+	}
+}
+
+func (p *HelmRepoCreateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	proj, _ := r.Context().Value(types.ProjectScope).(*models.Project)
+
+	request := &types.CreateHelmRepoRequest{}
+
+	ok := p.DecodeAndValidate(w, r, request)
+
+	if !ok {
+		return
+	}
+
+	// if a basic integration is specified, verify that it exists in the project
+	if request.BasicIntegrationID != 0 {
+		_, err := p.Repo().BasicIntegration().ReadBasicIntegration(proj.ID, request.BasicIntegrationID)
+
+		if err != nil {
+			if errors.Is(err, gorm.ErrRecordNotFound) {
+				p.HandleAPIError(w, r, apierrors.NewErrForbidden(
+					fmt.Errorf("basic integration with id %d not found in project %d", request.BasicIntegrationID, proj.ID),
+				))
+
+				return
+			}
+
+			p.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+			return
+		}
+	}
+
+	hr := &models.HelmRepo{
+		Name:                   request.Name,
+		ProjectID:              proj.ID,
+		RepoURL:                request.URL,
+		BasicAuthIntegrationID: request.BasicIntegrationID,
+	}
+
+	// handle write to the database
+	hr, err := p.Repo().HelmRepo().CreateHelmRepo(hr)
+
+	if err != nil {
+		p.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		return
+	}
+
+	p.WriteResult(w, r, hr.ToHelmRepoType())
+}

api/server/handlers/helmrepo/list.go

@@ -0,0 +1,44 @@
+package helmrepo
+
+import (
+	"net/http"
+
+	"github.com/porter-dev/porter/api/server/handlers"
+	"github.com/porter-dev/porter/api/server/shared"
+	"github.com/porter-dev/porter/api/server/shared/apierrors"
+	"github.com/porter-dev/porter/api/server/shared/config"
+	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/internal/models"
+)
+
+type HelmRepoListHandler struct {
+	handlers.PorterHandlerWriter
+}
+
+func NewHelmRepoListHandler(
+	config *config.Config,
+	writer shared.ResultWriter,
+) *HelmRepoListHandler {
+	return &HelmRepoListHandler{
+		PorterHandlerWriter: handlers.NewDefaultPorterHandler(config, nil, writer),
+	}
+}
+
+func (c *HelmRepoListHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	proj, _ := r.Context().Value(types.ProjectScope).(*models.Project)
+
+	hrs, err := c.Repo().HelmRepo().ListHelmReposByProjectID(proj.ID)
+
+	if err != nil {
+		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		return
+	}
+
+	res := make([]*types.HelmRepo, 0)
+
+	for _, hr := range hrs {
+		res = append(res, hr.ToHelmRepoType())
+	}
+
+	c.WriteResult(w, r, res)
+}

api/server/handlers/helmrepo/list_charts.go

@@ -0,0 +1,63 @@
+package helmrepo
+
+import (
+	"net/http"
+
+	"k8s.io/helm/pkg/repo"
+
+	"github.com/porter-dev/porter/api/server/handlers"
+	"github.com/porter-dev/porter/api/server/shared"
+	"github.com/porter-dev/porter/api/server/shared/apierrors"
+	"github.com/porter-dev/porter/api/server/shared/config"
+	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/internal/helm/loader"
+	"github.com/porter-dev/porter/internal/models"
+)
+
+type ChartListHandler struct {
+	handlers.PorterHandlerReadWriter
+}
+
+func NewChartListHandler(
+	config *config.Config,
+	decoderValidator shared.RequestDecoderValidator,
+	writer shared.ResultWriter,
+) *ChartListHandler {
+	return &ChartListHandler{
+		PorterHandlerReadWriter: handlers.NewDefaultPorterHandler(config, decoderValidator, writer),
+	}
+}
+
+func (t *ChartListHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	proj, _ := r.Context().Value(types.ProjectScope).(*models.Project)
+	helmRepo, _ := r.Context().Value(types.HelmRepoScope).(*models.HelmRepo)
+
+	var repoIndex *repo.IndexFile
+	var err error
+
+	if helmRepo.BasicAuthIntegrationID != 0 {
+		// read the basic integration id
+		basic, err := t.Repo().BasicIntegration().ReadBasicIntegration(proj.ID, helmRepo.BasicAuthIntegrationID)
+
+		if err != nil {
+			t.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+			return
+		}
+
+		repoIndex, err = loader.LoadRepoIndex(&loader.BasicAuthClient{
+			Username: string(basic.Username),
+			Password: string(basic.Password),
+		}, helmRepo.RepoURL)
+	} else {
+		repoIndex, err = loader.LoadRepoIndexPublic(helmRepo.RepoURL)
+	}
+
+	if err != nil {
+		t.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		return
+	}
+
+	charts := loader.RepoIndexToPorterChartList(repoIndex)
+
+	t.WriteResult(w, r, charts)
+}

api/server/handlers/release/create_addon.go

@@ -15,6 +15,7 @@ import (
 	"github.com/porter-dev/porter/internal/helm/loader"
 	"github.com/porter-dev/porter/internal/models"
 	"github.com/porter-dev/porter/internal/oauth"
+	"helm.sh/helm/v3/pkg/chart"
 )
 
 type CreateAddonHandler struct {
@@ -35,6 +36,7 @@ func NewCreateAddonHandler(
 
 func (c *CreateAddonHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	user, _ := r.Context().Value(types.UserScope).(*models.User)
+	proj, _ := r.Context().Value(types.ProjectScope).(*models.Project)
 	cluster, _ := r.Context().Value(types.ClusterScope).(*models.Cluster)
 	namespace := r.Context().Value(types.NamespaceScope).(string)
 	operationID := oauth.CreateRandomState()
@@ -63,7 +65,12 @@ func (c *CreateAddonHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		request.TemplateVersion = ""
 	}
 
-	chart, err := loader.LoadChartPublic(request.RepoURL, request.TemplateName, request.TemplateVersion)
+	chart, err := LoadChart(c.Config(), &LoadAddonChartOpts{
+		ProjectID:       proj.ID,
+		RepoURL:         request.RepoURL,
+		TemplateName:    request.TemplateName,
+		TemplateVersion: request.TemplateVersion,
+	})
 
 	if err != nil {
 		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
@@ -112,3 +119,45 @@ func (c *CreateAddonHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		},
 	))
 }
+
+type LoadAddonChartOpts struct {
+	ProjectID                              uint
+	RepoURL, TemplateName, TemplateVersion string
+}
+
+func LoadChart(config *config.Config, opts *LoadAddonChartOpts) (*chart.Chart, error) {
+	// if the chart repo url is one of the specified application/addon charts, just load public
+	if opts.RepoURL == config.ServerConf.DefaultAddonHelmRepoURL || opts.RepoURL == config.ServerConf.DefaultApplicationHelmRepoURL {
+		return loader.LoadChartPublic(opts.RepoURL, opts.TemplateName, opts.TemplateVersion)
+	} else {
+		// load the helm repos in the project
+		hrs, err := config.Repo.HelmRepo().ListHelmReposByProjectID(opts.ProjectID)
+
+		if err != nil {
+			return nil, err
+		}
+
+		for _, hr := range hrs {
+			if hr.RepoURL == opts.RepoURL {
+				if hr.BasicAuthIntegrationID != 0 {
+					// read the basic integration id
+					basic, err := config.Repo.BasicIntegration().ReadBasicIntegration(opts.ProjectID, hr.BasicAuthIntegrationID)
+
+					if err != nil {
+
+						return nil, err
+					}
+
+					return loader.LoadChart(&loader.BasicAuthClient{
+						Username: string(basic.Username),
+						Password: string(basic.Password),
+					}, hr.RepoURL, opts.TemplateName, opts.TemplateVersion)
+				} else {
+					return loader.LoadChartPublic(hr.RepoURL, opts.TemplateName, opts.TemplateVersion)
+				}
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("chart repo not found")
+}

api/server/handlers/release/ugprade.go

@@ -14,7 +14,6 @@ import (
 	"github.com/porter-dev/porter/api/server/shared/config"
 	"github.com/porter-dev/porter/api/types"
 	"github.com/porter-dev/porter/internal/helm"
-	"github.com/porter-dev/porter/internal/helm/loader"
 	"github.com/porter-dev/porter/internal/integrations/slack"
 	"github.com/porter-dev/porter/internal/models"
 	"helm.sh/helm/v3/pkg/release"
@@ -94,11 +93,17 @@ func (c *UpgradeReleaseHandler) ServeHTTP(w http.ResponseWriter, r *http.Request
 			}
 		}
 
-		chart, err := loader.LoadChartPublic(
-			chartRepoURL,
-			helmRelease.Chart.Metadata.Name,
-			request.ChartVersion,
-		)
+		chart, err := LoadChart(c.Config(), &LoadAddonChartOpts{
+			ProjectID:       cluster.ProjectID,
+			RepoURL:         chartRepoURL,
+			TemplateName:    helmRelease.Chart.Metadata.Name,
+			TemplateVersion: request.ChartVersion,
+		})
+
+		if err != nil {
+			c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+			return
+		}
 
 		if err != nil {
 			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(

api/server/router/helm_repo.go

@@ -80,5 +80,34 @@ func getHelmRepoRoutes(
 		Router:   r,
 	})
 
+	//  GET /api/projects/{project_id}/helmrepos/{helm_repo_id}/charts -> helmrepo.NewChartListHandler
+	hrListEndpoint := factory.NewAPIEndpoint(
+		&types.APIRequestMetadata{
+			Verb:   types.APIVerbList,
+			Method: types.HTTPVerbGet,
+			Path: &types.Path{
+				Parent:       basePath,
+				RelativePath: relPath + "/charts",
+			},
+			Scopes: []types.PermissionScope{
+				types.UserScope,
+				types.ProjectScope,
+				types.HelmRepoScope,
+			},
+		},
+	)
+
+	hrListHandler := helmrepo.NewChartListHandler(
+		config,
+		factory.GetDecoderValidator(),
+		factory.GetResultWriter(),
+	)
+
+	routes = append(routes, &Route{
+		Endpoint: hrListEndpoint,
+		Handler:  hrListHandler,
+		Router:   r,
+	})
+
 	return routes, newPath
 }

api/server/router/project.go

@@ -5,6 +5,7 @@ import (
 	"github.com/porter-dev/porter/api/server/handlers/billing"
 	"github.com/porter-dev/porter/api/server/handlers/cluster"
 	"github.com/porter-dev/porter/api/server/handlers/gitinstallation"
+	"github.com/porter-dev/porter/api/server/handlers/helmrepo"
 	"github.com/porter-dev/porter/api/server/handlers/project"
 	"github.com/porter-dev/porter/api/server/handlers/provision"
 	"github.com/porter-dev/porter/api/server/handlers/registry"
@@ -806,5 +807,60 @@ func getProjectRoutes(
 		Router:   r,
 	})
 
+	//  POST /api/projects/{project_id}/helmrepos -> helmrepo.NewHelmRepoCreateHandler
+	hrCreateEndpoint := factory.NewAPIEndpoint(
+		&types.APIRequestMetadata{
+			Verb:   types.APIVerbCreate,
+			Method: types.HTTPVerbPost,
+			Path: &types.Path{
+				Parent:       basePath,
+				RelativePath: relPath + "/helmrepos",
+			},
+			Scopes: []types.PermissionScope{
+				types.UserScope,
+				types.ProjectScope,
+			},
+		},
+	)
+
+	hrCreateHandler := helmrepo.NewHelmRepoCreateHandler(
+		config,
+		factory.GetDecoderValidator(),
+		factory.GetResultWriter(),
+	)
+
+	routes = append(routes, &Route{
+		Endpoint: hrCreateEndpoint,
+		Handler:  hrCreateHandler,
+		Router:   r,
+	})
+
+	//  GET /api/projects/{project_id}/helmrepos -> helmrepo.NewHelmRepoListHandler
+	hrListEndpoint := factory.NewAPIEndpoint(
+		&types.APIRequestMetadata{
+			Verb:   types.APIVerbList,
+			Method: types.HTTPVerbGet,
+			Path: &types.Path{
+				Parent:       basePath,
+				RelativePath: relPath + "/helmrepos",
+			},
+			Scopes: []types.PermissionScope{
+				types.UserScope,
+				types.ProjectScope,
+			},
+		},
+	)
+
+	hrListHandler := helmrepo.NewHelmRepoListHandler(
+		config,
+		factory.GetResultWriter(),
+	)
+
+	routes = append(routes, &Route{
+		Endpoint: hrListEndpoint,
+		Handler:  hrListHandler,
+		Router:   r,
+	})
+
 	return routes, newPath
 }

api/types/helm_repo.go

@@ -13,3 +13,9 @@ type HelmRepo struct {
 }
 
 type GetHelmRepoResponse HelmRepo
+
+type CreateHelmRepoRequest struct {
+	URL                string `json:"url"`
+	Name               string `json:"name" form:"required"`
+	BasicIntegrationID uint   `json:"basic_integration_id"`
+}

api/types/release.go

@@ -51,6 +51,8 @@ type CreateReleaseRequest struct {
 
 type CreateAddonRequest struct {
 	*CreateReleaseBaseRequest
+
+	HelmRepoID uint `json:"helm_repo_id"`
 }
 
 type RollbackReleaseRequest struct {

cmd/app/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"log"
@@ -85,14 +86,14 @@ func initData(conf *config.Config) error {
 	// if the config specifies in-cluster connections are permitted, create a new project with a
 	// cluster that uses the in-cluster config. this will be the default project for this instance.
 	if conf.ServerConf.InitInCluster {
-		l := conf.Logger.Debug()
-		l.Msg("in-cluster config variable set: checking for default project and cluster")
+		l := conf.Logger
+		l.Debug().Msg("in-cluster config variable set: checking for default project and cluster")
 
 		// look for a project with id 1 with name of defaultProjectName
 		_, err := conf.Repo.Project().ReadProject(1)
 
-		if err == gorm.ErrRecordNotFound {
-			l.Msg("default project not found: attempting creation")
+		if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
+			l.Debug().Msg("default project not found: attempting creation")
 
 			_, err = conf.Repo.Project().CreateProject(&models.Project{
 				Name: defaultProjectName,
@@ -102,26 +103,27 @@ func initData(conf *config.Config) error {
 				return err
 			}
 
-			l.Msg("successfully created default project")
+			l.Debug().Msg("successfully created default project")
 		} else if err != nil {
 			return err
 		}
 
 		_, err = conf.Repo.Cluster().ReadCluster(1, 1)
 
-		if err == gorm.ErrRecordNotFound {
-			l.Msg("default cluster not found: attempting creation")
+		if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
+			l.Debug().Msg("default cluster not found: attempting creation")
 
 			_, err = conf.Repo.Cluster().CreateCluster(&models.Cluster{
 				Name:          defaultClusterName,
 				AuthMechanism: models.InCluster,
+				ProjectID:     1,
 			})
 
 			if err != nil {
 				return err
 			}
 
-			l.Msg("successfully created default cluster")
+			l.Debug().Msg("successfully created default cluster")
 		} else if err != nil {
 			return err
 		}

internal/helm/config.go

@@ -20,11 +20,12 @@ import (
 // Form represents the options for connecting to a cluster and
 // creating a Helm agent
 type Form struct {
-	Cluster           *models.Cluster `form:"required"`
-	Repo              repository.Repository
-	DigitalOceanOAuth *oauth2.Config
-	Storage           string `json:"storage" form:"oneof=secret configmap memory" default:"secret"`
-	Namespace         string `json:"namespace"`
+	Cluster                   *models.Cluster `form:"required"`
+	Repo                      repository.Repository
+	DigitalOceanOAuth         *oauth2.Config
+	Storage                   string `json:"storage" form:"oneof=secret configmap memory" default:"secret"`
+	Namespace                 string `json:"namespace"`
+	AllowInClusterConnections bool
 }
 
 // GetAgentOutOfClusterConfig creates a new Agent from outside the cluster using
@@ -32,10 +33,11 @@ type Form struct {
 func GetAgentOutOfClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
 	// create a kubernetes agent
 	conf := &kubernetes.OutOfClusterConfig{
-		Cluster:           form.Cluster,
-		DefaultNamespace:  form.Namespace,
-		Repo:              form.Repo,
-		DigitalOceanOAuth: form.DigitalOceanOAuth,
+		Cluster:                   form.Cluster,
+		DefaultNamespace:          form.Namespace,
+		Repo:                      form.Repo,
+		DigitalOceanOAuth:         form.DigitalOceanOAuth,
+		AllowInClusterConnections: form.AllowInClusterConnections,
 	}
 
 	k8sAgent, err := kubernetes.GetAgentOutOfClusterConfig(conf)

internal/redis_stream/global_stream.go

@@ -516,9 +516,10 @@ func createRDSEnvGroup(repo repository.Repository, config *config.Config, infra
 	}
 
 	ooc := &kubernetes.OutOfClusterConfig{
-		Repo:              config.Repo,
-		DigitalOceanOAuth: config.DOConf,
-		Cluster:           cluster,
+		Repo:                      config.Repo,
+		DigitalOceanOAuth:         config.DOConf,
+		Cluster:                   cluster,
+		AllowInClusterConnections: config.ServerConf.InitInCluster,
 	}
 
 	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)
@@ -563,9 +564,10 @@ func deleteRDSEnvGroup(repo repository.Repository, config *config.Config, infra
 	}
 
 	ooc := &kubernetes.OutOfClusterConfig{
-		Repo:              config.Repo,
-		DigitalOceanOAuth: config.DOConf,
-		Cluster:           cluster,
+		Repo:                      config.Repo,
+		DigitalOceanOAuth:         config.DOConf,
+		Cluster:                   cluster,
+		AllowInClusterConnections: config.ServerConf.InitInCluster,
 	}
 
 	agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)

internal/usage/usage.go

@@ -136,9 +136,10 @@ func getResourceUsage(opts *GetUsageOpts, clusters []*models.Cluster) (uint, uin
 
 	for _, cluster := range clusters {
 		ooc := &kubernetes.OutOfClusterConfig{
-			Cluster:           cluster,
-			Repo:              opts.Repo,
-			DigitalOceanOAuth: opts.DOConf,
+			Cluster:                   cluster,
+			Repo:                      opts.Repo,
+			DigitalOceanOAuth:         opts.DOConf,
+			AllowInClusterConnections: false,
 		}
 
 		agent, err := kubernetes.GetAgentOutOfClusterConfig(ooc)