update releases upgrades with postrenderer

internal/helm/agent.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+	"github.com/porter-dev/porter/internal/kubernetes"
+	"github.com/porter-dev/porter/internal/models"
+	"github.com/porter-dev/porter/internal/repository"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/release"
@@ -13,6 +16,7 @@ import (
 // Agent is a Helm agent for performing helm operations
 type Agent struct {
 	ActionConfig *action.Configuration
+	K8sAgent     *kubernetes.Agent
 }
 
 // ListReleases lists releases based on a ListFilter
@@ -49,9 +53,17 @@ func (a *Agent) GetReleaseHistory(
 	return cmd.Run(name)
 }
 
+type UpgradeReleaseConfig struct {
+	Name       string
+	Values     map[string]interface{}
+	Cluster    *models.Cluster
+	Repo       repository.Repository
+	Registries []*models.Registry
+}
+
 // UpgradeRelease upgrades a specific release with new values.yaml
 func (a *Agent) UpgradeRelease(
-	name string,
+	conf *UpgradeReleaseConfig,
 	values string,
 ) (*release.Release, error) {
 	valuesYaml, err := chartutil.ReadValues([]byte(values))
@@ -60,16 +72,17 @@ func (a *Agent) UpgradeRelease(
 		return nil, fmt.Errorf("Values could not be parsed: %v", err)
 	}
 
-	return a.UpgradeReleaseByValues(name, valuesYaml)
+	conf.Values = valuesYaml
+
+	return a.UpgradeReleaseByValues(conf)
 }
 
 // UpgradeReleaseByValues upgrades a release by unmarshaled yaml values
 func (a *Agent) UpgradeReleaseByValues(
-	name string,
-	values map[string]interface{},
+	conf *UpgradeReleaseConfig,
 ) (*release.Release, error) {
 	// grab the latest release
-	rel, err := a.GetRelease(name, 0)
+	rel, err := a.GetRelease(conf.Name, 0)
 
 	if err != nil {
 		return nil, fmt.Errorf("Could not get release to be upgraded: %v", err)
@@ -78,7 +91,22 @@ func (a *Agent) UpgradeReleaseByValues(
 	ch := rel.Chart
 
 	cmd := action.NewUpgrade(a.ActionConfig)
-	res, err := cmd.Run(name, ch, values)
+
+	if conf.Cluster != nil && a.K8sAgent != nil && conf.Registries != nil && len(conf.Registries) > 0 {
+		cmd.PostRenderer, err = NewDockerSecretsPostRenderer(
+			conf.Cluster,
+			conf.Repo,
+			a.K8sAgent,
+			rel.Namespace,
+			conf.Registries,
+		)
+
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	res, err := cmd.Run(conf.Name, ch, conf.Values)
 
 	if err != nil {
 		return nil, fmt.Errorf("Upgrade failed: %v", err)
@@ -89,10 +117,13 @@ func (a *Agent) UpgradeReleaseByValues(
 
 // InstallChartConfig is the config required to install a chart
 type InstallChartConfig struct {
-	Chart     *chart.Chart
-	Name      string
-	Namespace string
-	Values    map[string]interface{}
+	Chart      *chart.Chart
+	Name       string
+	Namespace  string
+	Values     map[string]interface{}
+	Cluster    *models.Cluster
+	Repo       repository.Repository
+	Registries []*models.Registry
 }
 
 // InstallChartFromValuesBytes reads the raw values and calls Agent.InstallChart
@@ -128,9 +159,22 @@ func (a *Agent) InstallChart(
 		return nil, err
 	}
 
-	// if chartRequested.Metadata.Deprecated {
-	// 	return nil, fmt.Errorf("This chart is deprecated")
-	// }
+	var err error
+
+	// only add the postrenderer if required fields exist
+	if conf.Cluster != nil && a.K8sAgent != nil && conf.Registries != nil && len(conf.Registries) > 0 {
+		cmd.PostRenderer, err = NewDockerSecretsPostRenderer(
+			conf.Cluster,
+			conf.Repo,
+			a.K8sAgent,
+			conf.Namespace,
+			conf.Registries,
+		)
+
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	if req := conf.Chart.Metadata.Dependencies; req != nil {
 		if err := action.CheckDependencies(conf.Chart, req); err != nil {

internal/helm/config.go

@@ -52,12 +52,15 @@ func GetAgentFromK8sAgent(stg string, ns string, l *logger.Logger, k8sAgent *kub
 	}
 
 	// use k8s agent to create Helm agent
-	return &Agent{&action.Configuration{
-		RESTClientGetter: k8sAgent.RESTClientGetter,
-		KubeClient:       kube.New(k8sAgent.RESTClientGetter),
-		Releases:         StorageMap[stg](l, clientset.CoreV1(), ns),
-		Log:              l.Printf,
-	}}, nil
+	return &Agent{
+		ActionConfig: &action.Configuration{
+			RESTClientGetter: k8sAgent.RESTClientGetter,
+			KubeClient:       kube.New(k8sAgent.RESTClientGetter),
+			Releases:         StorageMap[stg](l, clientset.CoreV1(), ns),
+			Log:              l.Printf,
+		},
+		K8sAgent: k8sAgent,
+	}, nil
 }
 
 // GetAgentInClusterConfig creates a new Agent from inside the cluster using
@@ -77,12 +80,15 @@ func GetAgentInClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
 	}
 
 	// use k8s agent to create Helm agent
-	return &Agent{&action.Configuration{
-		RESTClientGetter: k8sAgent.RESTClientGetter,
-		KubeClient:       kube.New(k8sAgent.RESTClientGetter),
-		Releases:         StorageMap[form.Storage](l, clientset.CoreV1(), form.Namespace),
-		Log:              l.Printf,
-	}}, nil
+	return &Agent{
+		ActionConfig: &action.Configuration{
+			RESTClientGetter: k8sAgent.RESTClientGetter,
+			KubeClient:       kube.New(k8sAgent.RESTClientGetter),
+			Releases:         StorageMap[form.Storage](l, clientset.CoreV1(), form.Namespace),
+			Log:              l.Printf,
+		},
+		K8sAgent: k8sAgent,
+	}, nil
 }
 
 // GetAgentTesting creates a new Agent using an optional existing storage class
@@ -93,14 +99,16 @@ func GetAgentTesting(form *Form, storage *storage.Storage, l *logger.Logger) *Ag
 		testStorage = StorageMap["memory"](nil, nil, "")
 	}
 
-	return &Agent{&action.Configuration{
-		Releases: testStorage,
-		KubeClient: &kubefake.FailingKubeClient{
-			PrintingKubeClient: kubefake.PrintingKubeClient{
-				Out: ioutil.Discard,
+	return &Agent{
+		ActionConfig: &action.Configuration{
+			Releases: testStorage,
+			KubeClient: &kubefake.FailingKubeClient{
+				PrintingKubeClient: kubefake.PrintingKubeClient{
+					Out: ioutil.Discard,
+				},
 			},
+			Capabilities: chartutil.DefaultCapabilities,
+			Log:          l.Printf,
 		},
-		Capabilities: chartutil.DefaultCapabilities,
-		Log:          l.Printf,
-	}}
+	}
 }

internal/helm/postrenderer.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"net/url"
 	"regexp"
 	"strings"
 
@@ -33,20 +34,56 @@ type DockerSecretsPostRenderer struct {
 
 	registries map[string]*models.Registry
 
-	podSpecs []resource
+	podSpecs  []resource
+	resources []resource
 }
 
 // while manifests are map[string]interface{} at the top level,
 // nested keys will be of type map[interface{}]interface{}
 type resource map[interface{}]interface{}
 
-func NewDockerSecretsPostRenderer() (postrender.PostRenderer, error) {
-	// Registries is a map of registry URLs to registry ids. Input
-	// registries should be parsed of protocol, trailing slashes, and
-	// whitespace -- TODO
+func NewDockerSecretsPostRenderer(
+	cluster *models.Cluster,
+	repo repository.Repository,
+	agent *kubernetes.Agent,
+	namespace string,
+	regs []*models.Registry,
+) (postrender.PostRenderer, error) {
+	// Registries is a map of registry URLs to registry ids
+	registries := make(map[string]*models.Registry)
+
+	for _, reg := range regs {
+		regURL := reg.URL
+
+		if !strings.Contains(regURL, "http") {
+			regURL = "https://" + regURL
+		}
+
+		parsedRegURL, err := url.Parse(regURL)
+
+		if err != nil {
+			continue
+		}
+
+		addReg := parsedRegURL.Host
+
+		if parsedRegURL.Path != "" {
+			addReg += "/" + parsedRegURL.Path
+		}
+
+		registries[addReg] = reg
+
+		fmt.Println("ADDED REGISTRIES", addReg)
+	}
 
 	return &DockerSecretsPostRenderer{
-		podSpecs: make([]resource, 0),
+		Cluster:    cluster,
+		Repo:       repo,
+		Agent:      agent,
+		Namespace:  namespace,
+		registries: registries,
+		podSpecs:   make([]resource, 0),
+		resources:  make([]resource, 0),
 	}, nil
 }
 
@@ -70,9 +107,25 @@ func (d *DockerSecretsPostRenderer) Run(
 		linkedRegs,
 	)
 
-	fmt.Println(secrets, err)
+	if err != nil {
+		return renderedManifests, nil
+	}
+
+	d.updatePodSpecs(secrets)
+
+	modifiedManifests = bytes.NewBuffer([]byte{})
+	encoder := yaml.NewEncoder(modifiedManifests)
+	defer encoder.Close()
+
+	for _, resource := range d.resources {
+		err = encoder.Encode(resource)
 
-	return renderedManifests, nil
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return modifiedManifests, nil
 }
 
 func (d *DockerSecretsPostRenderer) getRegistriesToLink(renderedManifests *bytes.Buffer) (map[string]*models.Registry, error) {
@@ -80,14 +133,14 @@ func (d *DockerSecretsPostRenderer) getRegistriesToLink(renderedManifests *bytes
 	// that a secret will be generated for, if it does not exist
 	linkedRegs := make(map[string]*models.Registry)
 
-	resources, err := d.decodeRenderedManifests(renderedManifests)
+	err := d.decodeRenderedManifests(renderedManifests)
 
 	if err != nil {
 		return linkedRegs, err
 	}
 
 	// read the pod specs into the post-renderer object
-	d.getPodSpecs(resources)
+	d.getPodSpecs(d.resources)
 
 	for _, podSpec := range d.podSpecs {
 		// get all images
@@ -133,12 +186,10 @@ func (d *DockerSecretsPostRenderer) getRegistriesToLink(renderedManifests *bytes
 
 func (d *DockerSecretsPostRenderer) decodeRenderedManifests(
 	renderedManifests *bytes.Buffer,
-) ([]resource, error) {
+) error {
 	// use the yaml decoder to parse the multi-document yaml.
 	decoder := yaml.NewDecoder(renderedManifests)
 
-	resources := make([]resource, 0)
-
 	for {
 		res := make(resource)
 		err := decoder.Decode(&res)
@@ -147,13 +198,13 @@ func (d *DockerSecretsPostRenderer) decodeRenderedManifests(
 		}
 
 		if err != nil {
-			return nil, err
+			return err
 		}
 
-		resources = append(resources, res)
+		d.resources = append(d.resources, res)
 	}
 
-	return resources, nil
+	return nil
 }
 
 func (d *DockerSecretsPostRenderer) getPodSpecs(resources []resource) {
@@ -199,6 +250,75 @@ func (d *DockerSecretsPostRenderer) getPodSpecs(resources []resource) {
 	return
 }
 
+func (d *DockerSecretsPostRenderer) updatePodSpecs(secrets map[string]string) {
+	for _, podSpec := range d.podSpecs {
+		fmt.Println("PARSING POD SPEC", podSpec)
+
+		containersVal, hasContainers := podSpec["containers"]
+
+		if !hasContainers {
+			continue
+		}
+
+		containers, ok := containersVal.([]interface{})
+
+		if !ok {
+			continue
+		}
+
+		var imagePullSecrets []map[string]interface{}
+		existingNames := map[string]bool{}
+		if existingPullSecrets, ok := podSpec["imagePullSecrets"]; ok {
+			imagePullSecrets = existingPullSecrets.([]map[string]interface{})
+			for _, s := range imagePullSecrets {
+				if name, ok := s["name"]; ok {
+					if n, ok := name.(string); ok {
+						existingNames[n] = true
+					}
+				}
+			}
+		}
+
+		for _, container := range containers {
+			_container, ok := container.(resource)
+
+			if !ok {
+				continue
+			}
+
+			image, ok := _container["image"].(string)
+
+			if !ok {
+				continue
+			}
+
+			named, err := reference.ParseNormalizedNamed(image)
+
+			if err != nil {
+				continue
+			}
+
+			domain := reference.Domain(named)
+			path := reference.Path(named)
+
+			regName := domain
+
+			if pathArr := strings.Split(path, "/"); len(pathArr) > 1 {
+				regName += "/" + strings.Join(pathArr[:len(pathArr)-1], "/")
+			}
+
+			imagePullSecrets = append(imagePullSecrets, map[string]interface{}{
+				"name": secrets[regName],
+			})
+		}
+
+		if len(imagePullSecrets) > 0 {
+			podSpec["imagePullSecrets"] = imagePullSecrets
+		}
+
+	}
+}
+
 func (d *DockerSecretsPostRenderer) getImageList(podSpec resource) []string {
 	images := make([]string, 0)
 

internal/helm/postrenderer_test.go

@@ -1,118 +0,0 @@
-package helm_test
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/porter-dev/porter/internal/helm"
-)
-
-func TestDockerSecretsPostRenderer(t *testing.T) {
-	testCases := []struct {
-		name   string
-		input  *bytes.Buffer
-		output *bytes.Buffer
-	}{
-		{
-			name: "it appends relevant image pull secret for nested lists of resources",
-			input: bytes.NewBuffer([]byte(`apiVersion: v1
-kind: PodTemplateList
-metadata:
-  annotations:
-    annotation-1: some-annotation
-  name: image-secret-test
-items:
-- kind: PodTemplate
-  template:
-    spec:
-      containers:
-      - command:
-        - sh
-        - -c
-        - echo 'foo'
-        env:
-        - name: SOME_ENV
-          value: env_value
-        image: example.com/bitnami/nginx:1.16.1-debian-10-r42
-        name: container-name
-      restartPolicy: Never
-- kind: PodTemplate
-  template:
-    spec:
-      containers:
-      - command:
-        - sh
-        - -c
-        - echo 'bar'
-        env:
-        - name: SOME_ENV
-          value: env_value
-        image: example.com/bitnami/nginx:1.16.1-debian-10-r42
-        name: container-name
-      restartPolicy: Never
----
-kind: Unknown
-other: doc
-`)),
-			output: bytes.NewBuffer([]byte(`apiVersion: v1
-items:
-- kind: PodTemplate
-  template:
-    spec:
-      containers:
-      - command:
-        - sh
-        - -c
-        - echo 'foo'
-        env:
-        - name: SOME_ENV
-          value: env_value
-        image: example.com/bitnami/nginx:1.16.1-debian-10-r42
-        name: container-name
-      imagePullSecrets:
-      - name: secret-1
-      restartPolicy: Never
-- kind: PodTemplate
-  template:
-    spec:
-      containers:
-      - command:
-        - sh
-        - -c
-        - echo 'bar'
-        env:
-        - name: SOME_ENV
-          value: env_value
-        image: example.com/bitnami/nginx:1.16.1-debian-10-r42
-        name: container-name
-      imagePullSecrets:
-      - name: secret-1
-      restartPolicy: Never
-kind: PodTemplateList
-metadata:
-  annotations:
-    annotation-1: some-annotation
-  name: image-secret-test
----
-kind: Unknown
-other: doc
-`)),
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			r, err := helm.NewDockerSecretsPostRenderer()
-			if err != nil {
-				t.Fatalf("%+v", err)
-			}
-
-			renderedManifests, err := r.Run(tc.input)
-
-			if got, want := renderedManifests.String(), tc.output.String(); !cmp.Equal(got, want) {
-				t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(want, got))
-			}
-		})
-	}
-}

internal/kubernetes/agent.go

@@ -394,9 +394,13 @@ func (a *Agent) CreateImagePullSecrets(
 			_, err := a.Clientset.CoreV1().Secrets(namespace).Update(
 				context.TODO(),
 				&v1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: secretName,
+					},
 					Data: map[string][]byte{
 						string(v1.DockerConfigJsonKey): data,
 					},
+					Type: v1.SecretTypeDockerConfigJson,
 				},
 				metav1.UpdateOptions{},
 			)

internal/templater/helm/values/writer.go

@@ -59,7 +59,12 @@ func (w *TemplateWriter) Update(
 		return nil, fmt.Errorf("release not set")
 	}
 
-	_, err := w.Agent.UpgradeReleaseByValues(w.ReleaseName, vals)
+	conf := &helm.UpgradeReleaseConfig{
+		Name:   w.ReleaseName,
+		Values: vals,
+	}
+
+	_, err := w.Agent.UpgradeReleaseByValues(conf)
 
 	if err != nil {
 		return nil, err

server/api/deploy_handler.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"strconv"
 
 	"github.com/go-chi/chi"
 	"github.com/porter-dev/porter/internal/forms"
@@ -15,6 +16,13 @@ import (
 
 // HandleDeployTemplate triggers a chart deployment from a template
 func (app *App) HandleDeployTemplate(w http.ResponseWriter, r *http.Request) {
+	projID, err := strconv.ParseUint(chi.URLParam(r, "project_id"), 0, 64)
+
+	if err != nil || projID == 0 {
+		app.handleErrorFormDecoding(err, ErrProjectDecode, w)
+		return
+	}
+
 	name := chi.URLParam(r, "name")
 	version := chi.URLParam(r, "version")
 
@@ -76,11 +84,21 @@ func (app *App) HandleDeployTemplate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	registries, err := app.Repo.Registry.ListRegistriesByProjectID(uint(projID))
+
+	if err != nil {
+		app.handleErrorDataRead(err, w)
+		return
+	}
+
 	conf := &helm.InstallChartConfig{
-		Chart:     chart,
-		Name:      form.ChartTemplateForm.Name,
-		Namespace: form.ReleaseForm.Form.Namespace,
-		Values:    form.ChartTemplateForm.FormValues,
+		Chart:      chart,
+		Name:       form.ChartTemplateForm.Name,
+		Namespace:  form.ReleaseForm.Form.Namespace,
+		Values:     form.ChartTemplateForm.FormValues,
+		Cluster:    form.ReleaseForm.Cluster,
+		Repo:       *app.Repo,
+		Registries: registries,
 	}
 
 	_, err = agent.InstallChart(conf)

server/api/release_handler.go

@@ -430,6 +430,13 @@ func (app *App) HandleGetReleaseToken(w http.ResponseWriter, r *http.Request) {
 
 // HandleUpgradeRelease upgrades a release with new values.yaml
 func (app *App) HandleUpgradeRelease(w http.ResponseWriter, r *http.Request) {
+	projID, err := strconv.ParseUint(chi.URLParam(r, "project_id"), 0, 64)
+
+	if err != nil || projID == 0 {
+		app.handleErrorFormDecoding(err, ErrProjectDecode, w)
+		return
+	}
+
 	name := chi.URLParam(r, "name")
 
 	vals, err := url.ParseQuery(r.URL.RawQuery)
@@ -469,7 +476,21 @@ func (app *App) HandleUpgradeRelease(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	_, err = agent.UpgradeRelease(form.Name, form.Values)
+	registries, err := app.Repo.Registry.ListRegistriesByProjectID(uint(projID))
+
+	if err != nil {
+		app.handleErrorDataRead(err, w)
+		return
+	}
+
+	conf := &helm.UpgradeReleaseConfig{
+		Name:       form.Name,
+		Cluster:    form.ReleaseForm.Cluster,
+		Repo:       *app.Repo,
+		Registries: registries,
+	}
+
+	_, err = agent.UpgradeRelease(conf, form.Values)
 
 	if err != nil {
 		app.sendExternalError(err, http.StatusInternalServerError, HTTPError{
@@ -533,7 +554,22 @@ func (app *App) HandleReleaseDeployHook(w http.ResponseWriter, r *http.Request)
 	newval := map[string]interface{}{}
 	newval["image"] = image
 
-	_, err = agent.UpgradeReleaseByValues(form.Name, newval)
+	registries, err := app.Repo.Registry.ListRegistriesByProjectID(uint(form.ReleaseForm.Cluster.ProjectID))
+
+	if err != nil {
+		app.handleErrorDataRead(err, w)
+		return
+	}
+
+	conf := &helm.UpgradeReleaseConfig{
+		Name:       form.Name,
+		Cluster:    form.ReleaseForm.Cluster,
+		Repo:       *app.Repo,
+		Registries: registries,
+		Values:     newval,
+	}
+
+	_, err = agent.UpgradeReleaseByValues(conf)
 
 	if err != nil {
 		app.sendExternalError(err, http.StatusInternalServerError, HTTPError{
@@ -610,7 +646,22 @@ func (app *App) HandleReleaseDeployWebhook(w http.ResponseWriter, r *http.Reques
 	newval := map[string]interface{}{}
 	newval["image"] = image
 
-	_, err = agent.UpgradeReleaseByValues(form.Name, newval)
+	registries, err := app.Repo.Registry.ListRegistriesByProjectID(uint(form.ReleaseForm.Cluster.ProjectID))
+
+	if err != nil {
+		app.handleErrorDataRead(err, w)
+		return
+	}
+
+	conf := &helm.UpgradeReleaseConfig{
+		Name:       form.Name,
+		Cluster:    form.ReleaseForm.Cluster,
+		Repo:       *app.Repo,
+		Registries: registries,
+		Values:     newval,
+	}
+
+	_, err = agent.UpgradeReleaseByValues(conf)
 
 	if err != nil {
 		app.sendExternalError(err, http.StatusInternalServerError, HTTPError{