form yaml helm values impl

cli/cmd/test.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/porter-dev/porter/internal/helm/loader"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/yaml"
+)
+
+var testCmd = &cobra.Command{
+	Use:   "test",
+	Short: "Testing",
+	Run: func(cmd *cobra.Command, args []string) {
+		chart, err := loader.LoadChart("https://porter-dev.github.io/chart-repo", "docker", "0.0.1")
+
+		if err != nil {
+			red := color.New(color.FgRed)
+			red.Println("Error running test:", err.Error())
+			os.Exit(1)
+		}
+
+		bytes, err := yaml.Marshal(chart)
+
+		if err != nil {
+			red := color.New(color.FgRed)
+			red.Println("Error running test:", err.Error())
+			os.Exit(1)
+		}
+
+		fmt.Println(string(bytes))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(testCmd)
+}

internal/forms/chart.go

@@ -0,0 +1,22 @@
+package forms
+
+import "net/url"
+
+// ChartForm is the base type for CRUD operations on charts
+type ChartForm struct {
+	RepoURL string
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
+// PopulateRepoURLFromQueryParams populates the repo url in the ChartForm using the passed
+// url.Values (the parsed query params)
+func (cf *ChartForm) PopulateRepoURLFromQueryParams(
+	vals url.Values,
+) error {
+	if repoURL, ok := vals["repo_url"]; ok && len(repoURL) == 1 {
+		cf.RepoURL = repoURL[0]
+	}
+
+	return nil
+}

internal/helm/config.go

@@ -40,6 +40,11 @@ func GetAgentOutOfClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
 		return nil, err
 	}
 
+	return GetAgentFromK8sAgent(form.Storage, form.Namespace, l, k8sAgent)
+}
+
+// GetAgentFromK8sAgent creates a new Agent
+func GetAgentFromK8sAgent(stg string, ns string, l *logger.Logger, k8sAgent *kubernetes.Agent) (*Agent, error) {
 	clientset, ok := k8sAgent.Clientset.(*k8s.Clientset)
 
 	if !ok {
@@ -50,7 +55,7 @@ func GetAgentOutOfClusterConfig(form *Form, l *logger.Logger) (*Agent, error) {
 	return &Agent{&action.Configuration{
 		RESTClientGetter: k8sAgent.RESTClientGetter,
 		KubeClient:       kube.New(k8sAgent.RESTClientGetter),
-		Releases:         StorageMap[form.Storage](l, clientset.CoreV1(), form.Namespace),
+		Releases:         StorageMap[stg](l, clientset.CoreV1(), ns),
 		Log:              l.Printf,
 	}}, nil
 }

internal/helm/loader/loader.go

@@ -0,0 +1,89 @@
+package loader
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"k8s.io/helm/pkg/repo"
+	"sigs.k8s.io/yaml"
+
+	"helm.sh/helm/v3/pkg/chart"
+	chartloader "helm.sh/helm/v3/pkg/chart/loader"
+)
+
+// LoadRepoIndex loads an index file from a remote Helm repo
+func LoadRepoIndex(indexURL string) (*repo.IndexFile, error) {
+	resp, err := http.Get(indexURL)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// index not found in the cache, parse it
+	index := &repo.IndexFile{}
+	err = yaml.Unmarshal(data, index)
+
+	if err != nil {
+		return index, err
+	}
+
+	index.SortEntries()
+
+	return index, nil
+}
+
+// LoadChart returns a Helm3 (v2) chart from a remote repo. If chartVersion is an
+// empty string, the most stable latest version is found.
+//
+// TODO: this is an expensive operation, so after retrieving the digest from the
+// repo index, this should check the digest in the cache
+func LoadChart(repoURL, chartName, chartVersion string) (*chart.Chart, error) {
+	trimmedRepoURL := strings.TrimSuffix(strings.TrimSpace(repoURL), "/")
+	repoIndex, err := LoadRepoIndex(trimmedRepoURL + "/index.yaml")
+
+	if err != nil {
+		return nil, err
+	}
+
+	cv, err := repoIndex.Get(chartName, chartVersion)
+
+	if err != nil {
+		return nil, err
+	} else if len(cv.URLs) == 0 {
+		return nil, fmt.Errorf("%s:%s no valid download urls", chartName, chartVersion)
+	}
+
+	chartURL := trimmedRepoURL + "/" + strings.TrimPrefix(cv.URLs[0], "/")
+
+	fmt.Println(chartURL)
+
+	// download tgz
+	resp, err := http.Get(chartURL)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+
+	// fmt.Println("DATA IS", string(data))
+
+	if err != nil {
+		return nil, err
+	}
+
+	return chartloader.LoadArchive(bytes.NewReader(data))
+}

internal/models/templates.go

@@ -23,35 +23,52 @@ type ChartYAML []struct {
 
 // PorterChart represents a bundled Porter template
 type PorterChart struct {
-	Name        string   `json:"name"`
-	Description string   `json:"description"`
-	Icon        string   `json:"icon"`
-	Form        FormYAML `json:"form"`
-	Markdown    string   `json:"markdown"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	Icon        string `json:"icon"`
+	Markdown    string `json:"markdown"`
+}
+
+// FormContext is the target context
+type FormContext struct {
+	Type   string            `yaml:"type" json:"type"`
+	Config map[string]string `yaml:"config" json:"config"`
+}
+
+// FormTab is a tab rendered in a form
+type FormTab struct {
+	Context  *FormContext   `yaml:"context" json:"context"`
+	Name     string         `yaml:"name" json:"name"`
+	Label    string         `yaml:"label" json:"label"`
+	Sections []*FormSection `yaml:"sections" json:"sections,omitempty"`
+}
+
+// FormSection is a section of a form
+type FormSection struct {
+	Context  *FormContext   `yaml:"context" json:"context"`
+	Name     string         `yaml:"name" json:"name"`
+	ShowIf   string         `yaml:"show_if" json:"show_if"`
+	Contents []*FormContent `yaml:"contents" json:"contents,omitempty"`
+}
+
+// FormContent is a form's atomic unit
+type FormContent struct {
+	Context  *FormContext `yaml:"context" json:"context"`
+	Type     string       `yaml:"type" json:"type"`
+	Label    string       `yaml:"label" json:"label"`
+	Name     string       `yaml:"name,omitempty" json:"name,omitempty"`
+	Value    interface{}  `yaml:"value,omitempty" json:"value,omitempty"`
+	Settings struct {
+		Default interface{} `yaml:"default,omitempty" json:"default,omitempty"`
+		Unit    interface{} `yaml:"unit,omitempty" json:"unit,omitempty"`
+	} `yaml:"settings,omitempty" json:"settings,omitempty"`
 }
 
 // FormYAML represents a chart's values.yaml form abstraction
 type FormYAML struct {
-	Name        string   `yaml:"name" json:"name"`
-	Icon        string   `yaml:"icon" json:"icon"`
-	Description string   `yaml:"description" json:"description"`
-	Tags        []string `yaml:"tags" json:"tags"`
-	Tabs        []struct {
-		Name     string `yaml:"name" json:"name"`
-		Label    string `yaml:"label" json:"label"`
-		Sections []struct {
-			Name     string `yaml:"name" json:"name"`
-			ShowIf   string `yaml:"show_if" json:"show_if"`
-			Contents []struct {
-				Type     string `yaml:"type" json:"type"`
-				Label    string `yaml:"label" json:"label"`
-				Name     string `yaml:"name,omitempty" json:"name,omitempty"`
-				Variable string `yaml:"variable,omitempty" json:"variable,omitempty"`
-				Settings struct {
-					Default interface{} `yaml:"default,omitempty" json:"default,omitempty"`
-					Unit    interface{} `yaml:"unit,omitempty" json:"unit,omitempty"`
-				} `yaml:"settings,omitempty" json:"settings,omitempty"`
-			} `yaml:"contents" json:"contents,omitempty"`
-		} `yaml:"sections" json:"sections,omitempty"`
-	} `yaml:"tabs" json:"tabs,omitempty"`
+	Name        string     `yaml:"name" json:"name"`
+	Icon        string     `yaml:"icon" json:"icon"`
+	Description string     `yaml:"description" json:"description"`
+	Tags        []string   `yaml:"tags" json:"tags"`
+	Tabs        []*FormTab `yaml:"tabs" json:"tabs,omitempty"`
 }

internal/templater/dynamic/reader.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"time"
 
-	"github.com/porter-dev/porter/cli/cmd/templater/utils"
+	"github.com/porter-dev/porter/internal/templater/utils"
 
-	"github.com/porter-dev/porter/cli/cmd/templater"
+	"github.com/porter-dev/porter/internal/templater"
 	"k8s.io/client-go/dynamic"
 	di "k8s.io/client-go/dynamic/dynamicinformer"
 	"k8s.io/client-go/tools/cache"

internal/templater/dynamic/writer.go

@@ -3,9 +3,9 @@ package dynamic
 import (
 	"context"
 
-	"github.com/porter-dev/porter/cli/cmd/templater/utils"
+	"github.com/porter-dev/porter/internal/templater/utils"
 
-	"github.com/porter-dev/porter/cli/cmd/templater"
+	"github.com/porter-dev/porter/internal/templater"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 

internal/templater/helm/manifests_reader.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/porter-dev/porter/cli/cmd/templater"
-	"github.com/porter-dev/porter/cli/cmd/templater/utils"
+	"github.com/porter-dev/porter/internal/templater"
+	"github.com/porter-dev/porter/internal/templater/utils"
 	"helm.sh/helm/v3/pkg/release"
 	"sigs.k8s.io/yaml"
 )

internal/templater/helm/values_reader.go

@@ -3,8 +3,8 @@ package helm
 import (
 	"fmt"
 
-	"github.com/porter-dev/porter/cli/cmd/templater"
-	"github.com/porter-dev/porter/cli/cmd/templater/utils"
+	"github.com/porter-dev/porter/internal/templater"
+	"github.com/porter-dev/porter/internal/templater/utils"
 
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/release"

internal/templater/parser/parser.go

@@ -0,0 +1,199 @@
+package parser
+
+import (
+	"fmt"
+
+	"github.com/porter-dev/porter/internal/helm"
+	"github.com/porter-dev/porter/internal/models"
+	"github.com/porter-dev/porter/internal/templater"
+	"github.com/porter-dev/porter/internal/templater/utils"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/release"
+	"k8s.io/client-go/dynamic"
+	"sigs.k8s.io/yaml"
+
+	td "github.com/porter-dev/porter/internal/templater/dynamic"
+	th "github.com/porter-dev/porter/internal/templater/helm"
+)
+
+// TODO -- handle all continue statements, errors should at least be logged if not
+// thrown
+
+type ClientConfigDefault struct {
+	DynamicClient dynamic.Interface
+	DynamicObject *td.Object
+
+	HelmAgent   *helm.Agent
+	HelmRelease *release.Release
+	HelmChart   *chart.Chart
+
+	// for installing a new chart
+	HelmChartPath string
+}
+
+func FormYAMLFromBytes(def *ClientConfigDefault, bytes []byte) (*models.FormYAML, error) {
+	form, err := unqueriedFormYAMLFromBytes(bytes)
+
+	if err != nil {
+		return nil, err
+	}
+
+	lookup := formToLookupTable(def, form)
+
+	// merge data from lookup
+	data := make(map[string]interface{})
+
+	for _, lookupVal := range lookup {
+		queryRes, err := lookupVal.TemplateReader.Read()
+
+		if err != nil {
+			continue
+		}
+
+		for queryResKey, queryResVal := range queryRes {
+			data[queryResKey] = queryResVal
+		}
+	}
+
+	for i, tab := range form.Tabs {
+		for j, section := range tab.Sections {
+			for k, content := range section.Contents {
+				key := fmt.Sprintf("tabs[%d].sections[%d].contents[%d]", i, j, k)
+
+				if val, ok := data[key]; ok {
+					content.Value = val
+				}
+			}
+		}
+	}
+
+	return form, nil
+}
+
+// unqueriedFormYAMLFromBytes returns a FormYAML without values queries populated
+func unqueriedFormYAMLFromBytes(bytes []byte) (*models.FormYAML, error) {
+	// parse bytes into object
+	form := &models.FormYAML{}
+
+	err := yaml.Unmarshal(bytes, form)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// populate all context fields, with default set to helm/values with no config
+	parent := &models.FormContext{
+		Type: "helm/values",
+	}
+
+	for _, tab := range form.Tabs {
+		if tab.Context == nil {
+			tab.Context = parent
+		}
+
+		for _, section := range tab.Sections {
+			if section.Context == nil {
+				section.Context = tab.Context
+			}
+
+			for _, content := range section.Contents {
+				if content.Context == nil {
+					content.Context = section.Context
+				}
+			}
+		}
+	}
+
+	return form, nil
+}
+
+type ContextConfig struct {
+	FromType       string   // "live" or "declared"
+	Capabilities   []string // "read", "write"
+	TemplateReader templater.TemplateReader
+	TemplateWriter templater.TemplateWriter
+}
+
+// create map[*FormContext]*ContextConfig
+// assumes all contexts populated
+func formToLookupTable(def *ClientConfigDefault, form *models.FormYAML) map[*models.FormContext]*ContextConfig {
+	lookup := make(map[*models.FormContext]*ContextConfig)
+
+	for i, tab := range form.Tabs {
+		for j, section := range tab.Sections {
+			for k, content := range section.Contents {
+				if content.Context == nil {
+					continue
+				}
+
+				if _, ok := lookup[content.Context]; !ok {
+					lookup[content.Context] = formContextToContextConfig(def, content.Context)
+				}
+
+				// TODO -- case on whether value is proper query string, if not resolve it to a
+				// proper query string
+				query, err := utils.NewQuery(
+					fmt.Sprintf("tabs[%d].sections[%d].contents[%d]", i, j, k),
+					fmt.Sprintf("%v", content.Value),
+				)
+
+				if err != nil {
+					continue
+				}
+
+				lookup[content.Context].TemplateReader.RegisterQuery(query)
+			}
+		}
+	}
+
+	return lookup
+}
+
+// TODO -- this needs to be able to construct new context configs based on
+// configuration for each context, but right now just uses the default config
+// for everything
+func formContextToContextConfig(def *ClientConfigDefault, context *models.FormContext) *ContextConfig {
+	res := &ContextConfig{}
+
+	switch context.Type {
+	case "helm/values":
+		res.FromType = "declared"
+
+		res.Capabilities = []string{"read", "write"}
+
+		res.TemplateReader = &th.ValuesTemplateReader{
+			Release: def.HelmRelease,
+			Chart:   def.HelmChart,
+		}
+
+		relName := ""
+
+		if def.HelmRelease != nil {
+			relName = def.HelmRelease.Name
+		}
+
+		res.TemplateWriter = &th.ValuesTemplateWriter{
+			Agent:       def.HelmAgent,
+			ChartPath:   def.HelmChartPath,
+			ReleaseName: relName,
+		}
+	case "helm/manifests":
+		res.FromType = "live"
+
+		res.Capabilities = []string{"read"}
+
+		res.TemplateReader = &th.ManifestsTemplateReader{
+			Release: def.HelmRelease,
+		}
+	case "cluster":
+		res.FromType = "live"
+
+		res.Capabilities = []string{"read"}
+
+		res.TemplateReader = td.NewDynamicTemplateReader(def.DynamicClient, def.DynamicObject)
+	default:
+		return nil
+	}
+
+	return res
+}

internal/templater/utils/query.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/porter-dev/porter/cli/cmd/templater"
+	"github.com/porter-dev/porter/internal/templater"
 	"k8s.io/client-go/util/jsonpath"
 )
 

server/api/template_handler.go

@@ -1,65 +1,41 @@
 package api
 
 import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
 	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 
-	"github.com/porter-dev/porter/internal/models"
+	"github.com/go-chi/chi"
+	"github.com/porter-dev/porter/internal/forms"
+	"github.com/porter-dev/porter/internal/helm/loader"
+	"github.com/porter-dev/porter/internal/templater/parser"
+	"helm.sh/helm/v3/pkg/chart"
 
-	"gopkg.in/yaml.v2"
+	"github.com/porter-dev/porter/internal/models"
 )
 
 // HandleListTemplates retrieves a list of Porter templates
 // TODO: test and reduce fragility (handle untar/parse error for individual charts)
 // TODO: separate markdown retrieval into its own query if necessary
 func (app *App) HandleListTemplates(w http.ResponseWriter, r *http.Request) {
-	baseURL := "https://porter-dev.github.io/chart-repo/"
-	resp, err := http.Get(baseURL + "index.yaml")
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	defer resp.Body.Close()
-	body, _ := ioutil.ReadAll(resp.Body)
+	repoIndex, err := loader.LoadRepoIndex("https://porter-dev.github.io/chart-repo/index.yaml")
 
-	form := models.IndexYAML{}
-	if err := yaml.Unmarshal([]byte(body), &form); err != nil {
-		fmt.Println(err)
+	if err != nil {
+		app.handleErrorFormDecoding(err, ErrReleaseDecode, w)
 		return
 	}
 
 	// Loop over charts in index.yaml
 	porterCharts := []models.PorterChart{}
-	for k := range form.Entries {
-		indexChart := form.Entries[k][0]
-		tarURL := indexChart.Urls[0]
-		if !strings.Contains(tarURL, "http://") {
-			tarURL = baseURL + tarURL
-		}
 
-		formData, markdown, err := processTarball(tarURL)
-		if err != nil {
-			fmt.Println(err)
-			return
-		}
+	for _, entry := range repoIndex.Entries {
+		indexChart := entry[0]
 
 		porterChart := models.PorterChart{}
 		porterChart.Name = indexChart.Name
 		porterChart.Description = indexChart.Description
 		porterChart.Icon = indexChart.Icon
-		porterChart.Form = *formData
-		if markdown != "" {
-			porterChart.Markdown = markdown
-		}
 
 		porterCharts = append(porterCharts, porterChart)
 	}
@@ -67,80 +43,60 @@ func (app *App) HandleListTemplates(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(porterCharts)
 }
 
-func processTarball(tarURL string) (*models.FormYAML, string, error) {
-	resp, err := http.Get(tarURL)
-	if err != nil {
-		fmt.Println(err)
-		return nil, "", err
+// ChartWithForm is a base helm chart with the form yaml appended
+type ChartWithForm struct {
+	*chart.Chart
+	Form *models.FormYAML `json:"form"`
+}
+
+// HandleReadTemplate reads a given template with name and version field
+func (app *App) HandleReadTemplate(w http.ResponseWriter, r *http.Request) {
+	name := chi.URLParam(r, "name")
+	version := chi.URLParam(r, "version")
+
+	// if version passed as latest, pass empty string to loader to get latest
+	if version == "latest" {
+		version = ""
+	}
+
+	form := &forms.ChartForm{
+		Name:    name,
+		Version: version,
+		RepoURL: "https://porter-dev.github.io/chart-repo/",
 	}
 
-	defer resp.Body.Close()
-	body, _ := ioutil.ReadAll(resp.Body)
-	buf := bytes.NewBuffer(body)
+	// if a repo_url is passed as query param, it will be populated
+	vals, err := url.ParseQuery(r.URL.RawQuery)
 
-	gzf, err := gzip.NewReader(buf)
 	if err != nil {
-		fmt.Println(err)
-		return nil, "", err
+		app.handleErrorFormDecoding(err, ErrReleaseDecode, w)
+		return
 	}
 
-	// Process tarball to generate FormYAML and retrieve markdown
-	tarReader := tar.NewReader(gzf)
-	markdown := ""
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			fmt.Println(err)
-			return nil, "", err
-		}
+	form.PopulateRepoURLFromQueryParams(vals)
+
+	chart, err := loader.LoadChart(form.RepoURL, form.Name, form.Version)
 
-		name := header.Name
-		switch header.Typeflag {
-		case tar.TypeDir:
-			continue
-		case tar.TypeReg:
+	if err != nil {
+		app.handleErrorFormDecoding(err, ErrReleaseDecode, w)
+		return
+	}
 
-			// Handle info.md if found
-			if strings.Contains(name, "README.md") {
-				bufMd := new(bytes.Buffer)
+	parserDef := &parser.ClientConfigDefault{
+		HelmChart: chart,
+	}
 
-				_, err := io.Copy(bufMd, tarReader)
-				if err != nil {
-					fmt.Println(err)
-					return nil, "", err
-				}
+	res := &ChartWithForm{chart, nil}
 
-				markdown = string(bufMd.Bytes())
-			}
+	for _, file := range chart.Files {
+		if strings.Contains(file.Name, "form.yaml") {
+			formYAML, err := parser.FormYAMLFromBytes(parserDef, file.Data)
 
-			// Handle form.yaml located in archive
-			if strings.Contains(name, "form.yaml") {
-				bufForm := new(bytes.Buffer)
-
-				_, err := io.Copy(bufForm, tarReader)
-				if err != nil {
-					fmt.Println(err)
-					return nil, "", err
-				}
-
-				// Unmarshal yaml byte buffer
-				form := models.FormYAML{}
-				if err := yaml.Unmarshal(bufForm.Bytes(), &form); err != nil {
-					fmt.Println(err)
-					return nil, "", err
-				}
-				return &form, markdown, nil
+			if err != nil {
+				break
 			}
-		default:
-			fmt.Printf("%s : %c %s %s\n",
-				"Unknown type",
-				header.Typeflag,
-				"in file",
-				name,
-			)
+
+			res.Form = formYAML
 		}
 	}
-	return nil, "", errors.New("no form.yaml found")
 }

server/router/router.go

@@ -121,6 +121,14 @@ func New(
 			),
 		)
 
+		r.Method(
+			"GET",
+			"/templates/{name}/{version}",
+			auth.BasicAuthenticate(
+				requestlog.NewHandler(a.HandleReadTemplate, l),
+			),
+		)
+
 		// /api/oauth routes
 		// r.Method(
 		// 	"GET",