run apply on multiple stacks in porter yaml (#3271)

* checkpoint

* run apply on multiple stacks in porter yaml

* init naming fixup

api/server/handlers/gitinstallation/get_porter_yaml.go

@@ -8,12 +8,12 @@ import (
 	"github.com/google/go-github/v41/github"
 	"github.com/porter-dev/porter/api/server/authz"
 	"github.com/porter-dev/porter/api/server/handlers"
-	"github.com/porter-dev/porter/api/server/handlers/porter_app"
 	"github.com/porter-dev/porter/api/server/shared"
 	"github.com/porter-dev/porter/api/server/shared/apierrors"
 	"github.com/porter-dev/porter/api/server/shared/commonutils"
 	"github.com/porter-dev/porter/api/server/shared/config"
 	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/cli/cmd/stack"
 	"github.com/porter-dev/porter/internal/telemetry"
 	"gopkg.in/yaml.v2"
 )
@@ -89,7 +89,7 @@ func (c *GithubGetPorterYamlHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	parsed := &porter_app.PorterStackYAML{}
+	parsed := &stack.PorterStackYAML{}
 	err = yaml.Unmarshal([]byte(fileData), parsed)
 	if err != nil {
 		err = telemetry.Error(ctx, span, err, "invalid porter yaml format")

api/server/handlers/porter_app/parse.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/porter-dev/porter/api/server/shared/config"
 	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/cli/cmd/stack"
 	"github.com/porter-dev/porter/internal/helm/loader"
 	"github.com/porter-dev/porter/internal/integrations/powerdns"
 	"github.com/porter-dev/porter/internal/kubernetes"
@@ -18,30 +19,6 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-type PorterStackYAML struct {
-	Version   *string             `yaml:"version"`
-	Build     *Build              `yaml:"build"`
-	Env       map[string]string   `yaml:"env"`
-	SyncedEnv []*SyncedEnvSection `yaml:"synced_env"`
-	Apps      map[string]*App     `yaml:"apps"`
-	Release   *App                `yaml:"release"`
-}
-
-type Build struct {
-	Context    *string   `yaml:"context" validate:"dir"`
-	Method     *string   `yaml:"method" validate:"required,oneof=pack docker registry"`
-	Builder    *string   `yaml:"builder" validate:"required_if=Method pack"`
-	Buildpacks []*string `yaml:"buildpacks"`
-	Dockerfile *string   `yaml:"dockerfile" validate:"required_if=Method docker"`
-	Image      *string   `yaml:"image" validate:"required_if=Method registry"`
-}
-
-type App struct {
-	Run    *string                `yaml:"run" validate:"required"`
-	Config map[string]interface{} `yaml:"config"`
-	Type   *string                `yaml:"type" validate:"oneof=web worker job"`
-}
-
 type SubdomainCreateOpts struct {
 	k8sAgent       *kubernetes.Agent
 	dnsRepo        repository.DNSRecordRepository
@@ -50,17 +27,6 @@ type SubdomainCreateOpts struct {
 	stackName      string
 }
 
-type SyncedEnvSection struct {
-	Name    string                `json:"name" yaml:"name"`
-	Version uint                  `json:"version" yaml:"version"`
-	Keys    []SyncedEnvSectionKey `json:"keys" yaml:"keys"`
-}
-
-type SyncedEnvSectionKey struct {
-	Name   string `json:"name" yaml:"name"`
-	Secret bool   `json:"secret" yaml:"secret"`
-}
-
 type ParseConf struct {
 	// PorterYaml is the raw porter yaml which is used to build the values + chart for helm upgrade
 	PorterYaml []byte
@@ -92,7 +58,7 @@ type ParseConf struct {
 }
 
 func parse(conf ParseConf) (*chart.Chart, map[string]interface{}, map[string]interface{}, error) {
-	parsed := &PorterStackYAML{}
+	parsed := &stack.PorterStackYAML{}
 
 	if conf.FullHelmValues != "" {
 		parsedHelmValues, err := convertHelmValuesToPorterYaml(conf.FullHelmValues)
@@ -107,7 +73,7 @@ func parse(conf ParseConf) (*chart.Chart, map[string]interface{}, map[string]int
 		}
 	}
 
-	synced_env := make([]*SyncedEnvSection, 0)
+	synced_env := make([]*stack.SyncedEnvSection, 0)
 
 	for i := range conf.EnvGroups {
 		cm, _, err := conf.SubdomainCreateOpts.k8sAgent.GetLatestVersionedConfigMap(conf.EnvGroups[i], conf.Namespace)
@@ -126,15 +92,15 @@ func parse(conf ParseConf) (*chart.Chart, map[string]interface{}, map[string]int
 
 		version := uint(versionInt)
 
-		newSection := &SyncedEnvSection{
+		newSection := &stack.SyncedEnvSection{
 			Name:    conf.EnvGroups[i],
 			Version: version,
 		}
 
-		newSectionKeys := make([]SyncedEnvSectionKey, 0)
+		newSectionKeys := make([]stack.SyncedEnvSectionKey, 0)
 
 		for key, val := range cm.Data {
-			newSectionKeys = append(newSectionKeys, SyncedEnvSectionKey{
+			newSectionKeys = append(newSectionKeys, stack.SyncedEnvSectionKey{
 				Name:   key,
 				Secret: strings.Contains(val, "PORTERSECRET"),
 			})
@@ -144,30 +110,34 @@ func parse(conf ParseConf) (*chart.Chart, map[string]interface{}, map[string]int
 		synced_env = append(synced_env, newSection)
 	}
 
-	parsed.SyncedEnv = synced_env
+	application, err := stack.CreateAppFromFile(parsed)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("%s: %w", "error parsing porter.yaml", err)
+	}
 
-	values, err := buildUmbrellaChartValues(parsed, conf.ImageInfo, conf.ExistingHelmValues, conf.SubdomainCreateOpts, conf.InjectLauncherToStartCommand, conf.ShouldValidateHelmValues, conf.UserUpdate)
+	values, err := buildUmbrellaChartValues(application, synced_env, conf.ImageInfo, conf.ExistingHelmValues, conf.SubdomainCreateOpts, conf.InjectLauncherToStartCommand, conf.ShouldValidateHelmValues, conf.UserUpdate)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("%s: %w", "error building values", err)
 	}
 	convertedValues := convertMap(values).(map[string]interface{})
 
-	chart, err := buildUmbrellaChart(parsed, conf.ServerConfig, conf.ProjectID, conf.ExistingChartDependencies)
+	chart, err := buildUmbrellaChart(application, conf.ServerConfig, conf.ProjectID, conf.ExistingChartDependencies)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("%s: %w", "error building chart", err)
 	}
 
 	// return the parsed release values for the release job chart, if they exist
 	var preDeployJobValues map[string]interface{}
-	if parsed.Release != nil && parsed.Release.Run != nil {
-		preDeployJobValues = buildPreDeployJobChartValues(parsed.Release, parsed.Env, parsed.SyncedEnv, conf.ImageInfo, conf.InjectLauncherToStartCommand, conf.ExistingHelmValues, strings.TrimSuffix(strings.TrimPrefix(conf.Namespace, "porter-stack-"), "")+"-r", conf.UserUpdate)
+	if application.Release != nil && application.Release.Run != nil {
+		preDeployJobValues = buildPreDeployJobChartValues(application.Release, application.Env, synced_env, conf.ImageInfo, conf.InjectLauncherToStartCommand, conf.ExistingHelmValues, strings.TrimSuffix(strings.TrimPrefix(conf.Namespace, "porter-stack-"), "")+"-r", conf.UserUpdate)
 	}
 
 	return chart, convertedValues, preDeployJobValues, nil
 }
 
 func buildUmbrellaChartValues(
-	parsed *PorterStackYAML,
+	application *stack.Application,
+	syncedEnv []*stack.SyncedEnvSection,
 	imageInfo types.ImageInfo,
 	existingValues map[string]interface{},
 	opts SubdomainCreateOpts,
@@ -177,21 +147,21 @@ func buildUmbrellaChartValues(
 ) (map[string]interface{}, error) {
 	values := make(map[string]interface{})
 
-	if parsed.Apps == nil {
+	if application.Services == nil {
 		if existingValues == nil {
-			return nil, fmt.Errorf("porter.yaml must contain at least one app, or pre-deploy must exist and have values")
+			return nil, fmt.Errorf("porter.yaml must contain at least one service, or pre-deploy must exist and have values")
 		}
 	}
 
-	for name, app := range parsed.Apps {
-		appType := getType(name, app)
+	for name, service := range application.Services {
+		serviceType := getType(name, service)
 
-		defaultValues := getDefaultValues(app, parsed.Env, parsed.SyncedEnv, appType, existingValues, name, userUpdate)
-		convertedConfig := convertMap(app.Config).(map[string]interface{})
+		defaultValues := getDefaultValues(service, application.Env, syncedEnv, serviceType, existingValues, name, userUpdate)
+		convertedConfig := convertMap(service.Config).(map[string]interface{})
 		helm_values := utils.DeepCoalesceValues(defaultValues, convertedConfig)
 
 		// required to identify the chart type because of https://github.com/helm/helm/issues/9214
-		helmName := getHelmName(name, appType)
+		helmName := getHelmName(name, serviceType)
 		if existingValues != nil {
 			if existingValues[helmName] != nil {
 				existingValuesMap := existingValues[helmName].(map[string]interface{})
@@ -199,7 +169,7 @@ func buildUmbrellaChartValues(
 			}
 		}
 
-		validateErr := validateHelmValues(helm_values, shouldValidateHelmValues, appType)
+		validateErr := validateHelmValues(helm_values, shouldValidateHelmValues, serviceType)
 		if validateErr != "" {
 			return nil, fmt.Errorf("error validating service \"%s\": %s", name, validateErr)
 		}
@@ -210,7 +180,7 @@ func buildUmbrellaChartValues(
 		}
 
 		// just in case this slips by
-		if appType == "web" {
+		if serviceType == "web" {
 			if helm_values["ingress"] == nil {
 				helm_values["ingress"] = map[string]interface{}{
 					"enabled": false,
@@ -284,7 +254,7 @@ func validateHelmValues(values map[string]interface{}, shouldValidateHelmValues
 	return ""
 }
 
-func buildPreDeployJobChartValues(release *App, env map[string]string, synced_env []*SyncedEnvSection, imageInfo types.ImageInfo, injectLauncher bool, existingValues map[string]interface{}, name string, userUpdate bool) map[string]interface{} {
+func buildPreDeployJobChartValues(release *stack.Service, env map[string]string, synced_env []*stack.SyncedEnvSection, imageInfo types.ImageInfo, injectLauncher bool, existingValues map[string]interface{}, name string, userUpdate bool) map[string]interface{} {
 	defaultValues := getDefaultValues(release, env, synced_env, "job", existingValues, name+"-r", userUpdate)
 	convertedConfig := convertMap(release.Config).(map[string]interface{})
 	helm_values := utils.DeepCoalesceValues(defaultValues, convertedConfig)
@@ -310,21 +280,26 @@ func buildPreDeployJobChartValues(release *App, env map[string]string, synced_en
 	return helm_values
 }
 
-func getType(name string, app *App) string {
-	if app.Type != nil {
-		return *app.Type
+func getType(name string, service *stack.Service) string {
+	if service.Type != nil {
+		return *service.Type
 	}
 	if strings.Contains(name, "web") {
 		return "web"
 	}
+
+	if strings.Contains(name, "job") {
+		return "job"
+	}
+
 	return "worker"
 }
 
-func getDefaultValues(app *App, env map[string]string, synced_env []*SyncedEnvSection, appType string, existingValues map[string]interface{}, name string, userUpdate bool) map[string]interface{} {
+func getDefaultValues(service *stack.Service, env map[string]string, synced_env []*stack.SyncedEnvSection, appType string, existingValues map[string]interface{}, name string, userUpdate bool) map[string]interface{} {
 	var defaultValues map[string]interface{}
 	var runCommand string
-	if app.Run != nil {
-		runCommand = *app.Run
+	if service.Run != nil {
+		runCommand = *service.Run
 	}
 	var syncedEnvs []map[string]interface{}
 	envConf, err := getStacksNestedMap(existingValues, name+"-"+appType, "container", "env")
@@ -347,7 +322,7 @@ func getDefaultValues(app *App, env map[string]string, synced_env []*SyncedEnvSe
 	return defaultValues
 }
 
-func deconstructSyncedEnvs(synced_env []*SyncedEnvSection, env map[string]string) []map[string]interface{} {
+func deconstructSyncedEnvs(synced_env []*stack.SyncedEnvSection, env map[string]string) []map[string]interface{} {
 	synced := make([]map[string]interface{}, 0)
 	for _, group := range synced_env {
 		keys := make([]map[string]interface{}, 0)
@@ -373,35 +348,35 @@ func deconstructSyncedEnvs(synced_env []*SyncedEnvSection, env map[string]string
 	return synced
 }
 
-func buildUmbrellaChart(parsed *PorterStackYAML, config *config.Config, projectID uint, existingDependencies []*chart.Dependency) (*chart.Chart, error) {
+func buildUmbrellaChart(application *stack.Application, config *config.Config, projectID uint, existingDependencies []*chart.Dependency) (*chart.Chart, error) {
 	deps := make([]*chart.Dependency, 0)
-	for alias, app := range parsed.Apps {
-		var appType string
+	for alias, service := range application.Services {
+		var serviceType string
 		if existingDependencies != nil {
 			for _, dep := range existingDependencies {
 				// this condition checks that the dependency is of the form <alias>-web or <alias>-wkr or <alias>-job, meaning it already exists in the chart
 				if strings.HasPrefix(dep.Alias, fmt.Sprintf("%s-", alias)) && (strings.HasSuffix(dep.Alias, "-web") || strings.HasSuffix(dep.Alias, "-wkr") || strings.HasSuffix(dep.Alias, "-job")) {
-					appType = getChartTypeFromHelmName(dep.Alias)
-					if appType == "" {
+					serviceType = getChartTypeFromHelmName(dep.Alias)
+					if serviceType == "" {
 						return nil, fmt.Errorf("unable to determine type of existing dependency")
 					}
 				}
 			}
 			// this is a new app, so we need to get the type from the app name or type
-			if appType == "" {
-				appType = getType(alias, app)
+			if serviceType == "" {
+				serviceType = getType(alias, service)
 			}
 		} else {
-			appType = getType(alias, app)
+			serviceType = getType(alias, service)
 		}
 		selectedRepo := config.ServerConf.DefaultApplicationHelmRepoURL
-		selectedVersion, err := getLatestTemplateVersion(appType, config, projectID)
+		selectedVersion, err := getLatestTemplateVersion(serviceType, config, projectID)
 		if err != nil {
 			return nil, err
 		}
-		helmName := getHelmName(alias, appType)
+		helmName := getHelmName(alias, serviceType)
 		deps = append(deps, &chart.Dependency{
-			Name:       appType,
+			Name:       serviceType,
 			Alias:      helmName,
 			Version:    selectedVersion,
 			Repository: selectedRepo,
@@ -740,13 +715,13 @@ func getStacksNestedMap(obj map[string]interface{}, fields ...string) ([]map[str
 	return result, nil
 }
 
-func convertHelmValuesToPorterYaml(helmValues string) (*PorterStackYAML, error) {
+func convertHelmValuesToPorterYaml(helmValues string) (*stack.PorterStackYAML, error) {
 	var values map[string]interface{}
 	err := yaml.Unmarshal([]byte(helmValues), &values)
 	if err != nil {
 		return nil, err
 	}
-	apps := make(map[string]*App)
+	services := make(map[string]*stack.Service)
 	for k, v := range values {
 		if k == "global" {
 			continue
@@ -755,12 +730,12 @@ func convertHelmValuesToPorterYaml(helmValues string) (*PorterStackYAML, error)
 		if serviceName == "" {
 			return nil, fmt.Errorf("invalid service key: %s. make sure that service key ends in either -web, -wkr, or -job", k)
 		}
-		apps[serviceName] = &App{
+		services[serviceName] = &stack.Service{
 			Config: convertMap(v).(map[string]interface{}),
 			Type:   &serviceType,
 		}
 	}
-	return &PorterStackYAML{
-		Apps: apps,
+	return &stack.PorterStackYAML{
+		Services: services,
 	}, nil
 }

cli/cmd/apply.go

@@ -157,70 +157,48 @@ func apply(_ *types.GetAuthenticatedUserResponse, client *api.Client, _ []string
 			return fmt.Errorf("error parsing porter.yaml: %w", err)
 		}
 	} else if previewVersion.Version == "v1stack" || previewVersion.Version == "" {
-		stackName := os.Getenv("PORTER_STACK_NAME")
-		if stackName == "" {
-			return fmt.Errorf("environment variable PORTER_STACK_NAME must be set")
-		}
 
-		// we need to know the builder so that we can inject launcher to the start command later if heroku builder is used
-		var builder string
-		resGroup, builder, err = stack.CreateV1BuildResources(client, fileBytes, stackName, cliConf.Project, cliConf.Cluster)
+		parsed, err := stack.ValidateAndMarshal(fileBytes)
 		if err != nil {
-			return fmt.Errorf("error parsing porter.yaml for build resources: %w", err)
+			return fmt.Errorf("error parsing porter.yaml: %w", err)
 		}
 
-		deployStackHook := &stack.DeployStackHook{
-			Client:               client,
-			StackName:            stackName,
-			ProjectID:            cliConf.Project,
-			ClusterID:            cliConf.Cluster,
-			BuildImageDriverName: stack.GetBuildImageDriverName(),
-			PorterYAML:           fileBytes,
-			Builder:              builder,
-		}
-		worker.RegisterHook("deploy-stack", deployStackHook)
-
-		if os.Getenv("GITHUB_RUN_ID") != "" {
-			// Create app event to signfy start of build
-			req := &types.CreateOrUpdatePorterAppEventRequest{
-				Status:             "PROGRESSING",
-				Type:               types.PorterAppEventType_Build,
-				TypeExternalSource: "GITHUB",
-				Metadata: map[string]any{
-					"action_run_id": os.Getenv("GITHUB_RUN_ID"),
-					"org":           os.Getenv("GITHUB_REPOSITORY_OWNER"),
+		resGroup = &switchboardTypes.ResourceGroup{
+			Version: "v1",
+			Resources: []*switchboardTypes.Resource{
+				{
+					Name:   "get-env",
+					Driver: "os-env",
 				},
-			}
-
-			repoNameSplit := strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")
-			if len(repoNameSplit) != 2 {
-				return fmt.Errorf("unable to parse GITHUB_REPOSITORY")
-			}
-			req.Metadata["repo"] = repoNameSplit[1]
+			},
+		}
 
-			actionRunID := os.Getenv("GITHUB_RUN_ID")
-			if actionRunID != "" {
-				arid, err := strconv.Atoi(actionRunID)
+		if parsed.Applications != nil {
+			for appName, app := range parsed.Applications {
+				resources, err := stack.CreateApplicationDeploy(client, worker, app, appName, cliConf)
 				if err != nil {
-					return fmt.Errorf("unable to parse GITHUB_RUN_ID as int: %w", err)
+					return fmt.Errorf("error parsing porter.yaml for build resources: %w", err)
 				}
-				req.Metadata["action_run_id"] = arid
+
+				resGroup.Resources = append(resGroup.Resources, resources...)
+			}
+		} else {
+			appName := os.Getenv("PORTER_STACK_NAME")
+			if appName == "" {
+				return fmt.Errorf("environment variable PORTER_STACK_NAME must be set")
 			}
+			app, err := stack.CreateAppFromFile(parsed)
 
-			repoOwnerAccountID := os.Getenv("GITHUB_REPOSITORY_OWNER_ID")
-			if repoOwnerAccountID != "" {
-				arid, err := strconv.Atoi(repoOwnerAccountID)
-				if err != nil {
-					return fmt.Errorf("unable to parse GITHUB_REPOSITORY_OWNER_ID as int: %w", err)
-				}
-				req.Metadata["github_account_id"] = arid
+			if err != nil {
+				return fmt.Errorf("error parsing porter.yaml for build resources: %w", err)
 			}
 
-			ctx := context.Background()
-			_, err := client.CreateOrUpdatePorterAppEvent(ctx, cliConf.Project, cliConf.Cluster, stackName, req)
+			resources, err := stack.CreateApplicationDeploy(client, worker, app, appName, cliConf)
 			if err != nil {
-				return fmt.Errorf("unable to create porter app build event: %w", err)
+				return fmt.Errorf("error parsing porter.yaml for build resources: %w", err)
 			}
+
+			resGroup.Resources = append(resGroup.Resources, resources...)
 		}
 	} else {
 		return fmt.Errorf("unknown porter.yaml version: %s", previewVersion.Version)

cli/cmd/stack/apply.go

@@ -3,6 +3,8 @@ package stack
 import (
 	"context"
 	"fmt"
+	"os"
+	"strconv"
 	"strings"
 
 	"github.com/fatih/color"
@@ -10,91 +12,159 @@ import (
 	"github.com/porter-dev/porter/api/types"
 	"github.com/porter-dev/porter/cli/cmd/config"
 	switchboardTypes "github.com/porter-dev/switchboard/pkg/types"
-	"gopkg.in/yaml.v2"
+	switchboardWorker "github.com/porter-dev/switchboard/pkg/worker"
+	"gopkg.in/yaml.v3"
 )
 
 type StackConf struct {
 	apiClient            *api.Client
-	rawBytes             []byte
-	parsed               *PorterStackYAML
+	parsed               *Application
 	stackName, namespace string
 	projectID, clusterID uint
 }
 
-func CreateV1BuildResources(client *api.Client, raw []byte, stackName string, projectID uint, clusterID uint) (*switchboardTypes.ResourceGroup, string, error) {
-	v1File := &switchboardTypes.ResourceGroup{
-		Version: "v1",
-		Resources: []*switchboardTypes.Resource{
-			{
-				Name:   "get-env",
-				Driver: "os-env",
-			},
-		},
-	}
+func CreateApplicationDeploy(client *api.Client, worker *switchboardWorker.Worker, app *Application, applicationName string, cliConf *config.CLIConfig) ([]*switchboardTypes.Resource, error) {
+	// we need to know the builder so that we can inject launcher to the start command later if heroku builder is used
 	var builder string
+	resources, builder, err := createV1BuildResources(client, app, applicationName, cliConf.Project, cliConf.Cluster)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing porter.yaml for build resources: %w", err)
+	}
 
-	stackConf, err := createStackConf(client, raw, stackName, projectID, clusterID)
+	applicationBytes, err := yaml.Marshal(app)
 	if err != nil {
-		return nil, "", err
+		return nil, fmt.Errorf("malformed application definition: %w", err)
 	}
 
-	var bi, pi *switchboardTypes.Resource
+	deployStackHook := &DeployAppHook{
+		Client:               client,
+		ApplicationName:      applicationName,
+		ProjectID:            cliConf.Project,
+		ClusterID:            cliConf.Cluster,
+		BuildImageDriverName: GetBuildImageDriverName(applicationName),
+		PorterYAML:           applicationBytes,
+		Builder:              builder,
+	}
 
-	if stackConf.parsed.Build != nil {
-		bi, pi, builder, err = createV1BuildResourcesFromPorterYaml(stackConf)
+	worker.RegisterHook("deploy-stack", deployStackHook)
+	if os.Getenv("GITHUB_RUN_ID") != "" {
+		err := createAppEvent(client, applicationName, cliConf)
 		if err != nil {
-			color.New(color.FgRed).Printf("Could not build using values specified in porter.yaml (%s), attempting to load stack build settings instead \n", err.Error())
-			bi, pi, builder, err = createV1BuildResourcesFromDB(client, stackConf)
-			if err != nil {
-				return nil, "", err
-			}
+			return nil, err
 		}
-	} else {
-		color.New(color.FgYellow).Printf("No build values specified in porter.yaml, attempting to load stack build settings instead \n")
-		bi, pi, builder, err = createV1BuildResourcesFromDB(client, stackConf)
+	}
+
+	return resources, nil
+}
+
+// Create app event to signfy start of build
+func createAppEvent(client *api.Client, applicationName string, cliConf *config.CLIConfig) error {
+	req := &types.CreateOrUpdatePorterAppEventRequest{
+		Status:             "PROGRESSING",
+		Type:               types.PorterAppEventType_Build,
+		TypeExternalSource: "GITHUB",
+		Metadata: map[string]any{
+			"action_run_id": os.Getenv("GITHUB_RUN_ID"),
+			"org":           os.Getenv("GITHUB_REPOSITORY_OWNER"),
+		},
+	}
+
+	repoNameSplit := strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")
+	if len(repoNameSplit) != 2 {
+		return fmt.Errorf("unable to parse GITHUB_REPOSITORY")
+	}
+	req.Metadata["repo"] = repoNameSplit[1]
+
+	actionRunID := os.Getenv("GITHUB_RUN_ID")
+	if actionRunID != "" {
+		arid, err := strconv.Atoi(actionRunID)
 		if err != nil {
-			return nil, "", err
+			return fmt.Errorf("unable to parse GITHUB_RUN_ID as int: %w", err)
 		}
+		req.Metadata["action_run_id"] = arid
 	}
 
-	v1File.Resources = append(v1File.Resources, bi, pi)
+	repoOwnerAccountID := os.Getenv("GITHUB_REPOSITORY_OWNER_ID")
+	if repoOwnerAccountID != "" {
+		arid, err := strconv.Atoi(repoOwnerAccountID)
+		if err != nil {
+			return fmt.Errorf("unable to parse GITHUB_REPOSITORY_OWNER_ID as int: %w", err)
+		}
+		req.Metadata["github_account_id"] = arid
+	}
 
-	preDeploy, cmd, err := createPreDeployResource(client,
-		stackConf.parsed.Release,
-		stackConf.stackName,
-		bi.Name,
-		pi.Name,
-		stackConf.projectID,
-		stackConf.clusterID,
-		stackConf.parsed.Env,
-	)
+	ctx := context.Background()
+	_, err := client.CreateOrUpdatePorterAppEvent(ctx, cliConf.Project, cliConf.Cluster, applicationName, req)
+	if err != nil {
+		return fmt.Errorf("unable to create porter app build event: %w", err)
+	}
+
+	return nil
+}
+
+func createV1BuildResources(client *api.Client, app *Application, stackName string, projectID uint, clusterID uint) ([]*switchboardTypes.Resource, string, error) {
+	var builder string
+	resources := make([]*switchboardTypes.Resource, 0)
+
+	stackConf, err := createStackConf(client, app, stackName, projectID, clusterID)
 	if err != nil {
 		return nil, "", err
 	}
 
-	if preDeploy != nil {
-		color.New(color.FgYellow).Printf("Found pre-deploy command to run before deploying apps: %s \n", cmd)
-		v1File.Resources = append(v1File.Resources, preDeploy)
-	} else {
-		color.New(color.FgYellow).Printf("No pre-deploy command found in porter.yaml or helm. \n")
+	var bi, pi *switchboardTypes.Resource
+
+	// look up build settings from DB if none specified in porter.yaml
+	if stackConf.parsed.Build == nil {
+		color.New(color.FgYellow).Printf("No build values specified in porter.yaml, attempting to load stack build settings instead \n")
+
+		res, err := client.GetPorterApp(context.Background(), stackConf.projectID, stackConf.clusterID, stackConf.stackName)
+
+		if err != nil {
+			return nil, "", fmt.Errorf("unable to read build info from DB: %w", err)
+		}
+
+		converted := convertToBuild(res)
+		stackConf.parsed.Build = &converted
 	}
 
-	return v1File, builder, nil
-}
+	// only include build and push steps if an image is not already specified
+	if stackConf.parsed.Build.Image == nil {
+		bi, pi, builder, err = createV1BuildResourcesFromPorterYaml(stackConf)
 
-func createStackConf(client *api.Client, raw []byte, stackName string, projectID uint, clusterID uint) (*StackConf, error) {
-	var parsed *PorterStackYAML
-	if raw == nil {
-		parsed = createDefaultPorterYaml()
-	} else {
-		parsed = &PorterStackYAML{}
-		err := yaml.Unmarshal(raw, parsed)
 		if err != nil {
-			errMsg := composePreviewMessage("error parsing porter.yaml", Error)
-			return nil, fmt.Errorf("%s: %w", errMsg, err)
+			return nil, "", err
+		}
+
+		resources = append(resources, bi, pi)
+
+		// also excluding use of pre-deploy with pre-built imges
+		preDeploy, cmd, err := createPreDeployResource(client,
+			stackConf.parsed.Release,
+			stackConf.stackName,
+			bi.Name,
+			pi.Name,
+			stackConf.projectID,
+			stackConf.clusterID,
+			stackConf.parsed.Env,
+		)
+
+		if err != nil {
+			return nil, "", err
+		}
+
+		if preDeploy != nil {
+			color.New(color.FgYellow).Printf("Found pre-deploy command to run before deploying apps: %s \n", cmd)
+			resources = append(resources, preDeploy)
+		} else {
+			color.New(color.FgYellow).Printf("No pre-deploy command found in porter.yaml or helm. \n")
 		}
 	}
 
+	return resources, builder, nil
+}
+
+func createStackConf(client *api.Client, app *Application, stackName string, projectID uint, clusterID uint) (*StackConf, error) {
+
 	err := config.ValidateCLIEnvironment()
 	if err != nil {
 		errMsg := composePreviewMessage("porter CLI is not configured correctly", Error)
@@ -104,13 +174,12 @@ func createStackConf(client *api.Client, raw []byte, stackName string, projectID
 	releaseEnvVars := getEnvFromRelease(client, stackName, projectID, clusterID)
 	if releaseEnvVars != nil {
 		color.New(color.FgYellow).Printf("Reading build env from release\n")
-		parsed.Env = mergeStringMaps(parsed.Env, releaseEnvVars)
+		app.Env = mergeStringMaps(app.Env, releaseEnvVars)
 	}
 
 	return &StackConf{
 		apiClient: client,
-		rawBytes:  raw,
-		parsed:    parsed,
+		parsed:    app,
 		stackName: stackName,
 		projectID: projectID,
 		clusterID: clusterID,
@@ -119,12 +188,12 @@ func createStackConf(client *api.Client, raw []byte, stackName string, projectID
 }
 
 func createV1BuildResourcesFromPorterYaml(stackConf *StackConf) (*switchboardTypes.Resource, *switchboardTypes.Resource, string, error) {
-	bi, err := stackConf.parsed.Build.getV1BuildImage(stackConf.parsed.Env, stackConf.namespace)
+	bi, err := stackConf.parsed.Build.getV1BuildImage(stackConf.stackName, stackConf.parsed.Env, stackConf.namespace)
 	if err != nil {
 		return nil, nil, "", err
 	}
 
-	pi, err := stackConf.parsed.Build.getV1PushImage(stackConf.namespace)
+	pi, err := stackConf.parsed.Build.getV1PushImage(stackConf.stackName, stackConf.namespace)
 	if err != nil {
 		return nil, nil, "", err
 	}
@@ -132,31 +201,6 @@ func createV1BuildResourcesFromPorterYaml(stackConf *StackConf) (*switchboardTyp
 	return bi, pi, stackConf.parsed.Build.GetBuilder(), nil
 }
 
-func createV1BuildResourcesFromDB(client *api.Client, stackConf *StackConf) (*switchboardTypes.Resource, *switchboardTypes.Resource, string, error) {
-	res, err := client.GetPorterApp(context.Background(), stackConf.projectID, stackConf.clusterID, stackConf.stackName)
-	if err != nil {
-		return nil, nil, "", fmt.Errorf("unable to read build info from DB: %w", err)
-	}
-
-	if res == nil {
-		return nil, nil, "", fmt.Errorf("stack %s not found", stackConf.stackName)
-	}
-
-	build := convertToBuild(res)
-
-	bi, err := build.getV1BuildImage(stackConf.parsed.Env, stackConf.namespace)
-	if err != nil {
-		return nil, nil, "", err
-	}
-
-	pi, err := build.getV1PushImage(stackConf.namespace)
-	if err != nil {
-		return nil, nil, "", err
-	}
-
-	return bi, pi, build.GetBuilder(), nil
-}
-
 func convertToBuild(porterApp *types.PorterApp) Build {
 	var context *string
 	if porterApp.BuildContext != "" {
@@ -211,12 +255,6 @@ func convertToBuild(porterApp *types.PorterApp) Build {
 	}
 }
 
-func createDefaultPorterYaml() *PorterStackYAML {
-	return &PorterStackYAML{
-		Apps: nil,
-	}
-}
-
 func getEnvFromRelease(client *api.Client, stackName string, projectID uint, clusterID uint) map[string]string {
 	var envVarsStringMap map[string]string
 	namespace := fmt.Sprintf("porter-stack-%s", stackName)

cli/cmd/stack/build.go

@@ -8,12 +8,12 @@ import (
 	"github.com/porter-dev/switchboard/pkg/types"
 )
 
-func (b *Build) GetName() string {
+func (b *Build) GetName(appName string) string {
 	if b == nil {
 		return ""
 	}
 
-	return getBuildImageName()
+	return appName
 }
 
 func (b *Build) GetContext() string {
@@ -74,7 +74,7 @@ func (b *Build) GetImage() string {
 	return *b.Image
 }
 
-func (b *Build) getV1BuildImage(env map[string]string, namespace string) (*types.Resource, error) {
+func (b *Build) getV1BuildImage(appName string, env map[string]string, namespace string) (*types.Resource, error) {
 	config := &preview.BuildDriverConfig{}
 
 	if b.GetMethod() == "pack" {
@@ -104,13 +104,13 @@ func (b *Build) getV1BuildImage(env map[string]string, namespace string) (*types
 	}
 
 	return &types.Resource{
-		Name:   fmt.Sprintf("%s-build-image", b.GetName()),
+		Name:   fmt.Sprintf("%s-build-image", b.GetName(appName)),
 		Driver: "build-image",
 		Source: map[string]any{
 			"name": "web",
 		},
 		Target: map[string]any{
-			"app_name":  b.GetName(),
+			"app_name":  b.GetName(appName),
 			"namespace": namespace,
 		},
 		DependsOn: []string{
@@ -120,18 +120,14 @@ func (b *Build) getV1BuildImage(env map[string]string, namespace string) (*types
 	}, nil
 }
 
-func getBuildImageName() string {
-	return "base-image"
+func GetBuildImageDriverName(appName string) string {
+	return fmt.Sprintf("%s-build-image", appName)
 }
 
-func GetBuildImageDriverName() string {
-	return fmt.Sprintf("%s-build-image", getBuildImageName())
-}
-
-func (b *Build) getV1PushImage(namespace string) (*types.Resource, error) {
+func (b *Build) getV1PushImage(appName string, namespace string) (*types.Resource, error) {
 	config := &preview.PushDriverConfig{}
 
-	config.Push.Image = fmt.Sprintf("{ .%s.image }", GetBuildImageDriverName())
+	config.Push.Image = fmt.Sprintf("{ .%s.image }", GetBuildImageDriverName(appName))
 
 	rawConfig := make(map[string]any)
 
@@ -141,14 +137,14 @@ func (b *Build) getV1PushImage(namespace string) (*types.Resource, error) {
 	}
 
 	return &types.Resource{
-		Name:   b.GetName(),
+		Name:   fmt.Sprintf("%s-push-image", b.GetName(appName)),
 		Driver: "push-image",
 		DependsOn: []string{
 			"get-env",
-			GetBuildImageDriverName(),
+			GetBuildImageDriverName(appName),
 		},
 		Target: map[string]any{
-			"app_name":  b.GetName(),
+			"app_name":  b.GetName(appName),
 			"namespace": namespace,
 		},
 		Config: rawConfig,

cli/cmd/stack/hooks.go

@@ -12,16 +12,16 @@ import (
 	"github.com/porter-dev/porter/cli/cmd/config"
 )
 
-type DeployStackHook struct {
+type DeployAppHook struct {
 	Client               *api.Client
-	StackName            string
+	ApplicationName      string
 	ProjectID, ClusterID uint
 	BuildImageDriverName string
 	PorterYAML           []byte
 	Builder              string
 }
 
-func (t *DeployStackHook) PreApply() error {
+func (t *DeployAppHook) PreApply() error {
 	err := config.ValidateCLIEnvironment()
 	if err != nil {
 		errMsg := composePreviewMessage("porter CLI is not configured correctly", Error)
@@ -30,38 +30,38 @@ func (t *DeployStackHook) PreApply() error {
 	return nil
 }
 
-func (t *DeployStackHook) DataQueries() map[string]interface{} {
+func (t *DeployAppHook) DataQueries() map[string]interface{} {
 	res := map[string]interface{}{
 		"image": fmt.Sprintf("{$.%s.image}", t.BuildImageDriverName),
 	}
 	return res
 }
 
-// deploy the stack
-func (t *DeployStackHook) PostApply(driverOutput map[string]interface{}) error {
+// deploy the app
+func (t *DeployAppHook) PostApply(driverOutput map[string]interface{}) error {
 	client := config.GetAPIClient()
-	namespace := fmt.Sprintf("porter-stack-%s", t.StackName)
+	namespace := fmt.Sprintf("porter-stack-%s", t.ApplicationName)
 
 	_, err := client.GetRelease(
 		context.Background(),
 		t.ProjectID,
 		t.ClusterID,
 		namespace,
-		t.StackName,
+		t.ApplicationName,
 	)
 
 	shouldCreate := err != nil
 
 	if err != nil {
-		color.New(color.FgYellow).Printf("Could not read release for stack %s (%s): attempting creation\n", t.StackName, err.Error())
+		color.New(color.FgYellow).Printf("Could not read release for app %s (%s): attempting creation\n", t.ApplicationName, err.Error())
 	} else {
-		color.New(color.FgGreen).Printf("Found release for stack %s: attempting update\n", t.StackName)
+		color.New(color.FgGreen).Printf("Found release for app %s: attempting update\n", t.ApplicationName)
 	}
 
-	return t.applyStack(client, shouldCreate, driverOutput)
+	return t.applyApp(client, shouldCreate, driverOutput)
 }
 
-func (t *DeployStackHook) applyStack(client *api.Client, shouldCreate bool, driverOutput map[string]interface{}) error {
+func (t *DeployAppHook) applyApp(client *api.Client, shouldCreate bool, driverOutput map[string]interface{}) error {
 	var imageInfo types.ImageInfo
 	image, ok := driverOutput["image"].(string)
 	// if it contains a $, then it means the query didn't resolve to anything
@@ -81,7 +81,7 @@ func (t *DeployStackHook) applyStack(client *api.Client, shouldCreate bool, driv
 		context.Background(),
 		t.ProjectID,
 		t.ClusterID,
-		t.StackName,
+		t.ApplicationName,
 		&types.CreatePorterAppRequest{
 			ClusterID:        t.ClusterID,
 			ProjectID:        t.ProjectID,
@@ -93,13 +93,13 @@ func (t *DeployStackHook) applyStack(client *api.Client, shouldCreate bool, driv
 	)
 	if err != nil {
 		if shouldCreate {
-			return fmt.Errorf("error creating stack %s: %w", t.StackName, err)
+			return fmt.Errorf("error creating app %s: %w", t.ApplicationName, err)
 		}
-		return fmt.Errorf("error updating stack %s: %w", t.StackName, err)
+		return fmt.Errorf("error updating app %s: %w", t.ApplicationName, err)
 	}
 
 	return nil
 }
 
-func (t *DeployStackHook) OnConsolidatedErrors(map[string]error) {}
-func (t *DeployStackHook) OnError(error)                         {}
+func (t *DeployAppHook) OnConsolidatedErrors(map[string]error) {}
+func (t *DeployAppHook) OnError(error)                         {}

cli/cmd/stack/preDeploy.go

@@ -12,7 +12,7 @@ import (
 	switchboardTypes "github.com/porter-dev/switchboard/pkg/types"
 )
 
-func createPreDeployResource(client *api.Client, release *App, stackName, buildResourceName, pushResourceName string, projectID, clusterID uint, env map[string]string) (*switchboardTypes.Resource, string, error) {
+func createPreDeployResource(client *api.Client, release *Service, stackName, buildResourceName, pushResourceName string, projectID, clusterID uint, env map[string]string) (*switchboardTypes.Resource, string, error) {
 	var finalCmd string
 	if release != nil && release.Run != nil {
 		finalCmd = *release.Run

cli/cmd/stack/types.go

@@ -1,11 +1,23 @@
 package stack
 
 type PorterStackYAML struct {
-	Version *string           `yaml:"version"`
-	Build   *Build            `yaml:"build"`
-	Env     map[string]string `yaml:"env"`
-	Apps    map[string]*App   `yaml:"apps"`
-	Release *App              `yaml:"release"`
+	Applications map[string]*Application `yaml:"applications" validate:"required_without=Services Apps"`
+	Version      *string                 `yaml:"version"`
+	Build        *Build                  `yaml:"build"`
+	Env          map[string]string       `yaml:"env"`
+	SyncedEnv    []*SyncedEnvSection     `yaml:"synced_env"`
+	Apps         map[string]*Service     `yaml:"apps" validate:"required_without=Applications Services"`
+	Services     map[string]*Service     `yaml:"services" validate:"required_without=Applications Apps"`
+
+	Release *Service `yaml:"release"`
+}
+
+type Application struct {
+	Services map[string]*Service `yaml:"services" validate:"required"`
+	Build    *Build              `yaml:"build"`
+	Env      map[string]string   `yaml:"env"`
+
+	Release *Service `yaml:"release"`
 }
 
 type Build struct {
@@ -17,8 +29,19 @@ type Build struct {
 	Image      *string   `yaml:"image" validate:"required_if=Method registry"`
 }
 
-type App struct {
-	Run    *string                `yaml:"run" validate:"required"`
+type Service struct {
+	Run    *string                `yaml:"run"`
 	Config map[string]interface{} `yaml:"config"`
 	Type   *string                `yaml:"type" validate:"required, oneof=web worker job"`
 }
+
+type SyncedEnvSection struct {
+	Name    string                `json:"name" yaml:"name"`
+	Version uint                  `json:"version" yaml:"version"`
+	Keys    []SyncedEnvSectionKey `json:"keys" yaml:"keys"`
+}
+
+type SyncedEnvSectionKey struct {
+	Name   string `json:"name" yaml:"name"`
+	Secret bool   `json:"secret" yaml:"secret"`
+}

cli/cmd/stack/validate.go

@@ -0,0 +1,59 @@
+package stack
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+func createDefaultPorterYaml() *PorterStackYAML {
+	return &PorterStackYAML{
+		Apps: nil,
+	}
+}
+
+func ValidateAndMarshal(raw []byte) (*PorterStackYAML, error) {
+	var parsed *PorterStackYAML
+
+	if raw == nil {
+		parsed = createDefaultPorterYaml()
+	} else {
+		parsed = &PorterStackYAML{}
+		err := yaml.Unmarshal(raw, parsed)
+		if err != nil {
+			errMsg := composePreviewMessage("error parsing porter.yaml", Error)
+			return nil, fmt.Errorf("%s: %w", errMsg, err)
+		}
+	}
+
+	return parsed, nil
+}
+
+func CreateAppFromFile(base *PorterStackYAML) (*Application, error) {
+	if base.Applications != nil {
+		errMsg := composePreviewMessage("error parsing porter.yaml", Error)
+		err := fmt.Errorf("expected one porter app, found many")
+		return nil, fmt.Errorf("%s: %w", errMsg, err)
+	}
+
+	if base.Apps != nil && base.Services != nil {
+		errMsg := composePreviewMessage("error parsing porter.yaml", Error)
+		err := fmt.Errorf("'apps' and 'services' are synonymous but both were defined")
+		return nil, fmt.Errorf("%s: %w", errMsg, err)
+	}
+
+	var services map[string]*Service
+
+	if base.Apps != nil {
+		services = base.Apps
+	} else {
+		services = base.Services
+	}
+
+	return &Application{
+		Env:      base.Env,
+		Services: services,
+		Build:    base.Build,
+		Release:  base.Release,
+	}, nil
+}