parse.go 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062
  1. package porter_app
  2. import (
  3. "context"
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. "github.com/porter-dev/porter/api/server/shared/config"
  8. "github.com/porter-dev/porter/api/types"
  9. porterAppUtils "github.com/porter-dev/porter/api/utils/porter_app"
  10. "github.com/porter-dev/porter/internal/helm/loader"
  11. "github.com/porter-dev/porter/internal/integrations/dns"
  12. "github.com/porter-dev/porter/internal/kubernetes"
  13. "github.com/porter-dev/porter/internal/kubernetes/domain"
  14. "github.com/porter-dev/porter/internal/kubernetes/environment_groups"
  15. "github.com/porter-dev/porter/internal/kubernetes/porter_app"
  16. "github.com/porter-dev/porter/internal/repository"
  17. "github.com/porter-dev/porter/internal/telemetry"
  18. "github.com/porter-dev/porter/internal/templater/utils"
  19. "github.com/stefanmcshane/helm/pkg/chart"
  20. "gopkg.in/yaml.v2"
  21. )
  22. type PorterStackYAML struct {
  23. Applications map[string]*Application `yaml:"applications" validate:"required_without=Services Apps"`
  24. Version *string `yaml:"version"`
  25. Build *Build `yaml:"build"`
  26. Env map[string]string `yaml:"env"`
  27. SyncedEnv []*SyncedEnvSection `yaml:"synced_env"`
  28. Apps map[string]*Service `yaml:"apps" validate:"required_without=Applications Services"`
  29. Services map[string]*Service `yaml:"services" validate:"required_without=Applications Apps"`
  30. Release *Service `yaml:"release"`
  31. }
  32. type Application struct {
  33. Services map[string]*Service `yaml:"services" validate:"required"`
  34. Build *Build `yaml:"build"`
  35. Env map[string]string `yaml:"env"`
  36. Release *Service `yaml:"release"`
  37. }
  38. type Build struct {
  39. Context *string `yaml:"context" validate:"dir"`
  40. Method *string `yaml:"method" validate:"required,oneof=pack docker registry"`
  41. Builder *string `yaml:"builder" validate:"required_if=Method pack"`
  42. Buildpacks []*string `yaml:"buildpacks"`
  43. Dockerfile *string `yaml:"dockerfile" validate:"required_if=Method docker"`
  44. Image *string `yaml:"image" validate:"required_if=Method registry"`
  45. }
  46. type Service struct {
  47. Run *string `yaml:"run"`
  48. Config map[string]interface{} `yaml:"config"`
  49. Type *string `yaml:"type" validate:"required, oneof=web worker job"`
  50. }
  51. type SyncedEnvSection struct {
  52. Name string `json:"name" yaml:"name"`
  53. Version uint `json:"version" yaml:"version"`
  54. Keys []SyncedEnvSectionKey `json:"keys" yaml:"keys"`
  55. }
  56. type SyncedEnvSectionKey struct {
  57. Name string `json:"name" yaml:"name"`
  58. Secret bool `json:"secret" yaml:"secret"`
  59. }
  60. type SubdomainCreateOpts struct {
  61. k8sAgent *kubernetes.Agent
  62. dnsRepo repository.DNSRecordRepository
  63. dnsClient *dns.Client
  64. appRootDomain string
  65. stackName string
  66. }
  67. type ParseConf struct {
  68. // PorterAppName is the name of the porter app
  69. PorterAppName string
  70. // PorterYaml is the raw porter yaml which is used to build the values + chart for helm upgrade
  71. PorterYaml []byte
  72. // ImageInfo contains the repository and tag of the image to use for the helm upgrade. Kept separate from the PorterYaml because the image info
  73. // is stored in the 'global' key of the values, which is not part of the porter yaml
  74. ImageInfo types.ImageInfo
  75. // ServerConfig is the server conf, used to find the default helm repo
  76. ServerConfig *config.Config
  77. // ProjectID
  78. ProjectID uint
  79. // UserUpdate used for synced env groups
  80. UserUpdate bool
  81. // EnvGroups used for synced env groups
  82. EnvGroups []string
  83. // EnvironmentGroups are used for syncing environment groups using ConfigMaps and Secrets from porter-env-groups namespace. This should be used instead of EnvGroups
  84. EnvironmentGroups []string
  85. // Namespace used for synced env groups
  86. Namespace string
  87. // ExistingHelmValues is the existing values for the helm release, if it exists
  88. ExistingHelmValues map[string]interface{}
  89. // ExistingChartDependencies is the existing dependencies for the helm release, if it exists
  90. ExistingChartDependencies []*chart.Dependency
  91. // SubdomainCreateOpts contains the necessary information to create a subdomain if necessary
  92. SubdomainCreateOpts SubdomainCreateOpts
  93. // InjectLauncherToStartCommand is a flag to determine whether to prepend the launcher to the start command
  94. InjectLauncherToStartCommand bool
  95. // ShouldValidateHelmValues is a flag to determine whether to validate helm values
  96. ShouldValidateHelmValues bool
  97. // FullHelmValues if provided, override anything specified in porter.yaml. Used as an escape hatch for support
  98. FullHelmValues string
  99. // AddCustomNodeSelector is a flag to determine whether to add porter.run/workload-kind: application to the nodeselector attribute of the helm values
  100. AddCustomNodeSelector bool
  101. // RemoveDeletedServices is a flag to determine whether to remove values and dependencies for services that are not defined in the porter.yaml
  102. RemoveDeletedServices bool
  103. }
  104. func parse(ctx context.Context, conf ParseConf) (*chart.Chart, map[string]interface{}, map[string]interface{}, error) {
  105. ctx, span := telemetry.NewSpan(ctx, "parse-porter-yaml")
  106. defer span.End()
  107. parsed := &PorterStackYAML{}
  108. if conf.FullHelmValues != "" {
  109. parsedHelmValues, err := convertHelmValuesToPorterYaml(conf.FullHelmValues)
  110. if err != nil {
  111. err = telemetry.Error(ctx, span, err, "error parsing raw helm values")
  112. return nil, nil, nil, err
  113. }
  114. parsed = parsedHelmValues
  115. } else {
  116. err := yaml.Unmarshal(conf.PorterYaml, parsed)
  117. if err != nil {
  118. err = telemetry.Error(ctx, span, err, "error parsing porter.yaml")
  119. return nil, nil, nil, err
  120. }
  121. }
  122. synced_env := make([]*SyncedEnvSection, 0)
  123. for i := range conf.EnvGroups {
  124. cm, _, err := conf.SubdomainCreateOpts.k8sAgent.GetLatestVersionedConfigMap(conf.EnvGroups[i], conf.Namespace)
  125. if err != nil {
  126. err = telemetry.Error(ctx, span, err, "error getting latest versioned config map")
  127. return nil, nil, nil, err
  128. }
  129. versionStr, ok := cm.ObjectMeta.Labels["version"]
  130. if !ok {
  131. err = telemetry.Error(ctx, span, nil, "error extracting version from config map")
  132. return nil, nil, nil, err
  133. }
  134. versionInt, err := strconv.Atoi(versionStr)
  135. if err != nil {
  136. err = telemetry.Error(ctx, span, err, "error converting version to int")
  137. return nil, nil, nil, err
  138. }
  139. version := uint(versionInt)
  140. newSection := &SyncedEnvSection{
  141. Name: conf.EnvGroups[i],
  142. Version: version,
  143. }
  144. newSectionKeys := make([]SyncedEnvSectionKey, 0)
  145. for key, val := range cm.Data {
  146. newSectionKeys = append(newSectionKeys, SyncedEnvSectionKey{
  147. Name: key,
  148. Secret: strings.Contains(val, "PORTERSECRET"),
  149. })
  150. }
  151. newSection.Keys = newSectionKeys
  152. synced_env = append(synced_env, newSection)
  153. }
  154. if parsed.Apps != nil && parsed.Services != nil {
  155. err := telemetry.Error(ctx, span, nil, "'apps' and 'services' are synonymous but both were defined")
  156. return nil, nil, nil, err
  157. }
  158. var services map[string]*Service
  159. if parsed.Apps != nil {
  160. services = parsed.Apps
  161. }
  162. if parsed.Services != nil {
  163. services = parsed.Services
  164. }
  165. for serviceName := range services {
  166. services[serviceName] = addLabelsToService(services[serviceName], conf.EnvironmentGroups, porter_app.LabelKey_PorterApplication)
  167. }
  168. application := &Application{
  169. Env: parsed.Env,
  170. Services: services,
  171. Build: parsed.Build,
  172. Release: parsed.Release,
  173. }
  174. values, err := buildUmbrellaChartValues(ctx, application, synced_env, conf.ImageInfo, conf.ExistingHelmValues, conf.SubdomainCreateOpts, conf.InjectLauncherToStartCommand, conf.ShouldValidateHelmValues, conf.UserUpdate, conf.Namespace, conf.AddCustomNodeSelector, conf.RemoveDeletedServices)
  175. if err != nil {
  176. err = telemetry.Error(ctx, span, err, "error building values")
  177. return nil, nil, nil, err
  178. }
  179. convertedValues, ok := convertMap(values).(map[string]interface{})
  180. if !ok {
  181. err = telemetry.Error(ctx, span, nil, "error converting values")
  182. return nil, nil, nil, err
  183. }
  184. umbrellaChart, err := buildUmbrellaChart(application, conf.ServerConfig, conf.ProjectID, conf.ExistingChartDependencies, conf.RemoveDeletedServices)
  185. if err != nil {
  186. err = telemetry.Error(ctx, span, err, "error building umbrella chart")
  187. return nil, nil, nil, err
  188. }
  189. // return the parsed release values for the release job chart, if they exist
  190. var preDeployJobValues map[string]interface{}
  191. if application.Release != nil && application.Release.Run != nil {
  192. application.Release = addLabelsToService(application.Release, conf.EnvironmentGroups, porter_app.LabelKey_PorterApplicationPreDeploy)
  193. preDeployJobValues = buildPreDeployJobChartValues(application.Release, application.Env, synced_env, conf.ImageInfo, conf.InjectLauncherToStartCommand, conf.ExistingHelmValues, porterAppUtils.PredeployJobNameFromPorterAppName(conf.PorterAppName), conf.UserUpdate, conf.AddCustomNodeSelector)
  194. }
  195. return umbrellaChart, convertedValues, preDeployJobValues, nil
  196. }
  197. func buildUmbrellaChartValues(
  198. ctx context.Context,
  199. application *Application,
  200. syncedEnv []*SyncedEnvSection,
  201. imageInfo types.ImageInfo,
  202. existingValues map[string]interface{},
  203. opts SubdomainCreateOpts,
  204. injectLauncher bool,
  205. shouldValidateHelmValues bool,
  206. userUpdate bool,
  207. namespace string,
  208. addCustomNodeSelector bool,
  209. removeDeletedValues bool,
  210. ) (map[string]interface{}, error) {
  211. values := make(map[string]interface{})
  212. if application.Services == nil {
  213. if existingValues == nil {
  214. return nil, fmt.Errorf("porter.yaml must contain at least one service, or pre-deploy must exist and have values")
  215. }
  216. }
  217. for name, service := range application.Services {
  218. serviceType := getType(name, service)
  219. defaultValues := getDefaultValues(service, application.Env, syncedEnv, serviceType, existingValues, name, userUpdate, addCustomNodeSelector)
  220. convertedConfig := convertMap(service.Config).(map[string]interface{})
  221. helm_values := utils.DeepCoalesceValues(defaultValues, convertedConfig)
  222. // required to identify the chart type because of https://github.com/helm/helm/issues/9214
  223. helmName := getHelmName(name, serviceType)
  224. if existingValues != nil {
  225. if existingValues[helmName] != nil {
  226. existingValuesMap := existingValues[helmName].(map[string]interface{})
  227. if removeDeletedValues {
  228. // strip the env variables before coalescing
  229. if existingValuesMap["container"] != nil {
  230. containerMap := existingValuesMap["container"].(map[string]interface{})
  231. if containerMap["env"] != nil {
  232. envMap := containerMap["env"].(map[string]interface{})
  233. if envMap["normal"] != nil {
  234. envMap["normal"] = make(map[string]interface{})
  235. }
  236. if envMap["synced"] != nil {
  237. envMap["synced"] = make([]map[string]interface{}, 0)
  238. }
  239. containerMap["env"] = envMap
  240. }
  241. existingValuesMap["container"] = containerMap
  242. }
  243. }
  244. helm_values = utils.DeepCoalesceValues(existingValuesMap, helm_values)
  245. }
  246. }
  247. validateErr := validateHelmValues(helm_values, shouldValidateHelmValues, serviceType)
  248. if validateErr != "" {
  249. return nil, fmt.Errorf("error validating service \"%s\": %s", name, validateErr)
  250. }
  251. err := syncEnvironmentGroupToNamespaceIfLabelsExist(ctx, opts.k8sAgent, service, namespace)
  252. if err != nil {
  253. return nil, fmt.Errorf("error syncing environment group to namespace: %w", err)
  254. }
  255. err = createSubdomainIfRequired(helm_values, opts) // modifies helm_values to add subdomains if necessary
  256. if err != nil {
  257. return nil, err
  258. }
  259. // just in case this slips by
  260. if serviceType == "web" {
  261. if helm_values["ingress"] == nil {
  262. helm_values["ingress"] = map[string]interface{}{
  263. "enabled": false,
  264. }
  265. }
  266. }
  267. values[helmName] = helm_values
  268. }
  269. if !removeDeletedValues {
  270. // add back in the existing services that were not overwritten
  271. for k, v := range existingValues {
  272. if values[k] == nil {
  273. values[k] = v
  274. }
  275. }
  276. }
  277. // prepend launcher to all start commands if we need to
  278. for _, v := range values {
  279. if serviceValues, ok := v.(map[string]interface{}); ok {
  280. if serviceValues["container"] != nil {
  281. containerMap := serviceValues["container"].(map[string]interface{})
  282. if containerMap["command"] != nil {
  283. command := containerMap["command"].(string)
  284. if injectLauncher && !strings.HasPrefix(command, "launcher") && !strings.HasPrefix(command, "/cnb/lifecycle/launcher") {
  285. containerMap["command"] = fmt.Sprintf("/cnb/lifecycle/launcher %s", command)
  286. }
  287. }
  288. }
  289. }
  290. }
  291. values["global"] = map[string]interface{}{
  292. "image": map[string]interface{}{
  293. "repository": imageInfo.Repository,
  294. "tag": imageInfo.Tag,
  295. },
  296. }
  297. return values, nil
  298. }
  299. // syncEnvironmentGroupToNamespaceIfLabelsExist will sync the latest version of the environment group to the target namespace if the service has the appropriate label.
  300. func syncEnvironmentGroupToNamespaceIfLabelsExist(ctx context.Context, agent *kubernetes.Agent, service *Service, targetNamespace string) error {
  301. var linkedGroupNames string
  302. // patchwork because we are not consistent with the type of labels
  303. if labels, ok := service.Config["labels"].(map[string]any); ok {
  304. if linkedGroup, ok := labels[environment_groups.LabelKey_LinkedEnvironmentGroup].(string); ok {
  305. linkedGroupNames = linkedGroup
  306. }
  307. }
  308. if labels, ok := service.Config["labels"].(map[string]string); ok {
  309. if linkedGroup, ok := labels[environment_groups.LabelKey_LinkedEnvironmentGroup]; ok {
  310. linkedGroupNames = linkedGroup
  311. }
  312. }
  313. service.Config["configMapRefs"] = []string{}
  314. service.Config["secretRefs"] = []string{}
  315. for _, linkedGroupName := range strings.Split(linkedGroupNames, ".") {
  316. inp := environment_groups.SyncLatestVersionToNamespaceInput{
  317. BaseEnvironmentGroupName: linkedGroupName,
  318. TargetNamespace: targetNamespace,
  319. }
  320. syncedEnvironment, err := environment_groups.SyncLatestVersionToNamespace(ctx, agent, inp, nil)
  321. if err != nil {
  322. return fmt.Errorf("error syncing environment group: %w", err)
  323. }
  324. if syncedEnvironment.EnvironmentGroupVersionedName != "" {
  325. if service.Config["configMapRefs"] == nil {
  326. service.Config["configMapRefs"] = []string{}
  327. }
  328. if service.Config["secretRefs"] == nil {
  329. service.Config["secretRefs"] = []string{}
  330. }
  331. switch service.Config["configMapRefs"].(type) {
  332. case []string:
  333. service.Config["configMapRefs"] = append(service.Config["configMapRefs"].([]string), syncedEnvironment.EnvironmentGroupVersionedName)
  334. case []any:
  335. service.Config["configMapRefs"] = append(service.Config["configMapRefs"].([]any), syncedEnvironment.EnvironmentGroupVersionedName)
  336. }
  337. switch service.Config["secretRefs"].(type) {
  338. case []string:
  339. service.Config["secretRefs"] = append(service.Config["secretRefs"].([]string), syncedEnvironment.EnvironmentGroupVersionedName)
  340. case []any:
  341. service.Config["secretRefs"] = append(service.Config["secretRefs"].([]any), syncedEnvironment.EnvironmentGroupVersionedName)
  342. }
  343. }
  344. }
  345. return nil
  346. }
  347. // we can add to this function up later or use an alternative
  348. func validateHelmValues(values map[string]interface{}, shouldValidateHelmValues bool, appType string) string {
  349. if shouldValidateHelmValues {
  350. // validate port for web services
  351. if appType == "web" {
  352. containerMap, err := getNestedMap(values, "container")
  353. if err != nil {
  354. return "error checking port: misformatted values"
  355. } else {
  356. portVal, portExists := containerMap["port"]
  357. if portExists {
  358. portStr, pOK := portVal.(string)
  359. if !pOK {
  360. return "error checking port: no port in container"
  361. }
  362. port, err := strconv.Atoi(portStr)
  363. if err != nil || port < 1024 || port > 65535 {
  364. return "port must be a number between 1024 and 65535"
  365. }
  366. } else {
  367. return "port must be specified for web services"
  368. }
  369. }
  370. }
  371. }
  372. return ""
  373. }
  374. func buildPreDeployJobChartValues(release *Service, env map[string]string, synced_env []*SyncedEnvSection, imageInfo types.ImageInfo, injectLauncher bool, existingValues map[string]interface{}, name string, userUpdate bool, addCustomNodeSelector bool) map[string]interface{} {
  375. defaultValues := getDefaultValues(release, env, synced_env, "job", existingValues, name+"-r", userUpdate, addCustomNodeSelector)
  376. convertedConfig := convertMap(release.Config).(map[string]interface{})
  377. helm_values := utils.DeepCoalesceValues(defaultValues, convertedConfig)
  378. if imageInfo.Repository != "" && imageInfo.Tag != "" {
  379. helm_values["image"] = map[string]interface{}{
  380. "repository": imageInfo.Repository,
  381. "tag": imageInfo.Tag,
  382. }
  383. }
  384. // prepend launcher if we need to
  385. if helm_values["container"] != nil {
  386. containerMap := helm_values["container"].(map[string]interface{})
  387. if containerMap["command"] != nil {
  388. command := containerMap["command"].(string)
  389. if injectLauncher && !strings.HasPrefix(command, "launcher") && !strings.HasPrefix(command, "/cnb/lifecycle/launcher") {
  390. containerMap["command"] = fmt.Sprintf("/cnb/lifecycle/launcher %s", command)
  391. }
  392. }
  393. }
  394. return helm_values
  395. }
  396. func getType(name string, service *Service) string {
  397. if service.Type != nil {
  398. return *service.Type
  399. }
  400. if strings.Contains(name, "web") {
  401. return "web"
  402. }
  403. if strings.Contains(name, "job") {
  404. return "job"
  405. }
  406. return "worker"
  407. }
  408. func getDefaultValues(service *Service, env map[string]string, synced_env []*SyncedEnvSection, appType string, existingValues map[string]interface{}, name string, userUpdate bool, addCustomNodeSelector bool) map[string]interface{} {
  409. var defaultValues map[string]interface{}
  410. var runCommand string
  411. if service.Run != nil {
  412. runCommand = *service.Run
  413. }
  414. var syncedEnvs []map[string]interface{}
  415. envConf, err := getStacksNestedMap(existingValues, name+"-"+appType, "container", "env")
  416. if !userUpdate && err == nil {
  417. syncedEnvs = envConf
  418. } else {
  419. syncedEnvs = deconstructSyncedEnvs(synced_env, env)
  420. }
  421. defaultValues = map[string]interface{}{
  422. "container": map[string]interface{}{
  423. "command": runCommand,
  424. "env": map[string]interface{}{
  425. "normal": CopyEnv(env),
  426. "synced": syncedEnvs,
  427. },
  428. },
  429. "nodeSelector": map[string]interface{}{},
  430. }
  431. if addCustomNodeSelector {
  432. defaultValues["nodeSelector"] = map[string]interface{}{
  433. "porter.run/workload-kind": "application",
  434. }
  435. }
  436. return defaultValues
  437. }
  438. func deconstructSyncedEnvs(synced_env []*SyncedEnvSection, env map[string]string) []map[string]interface{} {
  439. synced := make([]map[string]interface{}, 0)
  440. for _, group := range synced_env {
  441. keys := make([]map[string]interface{}, 0)
  442. for _, key := range group.Keys {
  443. if _, exists := env[key.Name]; !exists {
  444. // Only include keys not present in env
  445. keys = append(keys, map[string]interface{}{
  446. "name": key.Name,
  447. "secret": key.Secret,
  448. })
  449. }
  450. }
  451. syncedGroup := map[string]interface{}{
  452. "keys": keys,
  453. "name": group.Name,
  454. "version": group.Version,
  455. }
  456. synced = append(synced, syncedGroup)
  457. }
  458. return synced
  459. }
  460. func buildUmbrellaChart(application *Application, config *config.Config, projectID uint, existingDependencies []*chart.Dependency, removeDeletedDependencies bool) (*chart.Chart, error) {
  461. deps := make([]*chart.Dependency, 0)
  462. for alias, service := range application.Services {
  463. var serviceType string
  464. if existingDependencies != nil {
  465. for _, dep := range existingDependencies {
  466. // this condition checks that the dependency is of the form <alias>-web or <alias>-wkr or <alias>-job, meaning it already exists in the chart
  467. if strings.HasPrefix(dep.Alias, fmt.Sprintf("%s-", alias)) && (strings.HasSuffix(dep.Alias, "-web") || strings.HasSuffix(dep.Alias, "-wkr") || strings.HasSuffix(dep.Alias, "-job")) {
  468. serviceType = getChartTypeFromHelmName(dep.Alias)
  469. if serviceType == "" {
  470. return nil, fmt.Errorf("unable to determine type of existing dependency")
  471. }
  472. }
  473. }
  474. // this is a new app, so we need to get the type from the app name or type
  475. if serviceType == "" {
  476. serviceType = getType(alias, service)
  477. }
  478. } else {
  479. serviceType = getType(alias, service)
  480. }
  481. selectedRepo := config.ServerConf.DefaultApplicationHelmRepoURL
  482. selectedVersion, err := getLatestTemplateVersion(serviceType, config, projectID)
  483. if err != nil {
  484. return nil, err
  485. }
  486. helmName := getHelmName(alias, serviceType)
  487. deps = append(deps, &chart.Dependency{
  488. Name: serviceType,
  489. Alias: helmName,
  490. Version: selectedVersion,
  491. Repository: selectedRepo,
  492. })
  493. }
  494. if !removeDeletedDependencies {
  495. // add in the existing dependencies that were not overwritten
  496. for _, dep := range existingDependencies {
  497. if !dependencyExists(deps, dep) {
  498. // have to repair the dependency name because of https://github.com/helm/helm/issues/9214
  499. if strings.HasSuffix(dep.Name, "-web") || strings.HasSuffix(dep.Name, "-wkr") || strings.HasSuffix(dep.Name, "-job") {
  500. dep.Name = getChartTypeFromHelmName(dep.Name)
  501. if dep.Name == "" {
  502. return nil, fmt.Errorf("unable to determine type of existing dependency")
  503. }
  504. version, err := getLatestTemplateVersion(dep.Name, config, projectID)
  505. if err != nil {
  506. return nil, err
  507. }
  508. dep.Version = version
  509. }
  510. deps = append(deps, dep)
  511. }
  512. }
  513. }
  514. chart, err := createChartFromDependencies(deps)
  515. if err != nil {
  516. return nil, err
  517. }
  518. return chart, nil
  519. }
  520. func dependencyExists(deps []*chart.Dependency, dep *chart.Dependency) bool {
  521. for _, d := range deps {
  522. if d.Alias == dep.Alias {
  523. return true
  524. }
  525. }
  526. return false
  527. }
  528. func createChartFromDependencies(deps []*chart.Dependency) (*chart.Chart, error) {
  529. metadata := &chart.Metadata{
  530. Name: "umbrella",
  531. Description: "Web application that is exposed to external traffic.",
  532. Version: "0.96.0",
  533. APIVersion: "v2",
  534. Home: "https://getporter.dev/",
  535. Icon: "https://user-images.githubusercontent.com/65516095/111255214-07d3da80-85ed-11eb-99e2-fddcbdb99bdb.png",
  536. Keywords: []string{
  537. "porter",
  538. "application",
  539. "service",
  540. "umbrella",
  541. },
  542. Type: "application",
  543. Dependencies: deps,
  544. }
  545. // create a new chart object with the metadata
  546. c := &chart.Chart{
  547. Metadata: metadata,
  548. }
  549. return c, nil
  550. }
  551. func getLatestTemplateVersion(templateName string, config *config.Config, projectID uint) (string, error) {
  552. repoIndex, err := loader.LoadRepoIndexPublic(config.ServerConf.DefaultApplicationHelmRepoURL)
  553. if err != nil {
  554. return "", fmt.Errorf("%s: %w", "unable to load porter chart repo", err)
  555. }
  556. templates := loader.RepoIndexToPorterChartList(repoIndex, config.ServerConf.DefaultApplicationHelmRepoURL)
  557. if err != nil {
  558. return "", fmt.Errorf("%s: %w", "unable to load porter chart list", err)
  559. }
  560. var version string
  561. // find the matching template name
  562. for _, template := range templates {
  563. if templateName == template.Name {
  564. version = template.Versions[0]
  565. break
  566. }
  567. }
  568. if version == "" {
  569. return "", fmt.Errorf("matching template version not found")
  570. }
  571. return version, nil
  572. }
  573. func convertMap(m interface{}) interface{} {
  574. switch m := m.(type) {
  575. case map[string]interface{}:
  576. for k, v := range m {
  577. m[k] = convertMap(v)
  578. }
  579. case map[string]string:
  580. result := map[string]interface{}{}
  581. for k, v := range m {
  582. result[k] = v
  583. }
  584. return result
  585. case map[interface{}]interface{}:
  586. result := map[string]interface{}{}
  587. for k, v := range m {
  588. result[k.(string)] = convertMap(v)
  589. }
  590. return result
  591. case []interface{}:
  592. for i, v := range m {
  593. m[i] = convertMap(v)
  594. }
  595. }
  596. return m
  597. }
  598. func CopyEnv(env map[string]string) map[string]interface{} {
  599. envCopy := make(map[string]interface{})
  600. if env == nil {
  601. return envCopy
  602. }
  603. for k, v := range env {
  604. if k == "" || v == "" {
  605. continue
  606. }
  607. envCopy[k] = v
  608. }
  609. return envCopy
  610. }
  611. func createSubdomainIfRequired(
  612. mergedValues map[string]interface{},
  613. opts SubdomainCreateOpts,
  614. ) error {
  615. // look for ingress.enabled and no custom domains set
  616. ingressMap, err := getNestedMap(mergedValues, "ingress")
  617. if err == nil {
  618. enabledVal, enabledExists := ingressMap["enabled"]
  619. if enabledExists {
  620. enabled, eOK := enabledVal.(bool)
  621. if eOK && enabled {
  622. // if custom domain, we don't need to create a subdomain
  623. customDomVal, customDomExists := ingressMap["custom_domain"]
  624. if customDomExists {
  625. customDomain, cOK := customDomVal.(bool)
  626. if cOK && customDomain {
  627. return nil
  628. }
  629. }
  630. // subdomain already exists, no need to create one
  631. if porterHosts, ok := ingressMap["porter_hosts"].([]interface{}); ok && len(porterHosts) > 0 {
  632. return nil
  633. }
  634. // in the case of ingress enabled but no custom domain, create subdomain
  635. dnsRecord, err := createDNSRecord(opts)
  636. if err != nil {
  637. return fmt.Errorf("error creating subdomain: %s", err.Error())
  638. }
  639. subdomain := dnsRecord.ExternalURL
  640. if ingressVal, ok := mergedValues["ingress"]; !ok {
  641. mergedValues["ingress"] = map[string]interface{}{
  642. "porter_hosts": []string{
  643. subdomain,
  644. },
  645. }
  646. } else {
  647. ingressValMap := ingressVal.(map[string]interface{})
  648. ingressValMap["porter_hosts"] = []string{
  649. subdomain,
  650. }
  651. }
  652. }
  653. }
  654. }
  655. return nil
  656. }
  657. func createDNSRecord(opts SubdomainCreateOpts) (*types.DNSRecord, error) {
  658. if opts.dnsClient == nil {
  659. return nil, fmt.Errorf("cannot create subdomain because dns client is nil")
  660. }
  661. endpoint, found, err := domain.GetNGINXIngressServiceIP(opts.k8sAgent.Clientset)
  662. if err != nil {
  663. return nil, err
  664. }
  665. if !found {
  666. return nil, fmt.Errorf("target cluster does not have nginx ingress")
  667. }
  668. createDomain := domain.CreateDNSRecordConfig{
  669. ReleaseName: opts.stackName,
  670. RootDomain: opts.appRootDomain,
  671. Endpoint: endpoint,
  672. }
  673. record := createDomain.NewDNSRecordForEndpoint()
  674. record, err = opts.dnsRepo.CreateDNSRecord(record)
  675. if err != nil {
  676. return nil, err
  677. }
  678. _record := domain.DNSRecord(*record)
  679. err = _record.CreateDomain(opts.dnsClient)
  680. if err != nil {
  681. return nil, err
  682. }
  683. return record.ToDNSRecordType(), nil
  684. }
  685. func getNestedMap(obj map[string]interface{}, fields ...string) (map[string]interface{}, error) {
  686. var res map[string]interface{}
  687. curr := obj
  688. for _, field := range fields {
  689. objField, ok := curr[field]
  690. if !ok {
  691. return nil, fmt.Errorf("%s not found", field)
  692. }
  693. res, ok = objField.(map[string]interface{})
  694. if !ok {
  695. return nil, fmt.Errorf("%s is not a nested object", field)
  696. }
  697. curr = res
  698. }
  699. return res, nil
  700. }
  701. func getHelmName(alias string, t string) string {
  702. var suffix string
  703. if t == "web" {
  704. suffix = "-web"
  705. } else if t == "worker" {
  706. suffix = "-wkr"
  707. } else if t == "job" {
  708. suffix = "-job"
  709. }
  710. return fmt.Sprintf("%s%s", alias, suffix)
  711. }
  712. func getChartTypeFromHelmName(name string) string {
  713. if strings.HasSuffix(name, "-web") {
  714. return "web"
  715. } else if strings.HasSuffix(name, "-wkr") {
  716. return "worker"
  717. } else if strings.HasSuffix(name, "-job") {
  718. return "job"
  719. }
  720. return ""
  721. }
  722. func getServiceNameAndTypeFromHelmName(name string) (string, string) {
  723. if strings.HasSuffix(name, "-web") {
  724. return strings.TrimSuffix(name, "-web"), "web"
  725. } else if strings.HasSuffix(name, "-wkr") {
  726. return strings.TrimSuffix(name, "-wkr"), "worker"
  727. } else if strings.HasSuffix(name, "-job") {
  728. return strings.TrimSuffix(name, "-job"), "job"
  729. }
  730. return "", ""
  731. }
  732. func attemptToGetImageInfoFromRelease(values map[string]interface{}) types.ImageInfo {
  733. imageInfo := types.ImageInfo{}
  734. if values == nil {
  735. return imageInfo
  736. }
  737. globalImage, err := getNestedMap(values, "global", "image")
  738. if err != nil {
  739. return imageInfo
  740. }
  741. repoVal, okRepo := globalImage["repository"]
  742. tagVal, okTag := globalImage["tag"]
  743. if okRepo && okTag {
  744. imageInfo.Repository = repoVal.(string)
  745. imageInfo.Tag = tagVal.(string)
  746. }
  747. return imageInfo
  748. }
  749. func attemptToGetImageInfoFromFullHelmValues(fullHelmValues string) (types.ImageInfo, error) {
  750. imageInfo := types.ImageInfo{}
  751. var values map[string]interface{}
  752. err := yaml.Unmarshal([]byte(fullHelmValues), &values)
  753. if err != nil {
  754. return imageInfo, fmt.Errorf("error unmarshaling full helm values to read image info: %w", err)
  755. }
  756. convertedValues := convertMap(values).(map[string]interface{})
  757. return attemptToGetImageInfoFromRelease(convertedValues), nil
  758. }
  759. func getStacksNestedMap(obj map[string]interface{}, fields ...string) ([]map[string]interface{}, error) {
  760. var res map[string]interface{}
  761. curr := obj
  762. for _, field := range fields {
  763. objField, ok := curr[field]
  764. if !ok {
  765. return nil, fmt.Errorf("%s not found", field)
  766. }
  767. res, ok = objField.(map[string]interface{})
  768. if !ok {
  769. return nil, fmt.Errorf("%s is not a nested object", field)
  770. }
  771. curr = res
  772. }
  773. syncedInterface, ok := curr["synced"]
  774. if !ok {
  775. return nil, fmt.Errorf("synced not found")
  776. }
  777. synced, ok := syncedInterface.([]interface{})
  778. if !ok {
  779. return nil, fmt.Errorf("synced is not a slice of interface{}")
  780. }
  781. result := make([]map[string]interface{}, len(synced))
  782. for i, v := range synced {
  783. mapElement, ok := v.(map[string]interface{})
  784. if !ok {
  785. return nil, fmt.Errorf("element %d in synced is not a map[string]interface{}", i)
  786. }
  787. result[i] = mapElement
  788. }
  789. return result, nil
  790. }
  791. func convertHelmValuesToPorterYaml(helmValues string) (*PorterStackYAML, error) {
  792. var values map[string]interface{}
  793. err := yaml.Unmarshal([]byte(helmValues), &values)
  794. if err != nil {
  795. return nil, err
  796. }
  797. services := make(map[string]*Service)
  798. for k, v := range values {
  799. if k == "global" {
  800. continue
  801. }
  802. serviceName, serviceType := getServiceNameAndTypeFromHelmName(k)
  803. if serviceName == "" {
  804. return nil, fmt.Errorf("invalid service key: %s. make sure that service key ends in either -web, -wkr, or -job", k)
  805. }
  806. config := convertMap(v).(map[string]interface{})
  807. var runCommand string
  808. if config["container"] != nil {
  809. containerMap := config["container"].(map[string]interface{})
  810. if containerMap["command"] != nil {
  811. runCommand = containerMap["command"].(string)
  812. }
  813. }
  814. services[serviceName] = &Service{
  815. Run: &runCommand,
  816. Config: config,
  817. Type: &serviceType,
  818. }
  819. }
  820. return &PorterStackYAML{
  821. Services: services,
  822. }, nil
  823. }
  824. // addLabelsToService always adds the default label to the service, and if envGroups is not empty, it adds the corresponding environment group label as well.
  825. func addLabelsToService(service *Service, envGroups []string, defaultLabelKey string) *Service {
  826. if _, ok := service.Config["labels"]; !ok {
  827. service.Config["labels"] = make(map[string]string)
  828. }
  829. if len(envGroups) != 0 {
  830. // delete the env group label so we can replace it
  831. if _, ok := service.Config["labels"].(map[string]any); ok {
  832. delete(service.Config["labels"].(map[string]any), environment_groups.LabelKey_LinkedEnvironmentGroup)
  833. }
  834. }
  835. switch service.Config["labels"].(type) {
  836. case map[string]string:
  837. service.Config["labels"].(map[string]string)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  838. if len(envGroups) != 0 {
  839. service.Config["labels"].(map[string]string)[environment_groups.LabelKey_LinkedEnvironmentGroup] = strings.Join(envGroups, ".")
  840. }
  841. case map[string]any:
  842. service.Config["labels"].(map[string]any)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  843. if len(envGroups) != 0 {
  844. service.Config["labels"].(map[string]any)[environment_groups.LabelKey_LinkedEnvironmentGroup] = strings.Join(envGroups, ".")
  845. }
  846. case any:
  847. if val, ok := service.Config["labels"].(string); ok {
  848. if val == "" {
  849. service.Config["labels"] = map[string]string{
  850. defaultLabelKey: porter_app.LabelValue_PorterApplication,
  851. }
  852. if len(envGroups) != 0 {
  853. service.Config["labels"].(map[string]string)[environment_groups.LabelKey_LinkedEnvironmentGroup] = strings.Join(envGroups, ".")
  854. }
  855. }
  856. }
  857. }
  858. if _, ok := service.Config["podLabels"]; !ok {
  859. service.Config["podLabels"] = make(map[string]string)
  860. }
  861. switch service.Config["podLabels"].(type) {
  862. case map[string]string:
  863. service.Config["podLabels"].(map[string]string)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  864. case map[string]any:
  865. service.Config["podLabels"].(map[string]any)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  866. case any:
  867. if val, ok := service.Config["podLabels"].(string); ok {
  868. if val == "" {
  869. service.Config["podLabels"] = map[string]string{
  870. defaultLabelKey: porter_app.LabelValue_PorterApplication,
  871. }
  872. }
  873. }
  874. }
  875. return service
  876. }
  877. func getServiceDeploymentMetadataFromValues(values map[string]interface{}, status types.PorterAppEventStatus) map[string]types.ServiceDeploymentMetadata {
  878. serviceDeploymentMap := make(map[string]types.ServiceDeploymentMetadata)
  879. for key := range values {
  880. if key != "global" {
  881. serviceName, serviceType := getServiceNameAndTypeFromHelmName(key)
  882. externalURI := getServiceExternalURIFromServiceValues(values[key].(map[string]interface{}))
  883. // jobs don't technically have a deployment, so hardcode the deployment status to success
  884. serviceStatus := status
  885. if serviceType == "job" {
  886. serviceStatus = types.PorterAppEventStatus_Success
  887. }
  888. serviceDeploymentMap[serviceName] = types.ServiceDeploymentMetadata{
  889. ExternalURI: externalURI,
  890. Status: serviceStatus,
  891. Type: serviceType,
  892. }
  893. }
  894. }
  895. return serviceDeploymentMap
  896. }
  897. func getServiceExternalURIFromServiceValues(serviceValues map[string]interface{}) string {
  898. ingressMap, err := getNestedMap(serviceValues, "ingress")
  899. if err == nil {
  900. enabledVal, enabledExists := ingressMap["enabled"]
  901. if enabledExists {
  902. enabled, eOK := enabledVal.(bool)
  903. if eOK && enabled {
  904. customDomVal, customDomExists := ingressMap["custom_domain"]
  905. if customDomExists {
  906. customDomain, cOK := customDomVal.(bool)
  907. if cOK && customDomain {
  908. hostsExists, hostsExistsOK := ingressMap["hosts"]
  909. if hostsExistsOK {
  910. if hosts, hostsOK := hostsExists.([]interface{}); hostsOK && len(hosts) == 1 {
  911. return hosts[0].(string)
  912. }
  913. }
  914. }
  915. }
  916. if porterHosts, ok := ingressMap["porter_hosts"].([]interface{}); ok && len(porterHosts) == 1 {
  917. return porterHosts[0].(string)
  918. }
  919. }
  920. }
  921. }
  922. return ""
  923. }