parse.go 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065
  1. package porter_app
  2. import (
  3. "context"
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. "github.com/porter-dev/porter/api/server/shared/config"
  8. "github.com/porter-dev/porter/api/types"
  9. porterAppUtils "github.com/porter-dev/porter/api/utils/porter_app"
  10. "github.com/porter-dev/porter/internal/helm/loader"
  11. "github.com/porter-dev/porter/internal/integrations/dns"
  12. "github.com/porter-dev/porter/internal/kubernetes"
  13. "github.com/porter-dev/porter/internal/kubernetes/domain"
  14. "github.com/porter-dev/porter/internal/kubernetes/environment_groups"
  15. "github.com/porter-dev/porter/internal/kubernetes/porter_app"
  16. "github.com/porter-dev/porter/internal/repository"
  17. "github.com/porter-dev/porter/internal/telemetry"
  18. "github.com/porter-dev/porter/internal/templater/utils"
  19. "github.com/stefanmcshane/helm/pkg/chart"
  20. "gopkg.in/yaml.v2"
  21. )
  22. type PorterStackYAML struct {
  23. Applications map[string]*Application `yaml:"applications" validate:"required_without=Services Apps"`
  24. Version *string `yaml:"version"`
  25. Build *Build `yaml:"build"`
  26. Env map[string]string `yaml:"env"`
  27. SyncedEnv []*SyncedEnvSection `yaml:"synced_env"`
  28. Apps map[string]*Service `yaml:"apps" validate:"required_without=Applications Services"`
  29. Services map[string]*Service `yaml:"services" validate:"required_without=Applications Apps"`
  30. Release *Service `yaml:"release"`
  31. }
  32. type Application struct {
  33. Services map[string]*Service `yaml:"services" validate:"required"`
  34. Build *Build `yaml:"build"`
  35. Env map[string]string `yaml:"env"`
  36. Release *Service `yaml:"release"`
  37. }
  38. type Build struct {
  39. Context *string `yaml:"context" validate:"dir"`
  40. Method *string `yaml:"method" validate:"required,oneof=pack docker registry"`
  41. Builder *string `yaml:"builder" validate:"required_if=Method pack"`
  42. Buildpacks []*string `yaml:"buildpacks"`
  43. Dockerfile *string `yaml:"dockerfile" validate:"required_if=Method docker"`
  44. Image *string `yaml:"image" validate:"required_if=Method registry"`
  45. }
  46. type Service struct {
  47. Run *string `yaml:"run"`
  48. Config map[string]interface{} `yaml:"config"`
  49. Type *string `yaml:"type" validate:"required, oneof=web worker job"`
  50. }
  51. type SyncedEnvSection struct {
  52. Name string `json:"name" yaml:"name"`
  53. Version uint `json:"version" yaml:"version"`
  54. Keys []SyncedEnvSectionKey `json:"keys" yaml:"keys"`
  55. }
  56. type SyncedEnvSectionKey struct {
  57. Name string `json:"name" yaml:"name"`
  58. Secret bool `json:"secret" yaml:"secret"`
  59. }
  60. type SubdomainCreateOpts struct {
  61. k8sAgent *kubernetes.Agent
  62. dnsRepo repository.DNSRecordRepository
  63. dnsClient *dns.Client
  64. appRootDomain string
  65. stackName string
  66. }
  67. type ParseConf struct {
  68. // PorterAppName is the name of the porter app
  69. PorterAppName string
  70. // PorterYaml is the raw porter yaml which is used to build the values + chart for helm upgrade
  71. PorterYaml []byte
  72. // ImageInfo contains the repository and tag of the image to use for the helm upgrade. Kept separate from the PorterYaml because the image info
  73. // is stored in the 'global' key of the values, which is not part of the porter yaml
  74. ImageInfo types.ImageInfo
  75. // ServerConfig is the server conf, used to find the default helm repo
  76. ServerConfig *config.Config
  77. // ProjectID
  78. ProjectID uint
  79. // UserUpdate used for synced env groups
  80. UserUpdate bool
  81. // EnvGroups used for synced env groups
  82. EnvGroups []string
  83. // EnvironmentGroups are used for syncing environment groups using ConfigMaps and Secrets from porter-env-groups namespace. This should be used instead of EnvGroups
  84. EnvironmentGroups []string
  85. // Namespace used for synced env groups
  86. Namespace string
  87. // ExistingHelmValues is the existing values for the helm release, if it exists
  88. ExistingHelmValues map[string]interface{}
  89. // ExistingChartDependencies is the existing dependencies for the helm release, if it exists
  90. ExistingChartDependencies []*chart.Dependency
  91. // SubdomainCreateOpts contains the necessary information to create a subdomain if necessary
  92. SubdomainCreateOpts SubdomainCreateOpts
  93. // InjectLauncherToStartCommand is a flag to determine whether to prepend the launcher to the start command
  94. InjectLauncherToStartCommand bool
  95. // ShouldValidateHelmValues is a flag to determine whether to validate helm values
  96. ShouldValidateHelmValues bool
  97. // FullHelmValues if provided, override anything specified in porter.yaml. Used as an escape hatch for support
  98. FullHelmValues string
  99. // AddCustomNodeSelector is a flag to determine whether to add porter.run/workload-kind: application to the nodeselector attribute of the helm values
  100. AddCustomNodeSelector bool
  101. // RemoveDeletedServices is a flag to determine whether to remove values and dependencies for services that are not defined in the porter.yaml
  102. RemoveDeletedServices bool
  103. }
  104. func parse(ctx context.Context, conf ParseConf) (*chart.Chart, map[string]interface{}, map[string]interface{}, error) {
  105. ctx, span := telemetry.NewSpan(ctx, "parse-porter-yaml")
  106. defer span.End()
  107. parsed := &PorterStackYAML{}
  108. if err := yaml.Unmarshal(conf.PorterYaml, parsed); err != nil {
  109. err = telemetry.Error(ctx, span, err, "error parsing porter.yaml")
  110. return nil, nil, nil, err
  111. }
  112. if conf.FullHelmValues != "" {
  113. parsedHelmValues, err := convertHelmValuesToPorterYaml(conf.FullHelmValues)
  114. if err != nil {
  115. err = telemetry.Error(ctx, span, err, "error parsing raw helm values")
  116. return nil, nil, nil, err
  117. }
  118. if parsed.Release != nil && parsed.Release.Run != nil {
  119. parsedHelmValues.Release = parsed.Release
  120. }
  121. parsed = parsedHelmValues
  122. }
  123. synced_env := make([]*SyncedEnvSection, 0)
  124. for i := range conf.EnvGroups {
  125. cm, _, err := conf.SubdomainCreateOpts.k8sAgent.GetLatestVersionedConfigMap(conf.EnvGroups[i], conf.Namespace)
  126. if err != nil {
  127. err = telemetry.Error(ctx, span, err, "error getting latest versioned config map")
  128. return nil, nil, nil, err
  129. }
  130. versionStr, ok := cm.ObjectMeta.Labels["version"]
  131. if !ok {
  132. err = telemetry.Error(ctx, span, nil, "error extracting version from config map")
  133. return nil, nil, nil, err
  134. }
  135. versionInt, err := strconv.Atoi(versionStr)
  136. if err != nil {
  137. err = telemetry.Error(ctx, span, err, "error converting version to int")
  138. return nil, nil, nil, err
  139. }
  140. version := uint(versionInt)
  141. newSection := &SyncedEnvSection{
  142. Name: conf.EnvGroups[i],
  143. Version: version,
  144. }
  145. newSectionKeys := make([]SyncedEnvSectionKey, 0)
  146. for key, val := range cm.Data {
  147. newSectionKeys = append(newSectionKeys, SyncedEnvSectionKey{
  148. Name: key,
  149. Secret: strings.Contains(val, "PORTERSECRET"),
  150. })
  151. }
  152. newSection.Keys = newSectionKeys
  153. synced_env = append(synced_env, newSection)
  154. }
  155. if parsed.Apps != nil && parsed.Services != nil {
  156. err := telemetry.Error(ctx, span, nil, "'apps' and 'services' are synonymous but both were defined")
  157. return nil, nil, nil, err
  158. }
  159. var services map[string]*Service
  160. if parsed.Apps != nil {
  161. services = parsed.Apps
  162. }
  163. if parsed.Services != nil {
  164. services = parsed.Services
  165. }
  166. for serviceName := range services {
  167. services[serviceName] = addLabelsToService(services[serviceName], conf.EnvironmentGroups, porter_app.LabelKey_PorterApplication)
  168. }
  169. application := &Application{
  170. Env: parsed.Env,
  171. Services: services,
  172. Build: parsed.Build,
  173. Release: parsed.Release,
  174. }
  175. values, err := buildUmbrellaChartValues(ctx, application, synced_env, conf.ImageInfo, conf.ExistingHelmValues, conf.SubdomainCreateOpts, conf.InjectLauncherToStartCommand, conf.ShouldValidateHelmValues, conf.UserUpdate, conf.Namespace, conf.AddCustomNodeSelector, conf.RemoveDeletedServices)
  176. if err != nil {
  177. err = telemetry.Error(ctx, span, err, "error building values")
  178. return nil, nil, nil, err
  179. }
  180. convertedValues, ok := convertMap(values).(map[string]interface{})
  181. if !ok {
  182. err = telemetry.Error(ctx, span, nil, "error converting values")
  183. return nil, nil, nil, err
  184. }
  185. umbrellaChart, err := buildUmbrellaChart(application, conf.ServerConfig, conf.ProjectID, conf.ExistingChartDependencies, conf.RemoveDeletedServices)
  186. if err != nil {
  187. err = telemetry.Error(ctx, span, err, "error building umbrella chart")
  188. return nil, nil, nil, err
  189. }
  190. // return the parsed release values for the release job chart, if they exist
  191. var preDeployJobValues map[string]interface{}
  192. if application.Release != nil && application.Release.Run != nil {
  193. application.Release = addLabelsToService(application.Release, conf.EnvironmentGroups, porter_app.LabelKey_PorterApplicationPreDeploy)
  194. preDeployJobValues = buildPreDeployJobChartValues(application.Release, application.Env, synced_env, conf.ImageInfo, conf.InjectLauncherToStartCommand, conf.ExistingHelmValues, porterAppUtils.PredeployJobNameFromPorterAppName(conf.PorterAppName), conf.UserUpdate, conf.AddCustomNodeSelector)
  195. }
  196. return umbrellaChart, convertedValues, preDeployJobValues, nil
  197. }
  198. func buildUmbrellaChartValues(
  199. ctx context.Context,
  200. application *Application,
  201. syncedEnv []*SyncedEnvSection,
  202. imageInfo types.ImageInfo,
  203. existingValues map[string]interface{},
  204. opts SubdomainCreateOpts,
  205. injectLauncher bool,
  206. shouldValidateHelmValues bool,
  207. userUpdate bool,
  208. namespace string,
  209. addCustomNodeSelector bool,
  210. removeDeletedValues bool,
  211. ) (map[string]interface{}, error) {
  212. values := make(map[string]interface{})
  213. if application.Services == nil {
  214. if existingValues == nil {
  215. return nil, fmt.Errorf("porter.yaml must contain at least one service, or pre-deploy must exist and have values")
  216. }
  217. }
  218. for name, service := range application.Services {
  219. serviceType := getType(name, service)
  220. defaultValues := getDefaultValues(service, application.Env, syncedEnv, serviceType, existingValues, name, userUpdate, addCustomNodeSelector)
  221. convertedConfig := convertMap(service.Config).(map[string]interface{})
  222. helm_values := utils.DeepCoalesceValues(defaultValues, convertedConfig)
  223. // required to identify the chart type because of https://github.com/helm/helm/issues/9214
  224. helmName := getHelmName(name, serviceType)
  225. if existingValues != nil {
  226. if existingValues[helmName] != nil {
  227. existingValuesMap := existingValues[helmName].(map[string]interface{})
  228. if removeDeletedValues {
  229. // strip the env variables before coalescing
  230. if existingValuesMap["container"] != nil {
  231. containerMap := existingValuesMap["container"].(map[string]interface{})
  232. if containerMap["env"] != nil {
  233. envMap := containerMap["env"].(map[string]interface{})
  234. if envMap["normal"] != nil {
  235. envMap["normal"] = make(map[string]interface{})
  236. }
  237. if envMap["synced"] != nil {
  238. envMap["synced"] = make([]map[string]interface{}, 0)
  239. }
  240. containerMap["env"] = envMap
  241. }
  242. existingValuesMap["container"] = containerMap
  243. }
  244. }
  245. helm_values = utils.DeepCoalesceValues(existingValuesMap, helm_values)
  246. }
  247. }
  248. validateErr := validateHelmValues(helm_values, shouldValidateHelmValues, serviceType)
  249. if validateErr != "" {
  250. return nil, fmt.Errorf("error validating service \"%s\": %s", name, validateErr)
  251. }
  252. err := syncEnvironmentGroupToNamespaceIfLabelsExist(ctx, opts.k8sAgent, service, namespace)
  253. if err != nil {
  254. return nil, fmt.Errorf("error syncing environment group to namespace: %w", err)
  255. }
  256. err = createSubdomainIfRequired(helm_values, opts) // modifies helm_values to add subdomains if necessary
  257. if err != nil {
  258. return nil, err
  259. }
  260. // just in case this slips by
  261. if serviceType == "web" {
  262. if helm_values["ingress"] == nil {
  263. helm_values["ingress"] = map[string]interface{}{
  264. "enabled": false,
  265. }
  266. }
  267. }
  268. values[helmName] = helm_values
  269. }
  270. if !removeDeletedValues {
  271. // add back in the existing services that were not overwritten
  272. for k, v := range existingValues {
  273. if values[k] == nil {
  274. values[k] = v
  275. }
  276. }
  277. }
  278. // prepend launcher to all start commands if we need to
  279. for _, v := range values {
  280. if serviceValues, ok := v.(map[string]interface{}); ok {
  281. if serviceValues["container"] != nil {
  282. containerMap := serviceValues["container"].(map[string]interface{})
  283. if containerMap["command"] != nil {
  284. command := containerMap["command"].(string)
  285. if injectLauncher && !strings.HasPrefix(command, "launcher") && !strings.HasPrefix(command, "/cnb/lifecycle/launcher") {
  286. containerMap["command"] = fmt.Sprintf("/cnb/lifecycle/launcher %s", command)
  287. }
  288. }
  289. }
  290. }
  291. }
  292. values["global"] = map[string]interface{}{
  293. "image": map[string]interface{}{
  294. "repository": imageInfo.Repository,
  295. "tag": imageInfo.Tag,
  296. },
  297. }
  298. return values, nil
  299. }
  300. // syncEnvironmentGroupToNamespaceIfLabelsExist will sync the latest version of the environment group to the target namespace if the service has the appropriate label.
  301. func syncEnvironmentGroupToNamespaceIfLabelsExist(ctx context.Context, agent *kubernetes.Agent, service *Service, targetNamespace string) error {
  302. var linkedGroupNames string
  303. // patchwork because we are not consistent with the type of labels
  304. if labels, ok := service.Config["labels"].(map[string]any); ok {
  305. if linkedGroup, ok := labels[environment_groups.LabelKey_LinkedEnvironmentGroup].(string); ok {
  306. linkedGroupNames = linkedGroup
  307. }
  308. }
  309. if labels, ok := service.Config["labels"].(map[string]string); ok {
  310. if linkedGroup, ok := labels[environment_groups.LabelKey_LinkedEnvironmentGroup]; ok {
  311. linkedGroupNames = linkedGroup
  312. }
  313. }
  314. service.Config["configMapRefs"] = []string{}
  315. service.Config["secretRefs"] = []string{}
  316. for _, linkedGroupName := range strings.Split(linkedGroupNames, ".") {
  317. inp := environment_groups.SyncLatestVersionToNamespaceInput{
  318. BaseEnvironmentGroupName: linkedGroupName,
  319. TargetNamespace: targetNamespace,
  320. }
  321. syncedEnvironment, err := environment_groups.SyncLatestVersionToNamespace(ctx, agent, inp, nil)
  322. if err != nil {
  323. return fmt.Errorf("error syncing environment group: %w", err)
  324. }
  325. if syncedEnvironment.EnvironmentGroupVersionedName != "" {
  326. if service.Config["configMapRefs"] == nil {
  327. service.Config["configMapRefs"] = []string{}
  328. }
  329. if service.Config["secretRefs"] == nil {
  330. service.Config["secretRefs"] = []string{}
  331. }
  332. switch service.Config["configMapRefs"].(type) {
  333. case []string:
  334. service.Config["configMapRefs"] = append(service.Config["configMapRefs"].([]string), syncedEnvironment.EnvironmentGroupVersionedName)
  335. case []any:
  336. service.Config["configMapRefs"] = append(service.Config["configMapRefs"].([]any), syncedEnvironment.EnvironmentGroupVersionedName)
  337. }
  338. switch service.Config["secretRefs"].(type) {
  339. case []string:
  340. service.Config["secretRefs"] = append(service.Config["secretRefs"].([]string), syncedEnvironment.EnvironmentGroupVersionedName)
  341. case []any:
  342. service.Config["secretRefs"] = append(service.Config["secretRefs"].([]any), syncedEnvironment.EnvironmentGroupVersionedName)
  343. }
  344. }
  345. }
  346. return nil
  347. }
  348. // we can add to this function up later or use an alternative
  349. func validateHelmValues(values map[string]interface{}, shouldValidateHelmValues bool, appType string) string {
  350. if shouldValidateHelmValues {
  351. // validate port for web services
  352. if appType == "web" {
  353. containerMap, err := getNestedMap(values, "container")
  354. if err != nil {
  355. return "error checking port: misformatted values"
  356. } else {
  357. portVal, portExists := containerMap["port"]
  358. if portExists {
  359. portStr, pOK := portVal.(string)
  360. if !pOK {
  361. return "error checking port: no port in container"
  362. }
  363. port, err := strconv.Atoi(portStr)
  364. if err != nil || port < 1024 || port > 65535 {
  365. return "port must be a number between 1024 and 65535"
  366. }
  367. } else {
  368. return "port must be specified for web services"
  369. }
  370. }
  371. }
  372. }
  373. return ""
  374. }
  375. func buildPreDeployJobChartValues(release *Service, env map[string]string, synced_env []*SyncedEnvSection, imageInfo types.ImageInfo, injectLauncher bool, existingValues map[string]interface{}, name string, userUpdate bool, addCustomNodeSelector bool) map[string]interface{} {
  376. defaultValues := getDefaultValues(release, env, synced_env, "job", existingValues, name+"-r", userUpdate, addCustomNodeSelector)
  377. convertedConfig := convertMap(release.Config).(map[string]interface{})
  378. helm_values := utils.DeepCoalesceValues(defaultValues, convertedConfig)
  379. if imageInfo.Repository != "" && imageInfo.Tag != "" {
  380. helm_values["image"] = map[string]interface{}{
  381. "repository": imageInfo.Repository,
  382. "tag": imageInfo.Tag,
  383. }
  384. }
  385. // prepend launcher if we need to
  386. if helm_values["container"] != nil {
  387. containerMap := helm_values["container"].(map[string]interface{})
  388. if containerMap["command"] != nil {
  389. command := containerMap["command"].(string)
  390. if injectLauncher && !strings.HasPrefix(command, "launcher") && !strings.HasPrefix(command, "/cnb/lifecycle/launcher") {
  391. containerMap["command"] = fmt.Sprintf("/cnb/lifecycle/launcher %s", command)
  392. }
  393. }
  394. }
  395. return helm_values
  396. }
  397. func getType(name string, service *Service) string {
  398. if service.Type != nil {
  399. return *service.Type
  400. }
  401. if strings.Contains(name, "web") {
  402. return "web"
  403. }
  404. if strings.Contains(name, "job") {
  405. return "job"
  406. }
  407. return "worker"
  408. }
  409. func getDefaultValues(service *Service, env map[string]string, synced_env []*SyncedEnvSection, appType string, existingValues map[string]interface{}, name string, userUpdate bool, addCustomNodeSelector bool) map[string]interface{} {
  410. var defaultValues map[string]interface{}
  411. var runCommand string
  412. if service.Run != nil {
  413. runCommand = *service.Run
  414. }
  415. var syncedEnvs []map[string]interface{}
  416. envConf, err := getStacksNestedMap(existingValues, name+"-"+appType, "container", "env")
  417. if !userUpdate && err == nil {
  418. syncedEnvs = envConf
  419. } else {
  420. syncedEnvs = deconstructSyncedEnvs(synced_env, env)
  421. }
  422. defaultValues = map[string]interface{}{
  423. "container": map[string]interface{}{
  424. "command": runCommand,
  425. "env": map[string]interface{}{
  426. "normal": CopyEnv(env),
  427. "synced": syncedEnvs,
  428. },
  429. },
  430. "nodeSelector": map[string]interface{}{},
  431. }
  432. if addCustomNodeSelector {
  433. defaultValues["nodeSelector"] = map[string]interface{}{
  434. "porter.run/workload-kind": "application",
  435. }
  436. }
  437. return defaultValues
  438. }
  439. func deconstructSyncedEnvs(synced_env []*SyncedEnvSection, env map[string]string) []map[string]interface{} {
  440. synced := make([]map[string]interface{}, 0)
  441. for _, group := range synced_env {
  442. keys := make([]map[string]interface{}, 0)
  443. for _, key := range group.Keys {
  444. if _, exists := env[key.Name]; !exists {
  445. // Only include keys not present in env
  446. keys = append(keys, map[string]interface{}{
  447. "name": key.Name,
  448. "secret": key.Secret,
  449. })
  450. }
  451. }
  452. syncedGroup := map[string]interface{}{
  453. "keys": keys,
  454. "name": group.Name,
  455. "version": group.Version,
  456. }
  457. synced = append(synced, syncedGroup)
  458. }
  459. return synced
  460. }
  461. func buildUmbrellaChart(application *Application, config *config.Config, projectID uint, existingDependencies []*chart.Dependency, removeDeletedDependencies bool) (*chart.Chart, error) {
  462. deps := make([]*chart.Dependency, 0)
  463. for alias, service := range application.Services {
  464. var serviceType string
  465. if existingDependencies != nil {
  466. for _, dep := range existingDependencies {
  467. // this condition checks that the dependency is of the form <alias>-web or <alias>-wkr or <alias>-job, meaning it already exists in the chart
  468. if strings.HasPrefix(dep.Alias, fmt.Sprintf("%s-", alias)) && (strings.HasSuffix(dep.Alias, "-web") || strings.HasSuffix(dep.Alias, "-wkr") || strings.HasSuffix(dep.Alias, "-job")) {
  469. serviceType = getChartTypeFromHelmName(dep.Alias)
  470. if serviceType == "" {
  471. return nil, fmt.Errorf("unable to determine type of existing dependency")
  472. }
  473. }
  474. }
  475. // this is a new app, so we need to get the type from the app name or type
  476. if serviceType == "" {
  477. serviceType = getType(alias, service)
  478. }
  479. } else {
  480. serviceType = getType(alias, service)
  481. }
  482. selectedRepo := config.ServerConf.DefaultApplicationHelmRepoURL
  483. selectedVersion, err := getLatestTemplateVersion(serviceType, config, projectID)
  484. if err != nil {
  485. return nil, err
  486. }
  487. helmName := getHelmName(alias, serviceType)
  488. deps = append(deps, &chart.Dependency{
  489. Name: serviceType,
  490. Alias: helmName,
  491. Version: selectedVersion,
  492. Repository: selectedRepo,
  493. })
  494. }
  495. if !removeDeletedDependencies {
  496. // add in the existing dependencies that were not overwritten
  497. for _, dep := range existingDependencies {
  498. if !dependencyExists(deps, dep) {
  499. // have to repair the dependency name because of https://github.com/helm/helm/issues/9214
  500. if strings.HasSuffix(dep.Name, "-web") || strings.HasSuffix(dep.Name, "-wkr") || strings.HasSuffix(dep.Name, "-job") {
  501. dep.Name = getChartTypeFromHelmName(dep.Name)
  502. if dep.Name == "" {
  503. return nil, fmt.Errorf("unable to determine type of existing dependency")
  504. }
  505. version, err := getLatestTemplateVersion(dep.Name, config, projectID)
  506. if err != nil {
  507. return nil, err
  508. }
  509. dep.Version = version
  510. }
  511. deps = append(deps, dep)
  512. }
  513. }
  514. }
  515. chart, err := createChartFromDependencies(deps)
  516. if err != nil {
  517. return nil, err
  518. }
  519. return chart, nil
  520. }
  521. func dependencyExists(deps []*chart.Dependency, dep *chart.Dependency) bool {
  522. for _, d := range deps {
  523. if d.Alias == dep.Alias {
  524. return true
  525. }
  526. }
  527. return false
  528. }
  529. func createChartFromDependencies(deps []*chart.Dependency) (*chart.Chart, error) {
  530. metadata := &chart.Metadata{
  531. Name: "umbrella",
  532. Description: "Web application that is exposed to external traffic.",
  533. Version: "0.96.0",
  534. APIVersion: "v2",
  535. Home: "https://getporter.dev/",
  536. Icon: "https://user-images.githubusercontent.com/65516095/111255214-07d3da80-85ed-11eb-99e2-fddcbdb99bdb.png",
  537. Keywords: []string{
  538. "porter",
  539. "application",
  540. "service",
  541. "umbrella",
  542. },
  543. Type: "application",
  544. Dependencies: deps,
  545. }
  546. // create a new chart object with the metadata
  547. c := &chart.Chart{
  548. Metadata: metadata,
  549. }
  550. return c, nil
  551. }
  552. func getLatestTemplateVersion(templateName string, config *config.Config, projectID uint) (string, error) {
  553. repoIndex, err := loader.LoadRepoIndexPublic(config.ServerConf.DefaultApplicationHelmRepoURL)
  554. if err != nil {
  555. return "", fmt.Errorf("%s: %w", "unable to load porter chart repo", err)
  556. }
  557. templates := loader.RepoIndexToPorterChartList(repoIndex, config.ServerConf.DefaultApplicationHelmRepoURL)
  558. if err != nil {
  559. return "", fmt.Errorf("%s: %w", "unable to load porter chart list", err)
  560. }
  561. var version string
  562. // find the matching template name
  563. for _, template := range templates {
  564. if templateName == template.Name {
  565. version = template.Versions[0]
  566. break
  567. }
  568. }
  569. if version == "" {
  570. return "", fmt.Errorf("matching template version not found")
  571. }
  572. return version, nil
  573. }
  574. func convertMap(m interface{}) interface{} {
  575. switch m := m.(type) {
  576. case map[string]interface{}:
  577. for k, v := range m {
  578. m[k] = convertMap(v)
  579. }
  580. case map[string]string:
  581. result := map[string]interface{}{}
  582. for k, v := range m {
  583. result[k] = v
  584. }
  585. return result
  586. case map[interface{}]interface{}:
  587. result := map[string]interface{}{}
  588. for k, v := range m {
  589. result[k.(string)] = convertMap(v)
  590. }
  591. return result
  592. case []interface{}:
  593. for i, v := range m {
  594. m[i] = convertMap(v)
  595. }
  596. }
  597. return m
  598. }
  599. func CopyEnv(env map[string]string) map[string]interface{} {
  600. envCopy := make(map[string]interface{})
  601. if env == nil {
  602. return envCopy
  603. }
  604. for k, v := range env {
  605. if k == "" || v == "" {
  606. continue
  607. }
  608. envCopy[k] = v
  609. }
  610. return envCopy
  611. }
  612. func createSubdomainIfRequired(
  613. mergedValues map[string]interface{},
  614. opts SubdomainCreateOpts,
  615. ) error {
  616. // look for ingress.enabled and no custom domains set
  617. ingressMap, err := getNestedMap(mergedValues, "ingress")
  618. if err == nil {
  619. enabledVal, enabledExists := ingressMap["enabled"]
  620. if enabledExists {
  621. enabled, eOK := enabledVal.(bool)
  622. if eOK && enabled {
  623. // if custom domain, we don't need to create a subdomain
  624. customDomVal, customDomExists := ingressMap["custom_domain"]
  625. if customDomExists {
  626. customDomain, cOK := customDomVal.(bool)
  627. if cOK && customDomain {
  628. return nil
  629. }
  630. }
  631. // subdomain already exists, no need to create one
  632. if porterHosts, ok := ingressMap["porter_hosts"].([]interface{}); ok && len(porterHosts) > 0 {
  633. return nil
  634. }
  635. // in the case of ingress enabled but no custom domain, create subdomain
  636. dnsRecord, err := createDNSRecord(opts)
  637. if err != nil {
  638. return fmt.Errorf("error creating subdomain: %s", err.Error())
  639. }
  640. subdomain := dnsRecord.ExternalURL
  641. if ingressVal, ok := mergedValues["ingress"]; !ok {
  642. mergedValues["ingress"] = map[string]interface{}{
  643. "porter_hosts": []string{
  644. subdomain,
  645. },
  646. }
  647. } else {
  648. ingressValMap := ingressVal.(map[string]interface{})
  649. ingressValMap["porter_hosts"] = []string{
  650. subdomain,
  651. }
  652. }
  653. }
  654. }
  655. }
  656. return nil
  657. }
  658. func createDNSRecord(opts SubdomainCreateOpts) (*types.DNSRecord, error) {
  659. if opts.dnsClient == nil {
  660. return nil, fmt.Errorf("cannot create subdomain because dns client is nil")
  661. }
  662. endpoint, found, err := domain.GetNGINXIngressServiceIP(opts.k8sAgent.Clientset)
  663. if err != nil {
  664. return nil, err
  665. }
  666. if !found {
  667. return nil, fmt.Errorf("target cluster does not have nginx ingress")
  668. }
  669. createDomain := domain.CreateDNSRecordConfig{
  670. ReleaseName: opts.stackName,
  671. RootDomain: opts.appRootDomain,
  672. Endpoint: endpoint,
  673. }
  674. record := createDomain.NewDNSRecordForEndpoint()
  675. record, err = opts.dnsRepo.CreateDNSRecord(record)
  676. if err != nil {
  677. return nil, err
  678. }
  679. _record := domain.DNSRecord(*record)
  680. err = _record.CreateDomain(opts.dnsClient)
  681. if err != nil {
  682. return nil, err
  683. }
  684. return record.ToDNSRecordType(), nil
  685. }
  686. func getNestedMap(obj map[string]interface{}, fields ...string) (map[string]interface{}, error) {
  687. var res map[string]interface{}
  688. curr := obj
  689. for _, field := range fields {
  690. objField, ok := curr[field]
  691. if !ok {
  692. return nil, fmt.Errorf("%s not found", field)
  693. }
  694. res, ok = objField.(map[string]interface{})
  695. if !ok {
  696. return nil, fmt.Errorf("%s is not a nested object", field)
  697. }
  698. curr = res
  699. }
  700. return res, nil
  701. }
  702. func getHelmName(alias string, t string) string {
  703. var suffix string
  704. if t == "web" {
  705. suffix = "-web"
  706. } else if t == "worker" {
  707. suffix = "-wkr"
  708. } else if t == "job" {
  709. suffix = "-job"
  710. }
  711. return fmt.Sprintf("%s%s", alias, suffix)
  712. }
  713. func getChartTypeFromHelmName(name string) string {
  714. if strings.HasSuffix(name, "-web") {
  715. return "web"
  716. } else if strings.HasSuffix(name, "-wkr") {
  717. return "worker"
  718. } else if strings.HasSuffix(name, "-job") {
  719. return "job"
  720. }
  721. return ""
  722. }
  723. func getServiceNameAndTypeFromHelmName(name string) (string, string) {
  724. if strings.HasSuffix(name, "-web") {
  725. return strings.TrimSuffix(name, "-web"), "web"
  726. } else if strings.HasSuffix(name, "-wkr") {
  727. return strings.TrimSuffix(name, "-wkr"), "worker"
  728. } else if strings.HasSuffix(name, "-job") {
  729. return strings.TrimSuffix(name, "-job"), "job"
  730. }
  731. return "", ""
  732. }
  733. func attemptToGetImageInfoFromRelease(values map[string]interface{}) types.ImageInfo {
  734. imageInfo := types.ImageInfo{}
  735. if values == nil {
  736. return imageInfo
  737. }
  738. globalImage, err := getNestedMap(values, "global", "image")
  739. if err != nil {
  740. return imageInfo
  741. }
  742. repoVal, okRepo := globalImage["repository"]
  743. tagVal, okTag := globalImage["tag"]
  744. if okRepo && okTag {
  745. imageInfo.Repository = repoVal.(string)
  746. imageInfo.Tag = tagVal.(string)
  747. }
  748. return imageInfo
  749. }
  750. func attemptToGetImageInfoFromFullHelmValues(fullHelmValues string) (types.ImageInfo, error) {
  751. imageInfo := types.ImageInfo{}
  752. var values map[string]interface{}
  753. err := yaml.Unmarshal([]byte(fullHelmValues), &values)
  754. if err != nil {
  755. return imageInfo, fmt.Errorf("error unmarshaling full helm values to read image info: %w", err)
  756. }
  757. convertedValues := convertMap(values).(map[string]interface{})
  758. return attemptToGetImageInfoFromRelease(convertedValues), nil
  759. }
  760. func getStacksNestedMap(obj map[string]interface{}, fields ...string) ([]map[string]interface{}, error) {
  761. var res map[string]interface{}
  762. curr := obj
  763. for _, field := range fields {
  764. objField, ok := curr[field]
  765. if !ok {
  766. return nil, fmt.Errorf("%s not found", field)
  767. }
  768. res, ok = objField.(map[string]interface{})
  769. if !ok {
  770. return nil, fmt.Errorf("%s is not a nested object", field)
  771. }
  772. curr = res
  773. }
  774. syncedInterface, ok := curr["synced"]
  775. if !ok {
  776. return nil, fmt.Errorf("synced not found")
  777. }
  778. synced, ok := syncedInterface.([]interface{})
  779. if !ok {
  780. return nil, fmt.Errorf("synced is not a slice of interface{}")
  781. }
  782. result := make([]map[string]interface{}, len(synced))
  783. for i, v := range synced {
  784. mapElement, ok := v.(map[string]interface{})
  785. if !ok {
  786. return nil, fmt.Errorf("element %d in synced is not a map[string]interface{}", i)
  787. }
  788. result[i] = mapElement
  789. }
  790. return result, nil
  791. }
  792. func convertHelmValuesToPorterYaml(helmValues string) (*PorterStackYAML, error) {
  793. var values map[string]interface{}
  794. err := yaml.Unmarshal([]byte(helmValues), &values)
  795. if err != nil {
  796. return nil, err
  797. }
  798. services := make(map[string]*Service)
  799. for k, v := range values {
  800. if k == "global" {
  801. continue
  802. }
  803. serviceName, serviceType := getServiceNameAndTypeFromHelmName(k)
  804. if serviceName == "" {
  805. return nil, fmt.Errorf("invalid service key: %s. make sure that service key ends in either -web, -wkr, or -job", k)
  806. }
  807. config := convertMap(v).(map[string]interface{})
  808. var runCommand string
  809. if config["container"] != nil {
  810. containerMap := config["container"].(map[string]interface{})
  811. if containerMap["command"] != nil {
  812. runCommand = containerMap["command"].(string)
  813. }
  814. }
  815. services[serviceName] = &Service{
  816. Run: &runCommand,
  817. Config: config,
  818. Type: &serviceType,
  819. }
  820. }
  821. return &PorterStackYAML{
  822. Services: services,
  823. }, nil
  824. }
  825. // addLabelsToService always adds the default label to the service, and if envGroups is not empty, it adds the corresponding environment group label as well.
  826. func addLabelsToService(service *Service, envGroups []string, defaultLabelKey string) *Service {
  827. if _, ok := service.Config["labels"]; !ok {
  828. service.Config["labels"] = make(map[string]string)
  829. }
  830. if len(envGroups) != 0 {
  831. // delete the env group label so we can replace it
  832. if _, ok := service.Config["labels"].(map[string]any); ok {
  833. delete(service.Config["labels"].(map[string]any), environment_groups.LabelKey_LinkedEnvironmentGroup)
  834. }
  835. }
  836. switch service.Config["labels"].(type) {
  837. case map[string]string:
  838. service.Config["labels"].(map[string]string)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  839. if len(envGroups) != 0 {
  840. service.Config["labels"].(map[string]string)[environment_groups.LabelKey_LinkedEnvironmentGroup] = strings.Join(envGroups, ".")
  841. }
  842. case map[string]any:
  843. service.Config["labels"].(map[string]any)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  844. if len(envGroups) != 0 {
  845. service.Config["labels"].(map[string]any)[environment_groups.LabelKey_LinkedEnvironmentGroup] = strings.Join(envGroups, ".")
  846. }
  847. case any:
  848. if val, ok := service.Config["labels"].(string); ok {
  849. if val == "" {
  850. service.Config["labels"] = map[string]string{
  851. defaultLabelKey: porter_app.LabelValue_PorterApplication,
  852. }
  853. if len(envGroups) != 0 {
  854. service.Config["labels"].(map[string]string)[environment_groups.LabelKey_LinkedEnvironmentGroup] = strings.Join(envGroups, ".")
  855. }
  856. }
  857. }
  858. }
  859. if _, ok := service.Config["podLabels"]; !ok {
  860. service.Config["podLabels"] = make(map[string]string)
  861. }
  862. switch service.Config["podLabels"].(type) {
  863. case map[string]string:
  864. service.Config["podLabels"].(map[string]string)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  865. case map[string]any:
  866. service.Config["podLabels"].(map[string]any)[defaultLabelKey] = porter_app.LabelValue_PorterApplication
  867. case any:
  868. if val, ok := service.Config["podLabels"].(string); ok {
  869. if val == "" {
  870. service.Config["podLabels"] = map[string]string{
  871. defaultLabelKey: porter_app.LabelValue_PorterApplication,
  872. }
  873. }
  874. }
  875. }
  876. return service
  877. }
  878. func getServiceDeploymentMetadataFromValues(values map[string]interface{}, status types.PorterAppEventStatus) map[string]types.ServiceDeploymentMetadata {
  879. serviceDeploymentMap := make(map[string]types.ServiceDeploymentMetadata)
  880. for key := range values {
  881. if key != "global" {
  882. serviceName, serviceType := getServiceNameAndTypeFromHelmName(key)
  883. externalURI := getServiceExternalURIFromServiceValues(values[key].(map[string]interface{}))
  884. // jobs don't technically have a deployment, so hardcode the deployment status to success
  885. serviceStatus := status
  886. if serviceType == "job" {
  887. serviceStatus = types.PorterAppEventStatus_Success
  888. }
  889. serviceDeploymentMap[serviceName] = types.ServiceDeploymentMetadata{
  890. ExternalURI: externalURI,
  891. Status: serviceStatus,
  892. Type: serviceType,
  893. }
  894. }
  895. }
  896. return serviceDeploymentMap
  897. }
  898. func getServiceExternalURIFromServiceValues(serviceValues map[string]interface{}) string {
  899. ingressMap, err := getNestedMap(serviceValues, "ingress")
  900. if err == nil {
  901. enabledVal, enabledExists := ingressMap["enabled"]
  902. if enabledExists {
  903. enabled, eOK := enabledVal.(bool)
  904. if eOK && enabled {
  905. customDomVal, customDomExists := ingressMap["custom_domain"]
  906. if customDomExists {
  907. customDomain, cOK := customDomVal.(bool)
  908. if cOK && customDomain {
  909. hostsExists, hostsExistsOK := ingressMap["hosts"]
  910. if hostsExistsOK {
  911. if hosts, hostsOK := hostsExists.([]interface{}); hostsOK && len(hosts) == 1 {
  912. return hosts[0].(string)
  913. }
  914. }
  915. }
  916. }
  917. if porterHosts, ok := ingressMap["porter_hosts"].([]interface{}); ok && len(porterHosts) == 1 {
  918. return porterHosts[0].(string)
  919. }
  920. }
  921. }
  922. }
  923. return ""
  924. }