app.go 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265
  1. package commands
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "os"
  8. "strings"
  9. "time"
  10. "github.com/fatih/color"
  11. api "github.com/porter-dev/porter/api/client"
  12. "github.com/porter-dev/porter/api/types"
  13. "github.com/porter-dev/porter/cli/cmd/config"
  14. "github.com/porter-dev/porter/cli/cmd/utils"
  15. v2 "github.com/porter-dev/porter/cli/cmd/v2"
  16. "github.com/spf13/cobra"
  17. batchv1 "k8s.io/api/batch/v1"
  18. v1 "k8s.io/api/core/v1"
  19. rbacv1 "k8s.io/api/rbac/v1"
  20. "k8s.io/apimachinery/pkg/api/resource"
  21. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  22. "k8s.io/apimachinery/pkg/fields"
  23. "k8s.io/apimachinery/pkg/watch"
  24. "k8s.io/kubectl/pkg/util/term"
  25. "k8s.io/apimachinery/pkg/runtime"
  26. "k8s.io/apimachinery/pkg/runtime/schema"
  27. "k8s.io/client-go/kubernetes"
  28. "k8s.io/client-go/rest"
  29. "k8s.io/client-go/tools/clientcmd"
  30. "k8s.io/client-go/tools/remotecommand"
  31. )
  32. var (
  33. appNamespace string
  34. appVerbose bool
  35. appExistingPod bool
  36. appInteractive bool
  37. appContainerName string
  38. appTag string
  39. appCpuMilli int
  40. appMemoryMi int
  41. )
  42. const (
  43. // CommandPrefix_CNB_LIFECYCLE_LAUNCHER is the prefix for the container start command if the image is built using heroku buildpacks
  44. CommandPrefix_CNB_LIFECYCLE_LAUNCHER = "/cnb/lifecycle/launcher"
  45. // CommandPrefix_LAUNCHER is a shortened form of the above
  46. CommandPrefix_LAUNCHER = "launcher"
  47. )
  48. func registerCommand_App(cliConf config.CLIConfig) *cobra.Command {
  49. appCmd := &cobra.Command{
  50. Use: "app",
  51. Short: "Runs a command for your application.",
  52. }
  53. // appRunCmd represents the "porter app run" subcommand
  54. appRunCmd := &cobra.Command{
  55. Use: "run [application] -- COMMAND [args...]",
  56. Args: cobra.MinimumNArgs(2),
  57. Short: "Runs a command inside a connected cluster container.",
  58. Run: func(cmd *cobra.Command, args []string) {
  59. err := checkLoginAndRunWithConfig(cmd, cliConf, args, appRun)
  60. if err != nil {
  61. os.Exit(1)
  62. }
  63. },
  64. }
  65. appRunFlags(appRunCmd)
  66. appCmd.AddCommand(appRunCmd)
  67. // appRunCleanupCmd represents the "porter app run cleanup" subcommand
  68. appRunCleanupCmd := &cobra.Command{
  69. Use: "cleanup",
  70. Args: cobra.NoArgs,
  71. Short: "Delete any lingering ephemeral pods that were created with \"porter app run\".",
  72. Run: func(cmd *cobra.Command, args []string) {
  73. err := checkLoginAndRunWithConfig(cmd, cliConf, args, appCleanup)
  74. if err != nil {
  75. os.Exit(1)
  76. }
  77. },
  78. }
  79. appRunCmd.AddCommand(appRunCleanupCmd)
  80. // appUpdateTagCmd represents the "porter app update-tag" subcommand
  81. appUpdateTagCmd := &cobra.Command{
  82. Use: "update-tag [application]",
  83. Args: cobra.MinimumNArgs(1),
  84. Short: "Updates the image tag for an application.",
  85. Run: func(cmd *cobra.Command, args []string) {
  86. err := checkLoginAndRunWithConfig(cmd, cliConf, args, appUpdateTag)
  87. if err != nil {
  88. os.Exit(1)
  89. }
  90. },
  91. }
  92. appUpdateTagCmd.PersistentFlags().StringVarP(
  93. &appTag,
  94. "tag",
  95. "t",
  96. "",
  97. "the specified tag to use, default is \"latest\"",
  98. )
  99. appCmd.AddCommand(appUpdateTagCmd)
  100. // appRollback represents the "porter app rollback" subcommand
  101. appRollbackCmd := &cobra.Command{
  102. Use: "rollback [application]",
  103. Args: cobra.MinimumNArgs(1),
  104. Short: "Rolls back an application to the last successful revision.",
  105. RunE: func(cmd *cobra.Command, args []string) error {
  106. return checkLoginAndRunWithConfig(cmd, cliConf, args, appRollback)
  107. },
  108. }
  109. appCmd.AddCommand(appRollbackCmd)
  110. return appCmd
  111. }
  112. func appRunFlags(appRunCmd *cobra.Command) {
  113. appRunCmd.PersistentFlags().BoolVarP(
  114. &appExistingPod,
  115. "existing_pod",
  116. "e",
  117. false,
  118. "whether to connect to an existing pod (default false)",
  119. )
  120. appRunCmd.PersistentFlags().BoolVarP(
  121. &appVerbose,
  122. "verbose",
  123. "v",
  124. false,
  125. "whether to print verbose output",
  126. )
  127. appRunCmd.PersistentFlags().BoolVar(
  128. &appInteractive,
  129. "interactive",
  130. false,
  131. "whether to run in interactive mode (default false)",
  132. )
  133. appRunCmd.PersistentFlags().IntVarP(
  134. &appCpuMilli,
  135. "cpu",
  136. "",
  137. 0,
  138. "cpu allocation in millicores (1000 millicores = 1 vCPU)",
  139. )
  140. appRunCmd.PersistentFlags().IntVarP(
  141. &appMemoryMi,
  142. "ram",
  143. "",
  144. 0,
  145. "ram allocation in Mi (1024 Mi = 1 GB)",
  146. )
  147. appRunCmd.PersistentFlags().StringVarP(
  148. &appContainerName,
  149. "container",
  150. "c",
  151. "",
  152. "name of the container inside pod to run the command in",
  153. )
  154. }
  155. func appRollback(ctx context.Context, _ *types.GetAuthenticatedUserResponse, client api.Client, cliConfig config.CLIConfig, _ config.FeatureFlags, _ *cobra.Command, args []string) error {
  156. project, err := client.GetProject(ctx, cliConfig.Project)
  157. if err != nil {
  158. return fmt.Errorf("could not retrieve project from Porter API. Please contact support@porter.run")
  159. }
  160. if !project.ValidateApplyV2 {
  161. return fmt.Errorf("rollback command is not enabled for this project")
  162. }
  163. appName := args[0]
  164. if appName == "" {
  165. return fmt.Errorf("app name must be specified")
  166. }
  167. err = v2.Rollback(ctx, v2.RollbackInput{
  168. CLIConfig: cliConfig,
  169. Client: client,
  170. AppName: appName,
  171. })
  172. if err != nil {
  173. return fmt.Errorf("failed to rollback app: %w", err)
  174. }
  175. return nil
  176. }
  177. func appRun(ctx context.Context, _ *types.GetAuthenticatedUserResponse, client api.Client, cliConfig config.CLIConfig, _ config.FeatureFlags, _ *cobra.Command, args []string) error {
  178. execArgs := args[1:]
  179. color.New(color.FgGreen).Println("Attempting to run", strings.Join(execArgs, " "), "for application", args[0])
  180. project, err := client.GetProject(ctx, cliConfig.Project)
  181. if err != nil {
  182. return fmt.Errorf("could not retrieve project from Porter API. Please contact support@porter.run")
  183. }
  184. var podsSimple []appPodSimple
  185. // updated exec args includes launcher command prepended if needed, otherwise it is the same as execArgs
  186. var updatedExecArgs []string
  187. if project.ValidateApplyV2 {
  188. podsSimple, updatedExecArgs, namespace, err = getPodsFromV2PorterYaml(ctx, execArgs, client, cliConfig, args[0])
  189. if err != nil {
  190. return err
  191. }
  192. appNamespace = namespace
  193. } else {
  194. appNamespace = fmt.Sprintf("porter-stack-%s", args[0])
  195. podsSimple, updatedExecArgs, err = getPodsFromV1PorterYaml(ctx, execArgs, client, cliConfig, args[0], appNamespace)
  196. if err != nil {
  197. return err
  198. }
  199. }
  200. // if length of pods is 0, throw error
  201. var selectedPod appPodSimple
  202. if len(podsSimple) == 0 {
  203. return fmt.Errorf("At least one pod must exist in this deployment.")
  204. } else if !appExistingPod || len(podsSimple) == 1 {
  205. selectedPod = podsSimple[0]
  206. } else {
  207. podNames := make([]string, 0)
  208. for _, podSimple := range podsSimple {
  209. podNames = append(podNames, podSimple.Name)
  210. }
  211. selectedPodName, err := utils.PromptSelect("Select the pod:", podNames)
  212. if err != nil {
  213. return err
  214. }
  215. // find selected pod
  216. for _, podSimple := range podsSimple {
  217. if selectedPodName == podSimple.Name {
  218. selectedPod = podSimple
  219. }
  220. }
  221. }
  222. var selectedContainerName string
  223. if len(selectedPod.ContainerNames) == 0 {
  224. return fmt.Errorf("At least one container must exist in the selected pod.")
  225. } else if len(selectedPod.ContainerNames) == 1 {
  226. if appContainerName != "" && appContainerName != selectedPod.ContainerNames[0] {
  227. return fmt.Errorf("provided container %s does not exist in pod %s", appContainerName, selectedPod.Name)
  228. }
  229. selectedContainerName = selectedPod.ContainerNames[0]
  230. }
  231. if appContainerName != "" && selectedContainerName == "" {
  232. // check if provided container name exists in the pod
  233. for _, name := range selectedPod.ContainerNames {
  234. if name == appContainerName {
  235. selectedContainerName = name
  236. break
  237. }
  238. }
  239. if selectedContainerName == "" {
  240. return fmt.Errorf("provided container %s does not exist in pod %s", appContainerName, selectedPod.Name)
  241. }
  242. }
  243. if selectedContainerName == "" {
  244. if !appInteractive {
  245. return fmt.Errorf("container name must be specified using the --container flag when not using interactive mode")
  246. }
  247. selectedContainer, err := utils.PromptSelect("Select the container:", selectedPod.ContainerNames)
  248. if err != nil {
  249. return err
  250. }
  251. selectedContainerName = selectedContainer
  252. }
  253. config := &AppPorterRunSharedConfig{
  254. Client: client,
  255. CLIConfig: cliConfig,
  256. }
  257. err = config.setSharedConfig(ctx)
  258. if err != nil {
  259. return fmt.Errorf("Could not retrieve kube credentials: %s", err.Error())
  260. }
  261. imageName, err := getImageNameFromPod(ctx, config.Clientset, appNamespace, selectedPod.Name, selectedContainerName)
  262. if err != nil {
  263. return err
  264. }
  265. if appExistingPod {
  266. _, _ = color.New(color.FgGreen).Printf("Connecting to existing pod which is running an image named: %s\n", imageName)
  267. return appExecuteRun(config, appNamespace, selectedPod.Name, selectedContainerName, updatedExecArgs)
  268. }
  269. _, _ = color.New(color.FgGreen).Println("Creating a copy pod using image: ", imageName)
  270. return appExecuteRunEphemeral(ctx, config, appNamespace, selectedPod.Name, selectedContainerName, updatedExecArgs)
  271. }
  272. func getImageNameFromPod(ctx context.Context, clientset *kubernetes.Clientset, namespace, podName, containerName string) (string, error) {
  273. pod, err := clientset.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
  274. if err != nil {
  275. return "", err
  276. }
  277. for _, container := range pod.Spec.Containers {
  278. if container.Name == containerName {
  279. return container.Image, nil
  280. }
  281. }
  282. return "", fmt.Errorf("could not find container %s in pod %s", containerName, podName)
  283. }
  284. func appCleanup(ctx context.Context, _ *types.GetAuthenticatedUserResponse, client api.Client, cliConfig config.CLIConfig, _ config.FeatureFlags, _ *cobra.Command, _ []string) error {
  285. config := &AppPorterRunSharedConfig{
  286. Client: client,
  287. CLIConfig: cliConfig,
  288. }
  289. err := config.setSharedConfig(ctx)
  290. if err != nil {
  291. return fmt.Errorf("Could not retrieve kube credentials: %s", err.Error())
  292. }
  293. proceed, err := utils.PromptSelect(
  294. fmt.Sprintf("You have chosen the '%s' namespace for cleanup. Do you want to proceed?", appNamespace),
  295. []string{"Yes", "No", "All namespaces"},
  296. )
  297. if err != nil {
  298. return err
  299. }
  300. if proceed == "No" {
  301. return nil
  302. }
  303. var podNames []string
  304. color.New(color.FgGreen).Println("Fetching ephemeral pods for cleanup")
  305. if proceed == "All namespaces" {
  306. namespaces, err := config.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
  307. if err != nil {
  308. return err
  309. }
  310. for _, namespace := range namespaces.Items {
  311. if pods, err := appGetEphemeralPods(ctx, namespace.Name, config.Clientset); err == nil {
  312. podNames = append(podNames, pods...)
  313. } else {
  314. return err
  315. }
  316. }
  317. } else {
  318. if pods, err := appGetEphemeralPods(ctx, appNamespace, config.Clientset); err == nil {
  319. podNames = append(podNames, pods...)
  320. } else {
  321. return err
  322. }
  323. }
  324. if len(podNames) == 0 {
  325. color.New(color.FgBlue).Println("No ephemeral pods to delete")
  326. return nil
  327. }
  328. selectedPods, err := utils.PromptMultiselect("Select ephemeral pods to delete", podNames)
  329. if err != nil {
  330. return err
  331. }
  332. for _, podName := range selectedPods {
  333. _, _ = color.New(color.FgBlue).Printf("Deleting ephemeral pod: %s\n", podName)
  334. err = config.Clientset.CoreV1().Pods(appNamespace).Delete(
  335. ctx, podName, metav1.DeleteOptions{},
  336. )
  337. if err != nil {
  338. return err
  339. }
  340. }
  341. return nil
  342. }
  343. func appGetEphemeralPods(ctx context.Context, namespace string, clientset *kubernetes.Clientset) ([]string, error) {
  344. var podNames []string
  345. pods, err := clientset.CoreV1().Pods(namespace).List(
  346. ctx, metav1.ListOptions{LabelSelector: "porter/ephemeral-pod"},
  347. )
  348. if err != nil {
  349. return nil, err
  350. }
  351. for _, pod := range pods.Items {
  352. podNames = append(podNames, pod.Name)
  353. }
  354. return podNames, nil
  355. }
  356. type AppPorterRunSharedConfig struct {
  357. Client api.Client
  358. RestConf *rest.Config
  359. Clientset *kubernetes.Clientset
  360. RestClient *rest.RESTClient
  361. CLIConfig config.CLIConfig
  362. }
  363. func (p *AppPorterRunSharedConfig) setSharedConfig(ctx context.Context) error {
  364. pID := p.CLIConfig.Project
  365. cID := p.CLIConfig.Cluster
  366. kubeResp, err := p.Client.GetKubeconfig(ctx, pID, cID, p.CLIConfig.Kubeconfig)
  367. if err != nil {
  368. return err
  369. }
  370. kubeBytes := kubeResp.Kubeconfig
  371. cmdConf, err := clientcmd.NewClientConfigFromBytes(kubeBytes)
  372. if err != nil {
  373. return err
  374. }
  375. restConf, err := cmdConf.ClientConfig()
  376. if err != nil {
  377. return err
  378. }
  379. restConf.GroupVersion = &schema.GroupVersion{
  380. Group: "api",
  381. Version: "v1",
  382. }
  383. restConf.NegotiatedSerializer = runtime.NewSimpleNegotiatedSerializer(runtime.SerializerInfo{})
  384. p.RestConf = restConf
  385. clientset, err := kubernetes.NewForConfig(restConf)
  386. if err != nil {
  387. return err
  388. }
  389. p.Clientset = clientset
  390. restClient, err := rest.RESTClientFor(restConf)
  391. if err != nil {
  392. return err
  393. }
  394. p.RestClient = restClient
  395. return nil
  396. }
  397. type appPodSimple struct {
  398. Name string
  399. ContainerNames []string
  400. }
  401. func appGetPodsV1PorterYaml(ctx context.Context, cliConfig config.CLIConfig, client api.Client, namespace, releaseName string) ([]appPodSimple, bool, error) {
  402. pID := cliConfig.Project
  403. cID := cliConfig.Cluster
  404. var containerHasLauncherStartCommand bool
  405. resp, err := client.GetK8sAllPods(ctx, pID, cID, namespace, releaseName)
  406. if err != nil {
  407. return nil, containerHasLauncherStartCommand, err
  408. }
  409. if resp == nil {
  410. return nil, containerHasLauncherStartCommand, errors.New("get pods response is nil")
  411. }
  412. pods := *resp
  413. if len(pods) == 0 {
  414. return nil, containerHasLauncherStartCommand, errors.New("no running pods found for this application")
  415. }
  416. for _, container := range pods[0].Spec.Containers {
  417. if len(container.Command) > 0 && (container.Command[0] == CommandPrefix_LAUNCHER || container.Command[0] == CommandPrefix_CNB_LIFECYCLE_LAUNCHER) {
  418. containerHasLauncherStartCommand = true
  419. }
  420. }
  421. res := make([]appPodSimple, 0)
  422. for _, pod := range pods {
  423. if pod.Status.Phase == v1.PodRunning {
  424. containerNames := make([]string, 0)
  425. for _, container := range pod.Spec.Containers {
  426. containerNames = append(containerNames, container.Name)
  427. }
  428. res = append(res, appPodSimple{
  429. Name: pod.ObjectMeta.Name,
  430. ContainerNames: containerNames,
  431. })
  432. }
  433. }
  434. return res, containerHasLauncherStartCommand, nil
  435. }
  436. func appGetPodsV2PorterYaml(ctx context.Context, cliConfig config.CLIConfig, client api.Client, porterAppName string) ([]appPodSimple, string, bool, error) {
  437. pID := cliConfig.Project
  438. cID := cliConfig.Cluster
  439. var containerHasLauncherStartCommand bool
  440. targetResp, err := client.DefaultDeploymentTarget(ctx, pID, cID)
  441. if err != nil {
  442. return nil, "", containerHasLauncherStartCommand, fmt.Errorf("error calling default deployment target endpoint: %w", err)
  443. }
  444. if targetResp.DeploymentTargetID == "" {
  445. return nil, "", containerHasLauncherStartCommand, errors.New("deployment target id is empty")
  446. }
  447. resp, err := client.PorterYamlV2Pods(ctx, pID, cID, porterAppName, &types.PorterYamlV2PodsRequest{
  448. DeploymentTargetID: targetResp.DeploymentTargetID,
  449. })
  450. if err != nil {
  451. return nil, "", containerHasLauncherStartCommand, err
  452. }
  453. if resp == nil {
  454. return nil, "", containerHasLauncherStartCommand, errors.New("get pods response is nil")
  455. }
  456. pods := *resp
  457. if len(pods) == 0 {
  458. return nil, "", containerHasLauncherStartCommand, errors.New("no running pods found for this application")
  459. }
  460. namespace := pods[0].Namespace
  461. for _, container := range pods[0].Spec.Containers {
  462. if len(container.Command) > 0 && (container.Command[0] == CommandPrefix_LAUNCHER || container.Command[0] == CommandPrefix_CNB_LIFECYCLE_LAUNCHER) {
  463. containerHasLauncherStartCommand = true
  464. }
  465. }
  466. res := make([]appPodSimple, 0)
  467. for _, pod := range pods {
  468. if pod.Status.Phase == v1.PodRunning {
  469. containerNames := make([]string, 0)
  470. for _, container := range pod.Spec.Containers {
  471. containerNames = append(containerNames, container.Name)
  472. }
  473. res = append(res, appPodSimple{
  474. Name: pod.ObjectMeta.Name,
  475. ContainerNames: containerNames,
  476. })
  477. }
  478. }
  479. return res, namespace, containerHasLauncherStartCommand, nil
  480. }
  481. func appExecuteRun(config *AppPorterRunSharedConfig, namespace, name, container string, args []string) error {
  482. req := config.RestClient.Post().
  483. Resource("pods").
  484. Name(name).
  485. Namespace(namespace).
  486. SubResource("exec")
  487. for _, arg := range args {
  488. req.Param("command", arg)
  489. }
  490. req.Param("stdin", "true")
  491. req.Param("stdout", "true")
  492. req.Param("tty", "true")
  493. req.Param("container", container)
  494. t := term.TTY{
  495. In: os.Stdin,
  496. Out: os.Stdout,
  497. Raw: true,
  498. }
  499. size := t.GetSize()
  500. sizeQueue := t.MonitorSize(size)
  501. return t.Safe(func() error {
  502. exec, err := remotecommand.NewSPDYExecutor(config.RestConf, "POST", req.URL())
  503. if err != nil {
  504. return err
  505. }
  506. return exec.Stream(remotecommand.StreamOptions{
  507. Stdin: os.Stdin,
  508. Stdout: os.Stdout,
  509. Stderr: os.Stderr,
  510. Tty: true,
  511. TerminalSizeQueue: sizeQueue,
  512. })
  513. })
  514. }
  515. func appExecuteRunEphemeral(ctx context.Context, config *AppPorterRunSharedConfig, namespace, name, container string, args []string) error {
  516. existing, err := appGetExistingPod(ctx, config, name, namespace)
  517. if err != nil {
  518. return err
  519. }
  520. newPod, err := appCreateEphemeralPodFromExisting(ctx, config, existing, container, args)
  521. if err != nil {
  522. return err
  523. }
  524. podName := newPod.ObjectMeta.Name
  525. // delete the ephemeral pod no matter what
  526. defer appDeletePod(ctx, config, podName, namespace) //nolint:errcheck,gosec // do not want to change logic of CLI. New linter error
  527. _, _ = color.New(color.FgYellow).Printf("Waiting for pod %s to be ready...", podName)
  528. if err = appWaitForPod(ctx, config, newPod); err != nil {
  529. color.New(color.FgRed).Println("failed")
  530. return appHandlePodAttachError(ctx, err, config, namespace, podName, container)
  531. }
  532. err = appCheckForPodDeletionCronJob(ctx, config)
  533. if err != nil {
  534. return err
  535. }
  536. // refresh pod info for latest status
  537. newPod, err = config.Clientset.CoreV1().
  538. Pods(newPod.Namespace).
  539. Get(ctx, newPod.Name, metav1.GetOptions{})
  540. // pod exited while we were waiting. maybe an error maybe not.
  541. // we dont know if the user wanted an interactive shell or not.
  542. // if it was an error the logs hopefully say so.
  543. if appIsPodExited(newPod) {
  544. color.New(color.FgGreen).Println("complete!")
  545. var writtenBytes int64
  546. writtenBytes, _ = appPipePodLogsToStdout(ctx, config, namespace, podName, container, false)
  547. if appVerbose || writtenBytes == 0 {
  548. color.New(color.FgYellow).Println("Could not get logs. Pod events:")
  549. _ = appPipeEventsToStdout(ctx, config, namespace, podName, container, false) //nolint:errcheck,gosec // do not want to change logic of CLI. New linter error
  550. }
  551. return nil
  552. }
  553. color.New(color.FgGreen).Println("ready!")
  554. color.New(color.FgYellow).Println("Attempting connection to the container. If you don't see a command prompt, try pressing enter.")
  555. req := config.RestClient.Post().
  556. Resource("pods").
  557. Name(podName).
  558. Namespace(namespace).
  559. SubResource("attach")
  560. req.Param("stdin", "true")
  561. req.Param("stdout", "true")
  562. req.Param("tty", "true")
  563. req.Param("container", container)
  564. t := term.TTY{
  565. In: os.Stdin,
  566. Out: os.Stdout,
  567. Raw: true,
  568. }
  569. size := t.GetSize()
  570. sizeQueue := t.MonitorSize(size)
  571. if err = t.Safe(func() error {
  572. exec, err := remotecommand.NewSPDYExecutor(config.RestConf, "POST", req.URL())
  573. if err != nil {
  574. return err
  575. }
  576. return exec.Stream(remotecommand.StreamOptions{
  577. Stdin: os.Stdin,
  578. Stdout: os.Stdout,
  579. Stderr: os.Stderr,
  580. Tty: true,
  581. TerminalSizeQueue: sizeQueue,
  582. })
  583. }); err != nil {
  584. // ugly way to catch no TTY errors, such as when running command "echo \"hello\""
  585. return appHandlePodAttachError(ctx, err, config, namespace, podName, container)
  586. }
  587. if appVerbose {
  588. color.New(color.FgYellow).Println("Pod events:")
  589. _ = appPipeEventsToStdout(ctx, config, namespace, podName, container, false) //nolint:errcheck,gosec // do not want to change logic of CLI. New linter error
  590. }
  591. return err
  592. }
  593. func appCheckForPodDeletionCronJob(ctx context.Context, config *AppPorterRunSharedConfig) error {
  594. // try and create the cron job and all of the other required resources as necessary,
  595. // starting with the service account, then role and then a role binding
  596. err := appCheckForServiceAccount(ctx, config)
  597. if err != nil {
  598. return err
  599. }
  600. err = appCheckForClusterRole(ctx, config)
  601. if err != nil {
  602. return err
  603. }
  604. err = appCheckForRoleBinding(ctx, config)
  605. if err != nil {
  606. return err
  607. }
  608. namespaces, err := config.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
  609. if err != nil {
  610. return err
  611. }
  612. for _, namespace := range namespaces.Items {
  613. cronJobs, err := config.Clientset.BatchV1().CronJobs(namespace.Name).List(
  614. ctx, metav1.ListOptions{},
  615. )
  616. if err != nil {
  617. return err
  618. }
  619. if namespace.Name == "default" {
  620. for _, cronJob := range cronJobs.Items {
  621. if cronJob.Name == "porter-ephemeral-pod-deletion-cronjob" {
  622. return nil
  623. }
  624. }
  625. } else {
  626. for _, cronJob := range cronJobs.Items {
  627. if cronJob.Name == "porter-ephemeral-pod-deletion-cronjob" {
  628. err = config.Clientset.BatchV1().CronJobs(namespace.Name).Delete(
  629. ctx, cronJob.Name, metav1.DeleteOptions{},
  630. )
  631. if err != nil {
  632. return err
  633. }
  634. }
  635. }
  636. }
  637. }
  638. // create the cronjob
  639. cronJob := &batchv1.CronJob{
  640. ObjectMeta: metav1.ObjectMeta{
  641. Name: "porter-ephemeral-pod-deletion-cronjob",
  642. },
  643. Spec: batchv1.CronJobSpec{
  644. Schedule: "0 * * * *",
  645. JobTemplate: batchv1.JobTemplateSpec{
  646. Spec: batchv1.JobSpec{
  647. Template: v1.PodTemplateSpec{
  648. Spec: v1.PodSpec{
  649. ServiceAccountName: "porter-ephemeral-pod-deletion-service-account",
  650. RestartPolicy: v1.RestartPolicyNever,
  651. Containers: []v1.Container{
  652. {
  653. Name: "ephemeral-pods-manager",
  654. Image: "public.ecr.aws/o1j4x7p4/porter-ephemeral-pods-manager:latest",
  655. ImagePullPolicy: v1.PullAlways,
  656. Args: []string{"delete"},
  657. },
  658. },
  659. },
  660. },
  661. },
  662. },
  663. },
  664. }
  665. _, err = config.Clientset.BatchV1().CronJobs("default").Create(
  666. ctx, cronJob, metav1.CreateOptions{},
  667. )
  668. if err != nil {
  669. return err
  670. }
  671. return nil
  672. }
  673. func appCheckForServiceAccount(ctx context.Context, config *AppPorterRunSharedConfig) error {
  674. namespaces, err := config.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
  675. if err != nil {
  676. return err
  677. }
  678. for _, namespace := range namespaces.Items {
  679. serviceAccounts, err := config.Clientset.CoreV1().ServiceAccounts(namespace.Name).List(
  680. ctx, metav1.ListOptions{},
  681. )
  682. if err != nil {
  683. return err
  684. }
  685. if namespace.Name == "default" {
  686. for _, svcAccount := range serviceAccounts.Items {
  687. if svcAccount.Name == "porter-ephemeral-pod-deletion-service-account" {
  688. return nil
  689. }
  690. }
  691. } else {
  692. for _, svcAccount := range serviceAccounts.Items {
  693. if svcAccount.Name == "porter-ephemeral-pod-deletion-service-account" {
  694. err = config.Clientset.CoreV1().ServiceAccounts(namespace.Name).Delete(
  695. ctx, svcAccount.Name, metav1.DeleteOptions{},
  696. )
  697. if err != nil {
  698. return err
  699. }
  700. }
  701. }
  702. }
  703. }
  704. serviceAccount := &v1.ServiceAccount{
  705. ObjectMeta: metav1.ObjectMeta{
  706. Name: "porter-ephemeral-pod-deletion-service-account",
  707. },
  708. }
  709. _, err = config.Clientset.CoreV1().ServiceAccounts("default").Create(
  710. ctx, serviceAccount, metav1.CreateOptions{},
  711. )
  712. if err != nil {
  713. return err
  714. }
  715. return nil
  716. }
  717. func appCheckForClusterRole(ctx context.Context, config *AppPorterRunSharedConfig) error {
  718. roles, err := config.Clientset.RbacV1().ClusterRoles().List(
  719. ctx, metav1.ListOptions{},
  720. )
  721. if err != nil {
  722. return err
  723. }
  724. for _, role := range roles.Items {
  725. if role.Name == "porter-ephemeral-pod-deletion-cluster-role" {
  726. return nil
  727. }
  728. }
  729. role := &rbacv1.ClusterRole{
  730. ObjectMeta: metav1.ObjectMeta{
  731. Name: "porter-ephemeral-pod-deletion-cluster-role",
  732. },
  733. Rules: []rbacv1.PolicyRule{
  734. {
  735. APIGroups: []string{""},
  736. Resources: []string{"pods"},
  737. Verbs: []string{"list", "delete"},
  738. },
  739. {
  740. APIGroups: []string{""},
  741. Resources: []string{"namespaces"},
  742. Verbs: []string{"list"},
  743. },
  744. },
  745. }
  746. _, err = config.Clientset.RbacV1().ClusterRoles().Create(
  747. ctx, role, metav1.CreateOptions{},
  748. )
  749. if err != nil {
  750. return err
  751. }
  752. return nil
  753. }
  754. func appCheckForRoleBinding(ctx context.Context, config *AppPorterRunSharedConfig) error {
  755. bindings, err := config.Clientset.RbacV1().ClusterRoleBindings().List(
  756. ctx, metav1.ListOptions{},
  757. )
  758. if err != nil {
  759. return err
  760. }
  761. for _, binding := range bindings.Items {
  762. if binding.Name == "porter-ephemeral-pod-deletion-cluster-rolebinding" {
  763. return nil
  764. }
  765. }
  766. binding := &rbacv1.ClusterRoleBinding{
  767. ObjectMeta: metav1.ObjectMeta{
  768. Name: "porter-ephemeral-pod-deletion-cluster-rolebinding",
  769. },
  770. RoleRef: rbacv1.RoleRef{
  771. APIGroup: "rbac.authorization.k8s.io",
  772. Kind: "ClusterRole",
  773. Name: "porter-ephemeral-pod-deletion-cluster-role",
  774. },
  775. Subjects: []rbacv1.Subject{
  776. {
  777. APIGroup: "",
  778. Kind: "ServiceAccount",
  779. Name: "porter-ephemeral-pod-deletion-service-account",
  780. Namespace: "default",
  781. },
  782. },
  783. }
  784. _, err = config.Clientset.RbacV1().ClusterRoleBindings().Create(
  785. ctx, binding, metav1.CreateOptions{},
  786. )
  787. if err != nil {
  788. return err
  789. }
  790. return nil
  791. }
  792. func appWaitForPod(ctx context.Context, config *AppPorterRunSharedConfig, pod *v1.Pod) error {
  793. var (
  794. w watch.Interface
  795. err error
  796. ok bool
  797. )
  798. // immediately after creating a pod, the API may return a 404. heuristically 1
  799. // second seems to be plenty.
  800. watchRetries := 3
  801. for i := 0; i < watchRetries; i++ {
  802. selector := fields.OneTermEqualSelector("metadata.name", pod.Name).String()
  803. w, err = config.Clientset.CoreV1().
  804. Pods(pod.Namespace).
  805. Watch(ctx, metav1.ListOptions{FieldSelector: selector})
  806. if err == nil {
  807. break
  808. }
  809. time.Sleep(time.Second)
  810. }
  811. if err != nil {
  812. return err
  813. }
  814. defer w.Stop()
  815. for {
  816. select {
  817. case <-time.Tick(time.Second):
  818. // poll every second in case we already missed the ready event while
  819. // creating the listener.
  820. pod, err = config.Clientset.CoreV1().
  821. Pods(pod.Namespace).
  822. Get(ctx, pod.Name, metav1.GetOptions{})
  823. if appIsPodReady(pod) || appIsPodExited(pod) {
  824. return nil
  825. }
  826. case evt := <-w.ResultChan():
  827. pod, ok = evt.Object.(*v1.Pod)
  828. if !ok {
  829. return fmt.Errorf("unexpected object type: %T", evt.Object)
  830. }
  831. if appIsPodReady(pod) || appIsPodExited(pod) {
  832. return nil
  833. }
  834. case <-time.After(time.Second * 10):
  835. return errors.New("timed out waiting for pod")
  836. }
  837. }
  838. }
  839. func appIsPodReady(pod *v1.Pod) bool {
  840. ready := false
  841. conditions := pod.Status.Conditions
  842. for i := range conditions {
  843. if conditions[i].Type == v1.PodReady {
  844. ready = pod.Status.Conditions[i].Status == v1.ConditionTrue
  845. }
  846. }
  847. return ready
  848. }
  849. func appIsPodExited(pod *v1.Pod) bool {
  850. return pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed
  851. }
  852. func appHandlePodAttachError(ctx context.Context, err error, config *AppPorterRunSharedConfig, namespace, podName, container string) error {
  853. if appVerbose {
  854. color.New(color.FgYellow).Fprintf(os.Stderr, "Error: %s\n", err)
  855. }
  856. color.New(color.FgYellow).Fprintln(os.Stderr, "Could not open a shell to this container. Container logs:")
  857. var writtenBytes int64
  858. writtenBytes, _ = appPipePodLogsToStdout(ctx, config, namespace, podName, container, false)
  859. if appVerbose || writtenBytes == 0 {
  860. color.New(color.FgYellow).Fprintln(os.Stderr, "Could not get logs. Pod events:")
  861. _ = appPipeEventsToStdout(ctx, config, namespace, podName, container, false) //nolint:errcheck,gosec // do not want to change logic of CLI. New linter error
  862. }
  863. return err
  864. }
  865. func appPipePodLogsToStdout(ctx context.Context, config *AppPorterRunSharedConfig, namespace, name, container string, follow bool) (int64, error) {
  866. podLogOpts := v1.PodLogOptions{
  867. Container: container,
  868. Follow: follow,
  869. }
  870. req := config.Clientset.CoreV1().Pods(namespace).GetLogs(name, &podLogOpts)
  871. podLogs, err := req.Stream(
  872. ctx,
  873. )
  874. if err != nil {
  875. return 0, err
  876. }
  877. defer podLogs.Close()
  878. return io.Copy(os.Stdout, podLogs)
  879. }
  880. func appPipeEventsToStdout(ctx context.Context, config *AppPorterRunSharedConfig, namespace, name, _ string, _ bool) error {
  881. // update the config in case the operation has taken longer than token expiry time
  882. config.setSharedConfig(ctx) //nolint:errcheck,gosec // do not want to change logic of CLI. New linter error
  883. // creates the clientset
  884. resp, err := config.Clientset.CoreV1().Events(namespace).List(
  885. ctx,
  886. metav1.ListOptions{
  887. FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=%s", name, namespace),
  888. },
  889. )
  890. if err != nil {
  891. return err
  892. }
  893. for _, event := range resp.Items {
  894. color.New(color.FgRed).Println(event.Message)
  895. }
  896. return nil
  897. }
  898. func appGetExistingPod(ctx context.Context, config *AppPorterRunSharedConfig, name, namespace string) (*v1.Pod, error) {
  899. return config.Clientset.CoreV1().Pods(namespace).Get(
  900. ctx,
  901. name,
  902. metav1.GetOptions{},
  903. )
  904. }
  905. func appDeletePod(ctx context.Context, config *AppPorterRunSharedConfig, name, namespace string) error {
  906. // update the config in case the operation has taken longer than token expiry time
  907. config.setSharedConfig(ctx) //nolint:errcheck,gosec // do not want to change logic of CLI. New linter error
  908. err := config.Clientset.CoreV1().Pods(namespace).Delete(
  909. ctx,
  910. name,
  911. metav1.DeleteOptions{},
  912. )
  913. if err != nil {
  914. color.New(color.FgRed).Fprintf(os.Stderr, "Could not delete ephemeral pod: %s\n", err.Error())
  915. return err
  916. }
  917. color.New(color.FgGreen).Println("Sucessfully deleted ephemeral pod")
  918. return nil
  919. }
  920. func appCreateEphemeralPodFromExisting(
  921. ctx context.Context,
  922. config *AppPorterRunSharedConfig,
  923. existing *v1.Pod,
  924. container string,
  925. args []string,
  926. ) (*v1.Pod, error) {
  927. newPod := existing.DeepCopy()
  928. // only copy the pod spec, overwrite metadata
  929. newPod.ObjectMeta = metav1.ObjectMeta{
  930. Name: strings.ToLower(fmt.Sprintf("%s-copy-%s", existing.ObjectMeta.Name, utils.String(4))),
  931. Namespace: existing.ObjectMeta.Namespace,
  932. }
  933. newPod.Status = v1.PodStatus{}
  934. // set restart policy to never
  935. newPod.Spec.RestartPolicy = v1.RestartPolicyNever
  936. // change the command in the pod to the passed in pod command
  937. cmdRoot := args[0]
  938. cmdArgs := make([]string, 0)
  939. // annotate with the ephemeral pod tag
  940. newPod.Labels = make(map[string]string)
  941. newPod.Labels["porter/ephemeral-pod"] = "true"
  942. if len(args) > 1 {
  943. cmdArgs = args[1:]
  944. }
  945. for i := 0; i < len(newPod.Spec.Containers); i++ {
  946. if newPod.Spec.Containers[i].Name == container {
  947. newPod.Spec.Containers[i].Command = []string{cmdRoot}
  948. newPod.Spec.Containers[i].Args = cmdArgs
  949. newPod.Spec.Containers[i].TTY = true
  950. newPod.Spec.Containers[i].Stdin = true
  951. newPod.Spec.Containers[i].StdinOnce = true
  952. var newCpu int
  953. if appCpuMilli != 0 {
  954. newCpu = appCpuMilli
  955. } else if newPod.Spec.Containers[i].Resources.Requests.Cpu() != nil && newPod.Spec.Containers[i].Resources.Requests.Cpu().MilliValue() > 500 {
  956. newCpu = 500
  957. }
  958. if newCpu != 0 {
  959. newPod.Spec.Containers[i].Resources.Limits[v1.ResourceCPU] = resource.MustParse(fmt.Sprintf("%dm", newCpu))
  960. newPod.Spec.Containers[i].Resources.Requests[v1.ResourceCPU] = resource.MustParse(fmt.Sprintf("%dm", newCpu))
  961. for j := 0; j < len(newPod.Spec.Containers[i].Env); j++ {
  962. if newPod.Spec.Containers[i].Env[j].Name == "PORTER_RESOURCES_CPU" {
  963. newPod.Spec.Containers[i].Env[j].Value = fmt.Sprintf("%dm", newCpu)
  964. break
  965. }
  966. }
  967. }
  968. var newMemory int
  969. if appMemoryMi != 0 {
  970. newMemory = appMemoryMi
  971. } else if newPod.Spec.Containers[i].Resources.Requests.Memory() != nil && newPod.Spec.Containers[i].Resources.Requests.Memory().Value() > 1000*1024*1024 {
  972. newMemory = 1000
  973. }
  974. if newMemory != 0 {
  975. newPod.Spec.Containers[i].Resources.Limits[v1.ResourceMemory] = resource.MustParse(fmt.Sprintf("%dMi", newMemory))
  976. newPod.Spec.Containers[i].Resources.Requests[v1.ResourceMemory] = resource.MustParse(fmt.Sprintf("%dMi", newMemory))
  977. for j := 0; j < len(newPod.Spec.Containers[i].Env); j++ {
  978. if newPod.Spec.Containers[i].Env[j].Name == "PORTER_RESOURCES_RAM" {
  979. newPod.Spec.Containers[i].Env[j].Value = fmt.Sprintf("%dMi", newMemory)
  980. break
  981. }
  982. }
  983. }
  984. }
  985. // remove health checks and probes
  986. newPod.Spec.Containers[i].LivenessProbe = nil
  987. newPod.Spec.Containers[i].ReadinessProbe = nil
  988. newPod.Spec.Containers[i].StartupProbe = nil
  989. }
  990. newPod.Spec.NodeName = ""
  991. // create the pod and return it
  992. return config.Clientset.CoreV1().Pods(existing.ObjectMeta.Namespace).Create(
  993. ctx,
  994. newPod,
  995. metav1.CreateOptions{},
  996. )
  997. }
  998. func appUpdateTag(ctx context.Context, user *types.GetAuthenticatedUserResponse, client api.Client, cliConf config.CLIConfig, featureFlags config.FeatureFlags, cmd *cobra.Command, args []string) error {
  999. project, err := client.GetProject(ctx, cliConf.Project)
  1000. if err != nil {
  1001. return fmt.Errorf("could not retrieve project from Porter API. Please contact support@porter.run")
  1002. }
  1003. if project.ValidateApplyV2 {
  1004. tag, err := v2.UpdateImage(ctx, appTag, client, cliConf.Project, cliConf.Cluster, args[0])
  1005. if err != nil {
  1006. return fmt.Errorf("error updating tag: %w", err)
  1007. }
  1008. _, _ = color.New(color.FgGreen).Printf("Successfully updated application %s to use tag \"%s\"\n", args[0], tag)
  1009. return nil
  1010. } else {
  1011. namespace := fmt.Sprintf("porter-stack-%s", args[0])
  1012. if appTag == "" {
  1013. appTag = "latest"
  1014. }
  1015. release, err := client.GetRelease(ctx, cliConf.Project, cliConf.Cluster, namespace, args[0])
  1016. if err != nil {
  1017. return fmt.Errorf("Unable to find application %s", args[0])
  1018. }
  1019. repository, ok := release.Config["global"].(map[string]interface{})["image"].(map[string]interface{})["repository"].(string)
  1020. if !ok || repository == "" {
  1021. return fmt.Errorf("Application %s does not have an associated image repository. Unable to update tag", args[0])
  1022. }
  1023. imageInfo := types.ImageInfo{
  1024. Repository: repository,
  1025. Tag: appTag,
  1026. }
  1027. createUpdatePorterAppRequest := &types.CreatePorterAppRequest{
  1028. ClusterID: cliConf.Cluster,
  1029. ProjectID: cliConf.Project,
  1030. ImageInfo: imageInfo,
  1031. OverrideRelease: false,
  1032. }
  1033. _, _ = color.New(color.FgGreen).Printf("Updating application %s to build using tag \"%s\"\n", args[0], appTag)
  1034. _, err = client.CreatePorterApp(
  1035. ctx,
  1036. cliConf.Project,
  1037. cliConf.Cluster,
  1038. args[0],
  1039. createUpdatePorterAppRequest,
  1040. )
  1041. if err != nil {
  1042. return fmt.Errorf("Unable to update application %s: %w", args[0], err)
  1043. }
  1044. _, _ = color.New(color.FgGreen).Printf("Successfully updated application %s to use tag \"%s\"\n", args[0], appTag)
  1045. return nil
  1046. }
  1047. }
  1048. func getPodsFromV1PorterYaml(ctx context.Context, execArgs []string, client api.Client, cliConfig config.CLIConfig, porterAppName string, namespace string) ([]appPodSimple, []string, error) {
  1049. podsSimple, containerHasLauncherStartCommand, err := appGetPodsV1PorterYaml(ctx, cliConfig, client, namespace, porterAppName)
  1050. if err != nil {
  1051. return nil, nil, fmt.Errorf("could not retrieve list of pods: %s", err.Error())
  1052. }
  1053. if len(execArgs) > 0 && execArgs[0] != CommandPrefix_CNB_LIFECYCLE_LAUNCHER && execArgs[0] != CommandPrefix_LAUNCHER && containerHasLauncherStartCommand {
  1054. execArgs = append([]string{CommandPrefix_CNB_LIFECYCLE_LAUNCHER}, execArgs...)
  1055. }
  1056. return podsSimple, execArgs, nil
  1057. }
  1058. func getPodsFromV2PorterYaml(ctx context.Context, execArgs []string, client api.Client, cliConfig config.CLIConfig, porterAppName string) ([]appPodSimple, []string, string, error) {
  1059. podsSimple, namespace, containerHasLauncherStartCommand, err := appGetPodsV2PorterYaml(ctx, cliConfig, client, porterAppName)
  1060. if err != nil {
  1061. return nil, nil, "", fmt.Errorf("could not retrieve list of pods: %w", err)
  1062. }
  1063. if len(execArgs) > 0 && execArgs[0] != CommandPrefix_CNB_LIFECYCLE_LAUNCHER && execArgs[0] != CommandPrefix_LAUNCHER && containerHasLauncherStartCommand {
  1064. execArgs = append([]string{CommandPrefix_CNB_LIFECYCLE_LAUNCHER}, execArgs...)
  1065. }
  1066. return podsSimple, execArgs, namespace, nil
  1067. }