Merge pull request #1603 from porter-dev/nafees/ephemeral-pods-cronjob

[POR-307] Cronjob to delete ephemeral pods

cli/cmd/run.go

@@ -14,7 +14,10 @@ import (
 	"github.com/porter-dev/porter/api/types"
 	"github.com/porter-dev/porter/cli/cmd/utils"
 	"github.com/spf13/cobra"
+	batchv1 "k8s.io/api/batch/v1"
+	batchv1beta1 "k8s.io/api/batch/v1beta1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/watch"
@@ -46,6 +49,20 @@ var runCmd = &cobra.Command{
 	},
 }
 
+// cleanupCmd represents the "porter run cleanup" subcommand
+var cleanupCmd = &cobra.Command{
+	Use:   "cleanup",
+	Args:  cobra.NoArgs,
+	Short: "Delete any lingering ephemeral pods that were created with \"porter run\".",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := checkLoginAndRun(args, cleanup)
+
+		if err != nil {
+			os.Exit(1)
+		}
+	},
+}
+
 var existingPod bool
 
 func init() {
@@ -73,6 +90,8 @@ func init() {
 		false,
 		"whether to print verbose output",
 	)
+
+	runCmd.AddCommand(cleanupCmd)
 }
 
 func run(_ *types.GetAuthenticatedUserResponse, client *api.Client, args []string) error {
@@ -146,6 +165,94 @@ func run(_ *types.GetAuthenticatedUserResponse, client *api.Client, args []strin
 	return executeRunEphemeral(config, namespace, selectedPod.Name, selectedContainerName, args[1:])
 }
 
+func cleanup(_ *types.GetAuthenticatedUserResponse, client *api.Client, _ []string) error {
+	config := &PorterRunSharedConfig{
+		Client: client,
+	}
+
+	err := config.setSharedConfig()
+	if err != nil {
+		return fmt.Errorf("Could not retrieve kube credentials: %s", err.Error())
+	}
+
+	proceed, err := utils.PromptSelect(
+		fmt.Sprintf("You have chosen the '%s' namespace for cleanup. Do you want to proceed?", namespace),
+		[]string{"Yes", "No", "All namespaces"},
+	)
+	if err != nil {
+		return err
+	}
+
+	if proceed == "No" {
+		return nil
+	}
+
+	var podNames []string
+
+	color.New(color.FgGreen).Println("Fetching ephemeral pods for cleanup")
+
+	if proceed == "All namespaces" {
+		namespaces, err := config.Clientset.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
+
+		for _, namespace := range namespaces.Items {
+			if pods, err := getEphemeralPods(namespace.Name, config.Clientset); err == nil {
+				podNames = append(podNames, pods...)
+			} else {
+				return err
+			}
+		}
+	} else {
+		if pods, err := getEphemeralPods(namespace, config.Clientset); err == nil {
+			podNames = append(podNames, pods...)
+		} else {
+			return err
+		}
+	}
+
+	if len(podNames) == 0 {
+		color.New(color.FgBlue).Println("No ephemeral pods to delete")
+		return nil
+	}
+
+	selectedPods, err := utils.PromptMultiselect("Select ephemeral pods to delete", podNames)
+	if err != nil {
+		return err
+	}
+
+	for _, podName := range selectedPods {
+		color.New(color.FgBlue).Printf("Deleting ephemeral pod: %s\n", podName)
+
+		err = config.Clientset.CoreV1().Pods(namespace).Delete(
+			context.Background(), podName, metav1.DeleteOptions{},
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func getEphemeralPods(namespace string, clientset *kubernetes.Clientset) ([]string, error) {
+	var podNames []string
+
+	pods, err := clientset.CoreV1().Pods(namespace).List(
+		context.Background(), metav1.ListOptions{LabelSelector: "porter/ephemeral-pod"},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, pod := range pods.Items {
+		podNames = append(podNames, pod.Name)
+	}
+
+	return podNames, nil
+}
+
 type PorterRunSharedConfig struct {
 	Client     *api.Client
 	RestConf   *rest.Config
@@ -288,7 +395,10 @@ func executeRunEphemeral(config *PorterRunSharedConfig, namespace, name, contain
 		return err
 	}
 
-	newPod, err := createPodFromExisting(config, existing, args)
+	newPod, err := createEphemeralPodFromExisting(config, existing, args)
+	if err != nil {
+		return err
+	}
 	podName := newPod.ObjectMeta.Name
 
 	// delete the ephemeral pod no matter what
@@ -300,6 +410,11 @@ func executeRunEphemeral(config *PorterRunSharedConfig, namespace, name, contain
 		return handlePodAttachError(err, config, namespace, podName, container)
 	}
 
+	err = checkForPodDeletionCronJob(config)
+	if err != nil {
+		return err
+	}
+
 	// refresh pod info for latest status
 	newPod, err = config.Clientset.CoreV1().
 		Pods(newPod.Namespace).
@@ -367,6 +482,195 @@ func executeRunEphemeral(config *PorterRunSharedConfig, namespace, name, contain
 	return err
 }
 
+func checkForPodDeletionCronJob(config *PorterRunSharedConfig) error {
+	namespaces, err := config.Clientset.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, namespace := range namespaces.Items {
+		cronJobs, err := config.Clientset.BatchV1beta1().CronJobs(namespace.Name).List(
+			context.Background(), metav1.ListOptions{},
+		)
+		if err != nil {
+			return err
+		}
+
+		for _, cronJob := range cronJobs.Items {
+			if cronJob.Name == "porter-ephemeral-pod-deletion-cronjob" {
+				return nil
+			}
+		}
+	}
+
+	// try and create the cron job and all of the other required resources as necessary,
+	// starting with the service account, then role and then a role binding
+
+	err = checkForServiceAccount(config)
+	if err != nil {
+		return err
+	}
+
+	err = checkForClusterRole(config)
+	if err != nil {
+		return err
+	}
+
+	err = checkForRoleBinding(config)
+	if err != nil {
+		return err
+	}
+
+	// create the cronjob
+
+	cronJob := &batchv1beta1.CronJob{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "porter-ephemeral-pod-deletion-cronjob",
+		},
+		Spec: batchv1beta1.CronJobSpec{
+			Schedule: "0 * * * *",
+			JobTemplate: batchv1beta1.JobTemplateSpec{
+				Spec: batchv1.JobSpec{
+					Template: v1.PodTemplateSpec{
+						Spec: v1.PodSpec{
+							ServiceAccountName: "porter-ephemeral-pod-deletion-service-account",
+							RestartPolicy:      v1.RestartPolicyNever,
+							Containers: []v1.Container{
+								{
+									Name:            "ephemeral-pods-manager",
+									Image:           "public.ecr.aws/o1j4x7p4/porter-ephemeral-pods-manager:latest",
+									ImagePullPolicy: v1.PullAlways,
+									Args:            []string{"delete"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	_, err = config.Clientset.BatchV1beta1().CronJobs(namespace).Create(
+		context.Background(), cronJob, metav1.CreateOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkForServiceAccount(config *PorterRunSharedConfig) error {
+	serviceAccounts, err := config.Clientset.CoreV1().ServiceAccounts(namespace).List(
+		context.Background(), metav1.ListOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	for _, serviceAccount := range serviceAccounts.Items {
+		if serviceAccount.Name == "porter-ephemeral-pod-deletion-service-account" {
+			return nil
+		}
+	}
+
+	serviceAccount := &v1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "porter-ephemeral-pod-deletion-service-account",
+		},
+	}
+	_, err = config.Clientset.CoreV1().ServiceAccounts(namespace).Create(
+		context.Background(), serviceAccount, metav1.CreateOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkForClusterRole(config *PorterRunSharedConfig) error {
+	roles, err := config.Clientset.RbacV1().ClusterRoles().List(
+		context.Background(), metav1.ListOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	for _, role := range roles.Items {
+		if role.Name == "porter-ephemeral-pod-deletion-cluster-role" {
+			return nil
+		}
+	}
+
+	role := &rbacv1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "porter-ephemeral-pod-deletion-cluster-role",
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"pods"},
+				Verbs:     []string{"list", "delete"},
+			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"namespaces"},
+				Verbs:     []string{"list"},
+			},
+		},
+	}
+	_, err = config.Clientset.RbacV1().ClusterRoles().Create(
+		context.Background(), role, metav1.CreateOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkForRoleBinding(config *PorterRunSharedConfig) error {
+	bindings, err := config.Clientset.RbacV1().ClusterRoleBindings().List(
+		context.Background(), metav1.ListOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	for _, binding := range bindings.Items {
+		if binding.Name == "porter-ephemeral-pod-deletion-cluster-rolebinding" {
+			return nil
+		}
+	}
+
+	binding := &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "porter-ephemeral-pod-deletion-cluster-rolebinding",
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     "porter-ephemeral-pod-deletion-cluster-role",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				APIGroup:  "",
+				Kind:      "ServiceAccount",
+				Name:      "porter-ephemeral-pod-deletion-service-account",
+				Namespace: "default",
+			},
+		},
+	}
+	_, err = config.Clientset.RbacV1().ClusterRoleBindings().Create(
+		context.Background(), binding, metav1.CreateOptions{},
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func waitForPod(config *PorterRunSharedConfig, pod *v1.Pod) error {
 	var (
 		w   watch.Interface
@@ -519,7 +823,7 @@ func deletePod(config *PorterRunSharedConfig, name, namespace string) error {
 	return nil
 }
 
-func createPodFromExisting(config *PorterRunSharedConfig, existing *v1.Pod, args []string) (*v1.Pod, error) {
+func createEphemeralPodFromExisting(config *PorterRunSharedConfig, existing *v1.Pod, args []string) (*v1.Pod, error) {
 	newPod := existing.DeepCopy()
 
 	// only copy the pod spec, overwrite metadata
@@ -540,6 +844,10 @@ func createPodFromExisting(config *PorterRunSharedConfig, existing *v1.Pod, args
 	cmdRoot := args[0]
 	cmdArgs := make([]string, 0)
 
+	// annotate with the ephemeral pod tag
+	newPod.Labels = make(map[string]string)
+	newPod.Labels["porter/ephemeral-pod"] = "true"
+
 	if len(args) > 1 {
 		cmdArgs = args[1:]
 	}

cli/cmd/utils/prompt.go

@@ -79,3 +79,16 @@ func PromptSelect(prompt string, options []string) (string, error) {
 
 	return ans.Response, err
 }
+
+func PromptMultiselect(prompt string, options []string) ([]string, error) {
+	query := &survey.MultiSelect{
+		Message: prompt,
+		Options: options,
+	}
+
+	var ans []string
+
+	err := survey.AskOne(query, &ans)
+
+	return ans, err
+}