Greatly simplify the amount of work the front-end has to do to find logs; push all that logic to the backend (#3101)

api/server/handlers/cluster/get_logs_pod_values.go

@@ -38,12 +38,7 @@ func (c *GetLogPodValuesHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	var namespace string
-	if request.Namespace != "" {
-		namespace = request.Namespace
-	}
-
-	agent, err := c.GetAgent(r, cluster, namespace)
+	agent, err := c.GetAgent(r, cluster, "")
 	if err != nil {
 		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
 		return

api/server/handlers/porter_app/get_logs_within_time_range.go

@@ -0,0 +1,146 @@
+package porter_app
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/porter-dev/porter/api/server/authz"
+	"github.com/porter-dev/porter/api/server/handlers"
+	"github.com/porter-dev/porter/api/server/shared"
+	"github.com/porter-dev/porter/api/server/shared/apierrors"
+	"github.com/porter-dev/porter/api/server/shared/config"
+	"github.com/porter-dev/porter/api/types"
+	porter_agent "github.com/porter-dev/porter/internal/kubernetes/porter_agent/v2"
+	"github.com/porter-dev/porter/internal/models"
+	"github.com/porter-dev/porter/internal/telemetry"
+	v1 "k8s.io/api/core/v1"
+)
+
+type GetLogsWithinTimeRangeHandler struct {
+	handlers.PorterHandlerReadWriter
+	authz.KubernetesAgentGetter
+}
+
+func NewGetLogsWithinTimeRangeHandler(
+	config *config.Config,
+	decoderValidator shared.RequestDecoderValidator,
+	writer shared.ResultWriter,
+) *GetLogsWithinTimeRangeHandler {
+	return &GetLogsWithinTimeRangeHandler{
+		PorterHandlerReadWriter: handlers.NewDefaultPorterHandler(config, decoderValidator, writer),
+		KubernetesAgentGetter:   authz.NewOutOfClusterAgentGetter(config),
+	}
+}
+
+func (c *GetLogsWithinTimeRangeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	ctx, span := telemetry.NewSpan(r.Context(), "serve-get-logs-within-time-range")
+	defer span.End()
+	r = r.Clone(ctx)
+	cluster, _ := ctx.Value(types.ClusterScope).(*models.Cluster)
+
+	request := &types.GetChartLogsWithinTimeRangeRequest{}
+	if ok := c.DecodeAndValidate(w, r, request); !ok {
+		return
+	}
+
+	if request.StartRange.IsZero() || request.EndRange.IsZero() {
+		err := telemetry.Error(ctx, span, nil, "must provide start and end range")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusBadRequest))
+		return
+	}
+
+	agent, err := c.GetAgent(r, cluster, "")
+	if err != nil {
+		_ = telemetry.Error(ctx, span, err, "unable to get agent")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(fmt.Errorf("unable to get agent"), http.StatusInternalServerError))
+		return
+	}
+
+	// get agent service
+	agentSvc, err := porter_agent.GetAgentService(agent.Clientset)
+	if err != nil {
+		_ = telemetry.Error(ctx, span, err, "unable to get agent service")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(fmt.Errorf("unable to get agent service"), http.StatusInternalServerError))
+		return
+	}
+
+	podValuesRequest := &types.GetPodValuesRequest{
+		StartRange:  &request.StartRange,
+		EndRange:    &request.EndRange,
+		Namespace:   request.Namespace,
+		MatchPrefix: request.ChartName,
+		Revision:    request.Revision,
+	}
+
+	var podSelector string
+	if request.ChartName == "" {
+		if request.PodSelector == "" {
+			err = telemetry.Error(ctx, span, nil, "must provide either chart name or pod selector")
+			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusBadRequest))
+			return
+		}
+		podSelector = request.PodSelector
+	} else {
+		// get the pod values which will be used to get the correct pod selector
+		podVals, err := porter_agent.GetPodValues(agent.Clientset, agentSvc, podValuesRequest)
+		if err != nil {
+			_ = telemetry.Error(ctx, span, err, "unable to get pod values")
+			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusInternalServerError))
+			return
+		}
+
+		if len(podVals) == 0 {
+			err = telemetry.Error(ctx, span, nil, "no pods found within timerange")
+			c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusNotFound))
+			return
+		}
+		if len(podVals) == 1 {
+			podSelector = podVals[0]
+		} else {
+			// TODO: why are pods being returned from get pod values whose timestamps don't overlap with the search range??
+			// hacky workaround for the above bug, only for jobs - get the pods, and then filter them by timestamp
+			var latestPod *v1.Pod
+			for _, v := range podVals {
+				name := strings.Split(v, "-hook")[0] + "-hook"
+				pods, err := agent.GetJobPods(request.Namespace, name)
+				if err != nil {
+					_ = telemetry.Error(ctx, span, err, "unable to get pods for job")
+					c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(fmt.Errorf("unable to get pods for job"), http.StatusInternalServerError))
+					return
+				}
+				for _, pod := range pods {
+					if pod.GetCreationTimestamp().Time.After(request.StartRange) && pod.GetCreationTimestamp().Time.Before(request.EndRange) {
+						if latestPod == nil || pod.GetCreationTimestamp().Time.After(latestPod.GetCreationTimestamp().Time) {
+							latestPod = &pod
+						}
+					}
+				}
+			}
+			if latestPod == nil {
+				err = telemetry.Error(ctx, span, nil, "no pods found within timerange")
+				c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(err, http.StatusNotFound))
+				return
+			}
+			podSelector = latestPod.Name
+		}
+	}
+
+	logRequest := &types.GetLogRequest{
+		Limit:       request.Limit,
+		StartRange:  &request.StartRange,
+		EndRange:    &request.EndRange,
+		Revision:    request.Revision,
+		PodSelector: podSelector,
+		Namespace:   request.Namespace,
+	}
+
+	logs, err := porter_agent.GetHistoricalLogs(agent.Clientset, agentSvc, logRequest)
+	if err != nil {
+		_ = telemetry.Error(ctx, span, err, "unable to get logs")
+		c.HandleAPIError(w, r, apierrors.NewErrPassThroughToClient(fmt.Errorf("unable to get logs for pod selector %s", podSelector), http.StatusInternalServerError))
+		return
+	}
+
+	c.WriteResult(w, r, logs)
+}

api/server/router/porter_app.go

@@ -284,5 +284,34 @@ func getStackRoutes(
 		Router:   r,
 	})
 
+	// GET /api/projects/{project_id}/clusters/{cluster_id}/stacks/logs -> cluster.NewGetChartLogsWithinTimeRangeHandler
+	getChartLogsWithinTimeRangeEndpoint := factory.NewAPIEndpoint(
+		&types.APIRequestMetadata{
+			Verb:   types.APIVerbGet,
+			Method: types.HTTPVerbGet,
+			Path: &types.Path{
+				Parent:       basePath,
+				RelativePath: fmt.Sprintf("%s/logs", relPath),
+			},
+			Scopes: []types.PermissionScope{
+				types.UserScope,
+				types.ProjectScope,
+				types.ClusterScope,
+			},
+		},
+	)
+
+	getChartLogsWithinTimeRangeHandler := porter_app.NewGetLogsWithinTimeRangeHandler(
+		config,
+		factory.GetDecoderValidator(),
+		factory.GetResultWriter(),
+	)
+
+	routes = append(routes, &router.Route{
+		Endpoint: getChartLogsWithinTimeRangeEndpoint,
+		Handler:  getChartLogsWithinTimeRangeHandler,
+		Router:   r,
+	})
+
 	return routes, newPath
 }

api/types/incident.go

@@ -112,6 +112,19 @@ type GetLogRequest struct {
 	Direction   string     `schema:"direction"`
 }
 
+// You may either provide the pod selector directly, or the chart name,
+// in which case we will attempt to find the correct pod within the timeframe.
+type GetChartLogsWithinTimeRangeRequest struct {
+	ChartName   string    `schema:"chart_name"`
+	Limit       uint      `schema:"limit"`
+	StartRange  time.Time `schema:"start_range,omitempty"`
+	EndRange    time.Time `schema:"end_range,omitempty"`
+	SearchParam string    `schema:"search_param"`
+	Revision    string    `schema:"revision"`
+	Namespace   string    `schema:"namespace"`
+	PodSelector string    `schema:"pod_selector"`
+}
+
 type GetPodValuesRequest struct {
 	StartRange  *time.Time `schema:"start_range"`
 	EndRange    *time.Time `schema:"end_range"`

dashboard/src/main/home/app-dashboard/expanded-app/ExpandedApp.tsx

@@ -917,24 +917,24 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
                         </>
                       }
                     >
-                        Your build was not successful.
-                        <Spacer inline width="5px" />
-                        <>
-                          <Link
-                            hasunderline
-                            onClick={() => setModalVisible(true)}
-                          >
-                            View logs
-                          </Link>
-                          {modalVisible && (
-                            <GHALogsModal
-                              appData={appData}
-                              logs={logs}
-                              modalVisible={modalVisible}
-                              setModalVisible={setModalVisible}
-                            />
-                          )}
-                        </>
+                      Your build was not successful.
+                      <Spacer inline width="5px" />
+                      <>
+                        <Link
+                          hasunderline
+                          onClick={() => setModalVisible(true)}
+                        >
+                          View logs
+                        </Link>
+                        {modalVisible && (
+                          <GHALogsModal
+                            appData={appData}
+                            logs={logs}
+                            modalVisible={modalVisible}
+                            setModalVisible={setModalVisible}
+                          />
+                        )}
+                      </>
 
                       <Spacer inline width="5px" />
                     </Banner>

dashboard/src/main/home/app-dashboard/expanded-app/activity-feed/events/PreDeployEventCard.tsx

@@ -18,6 +18,7 @@ import Link from "components/porter/Link";
 import LogsModal from "../../status/LogsModal";
 import api from "shared/api";
 import dayjs from "dayjs";
+import Anser from "anser";
 
 type Props = {
   event: PorterAppEvent;
@@ -28,7 +29,7 @@ const PreDeployEventCard: React.FC<Props> = ({ event, appData }) => {
   const [showModal, setShowModal] = useState<boolean>(false);
   const [modalContent, setModalContent] = useState<React.ReactNode>(null);
   const [logModalVisible, setLogModalVisible] = useState(false);
-  const [pods, setPods] = useState([]);
+  const [logs, setLogs] = useState([]);
 
   const renderStatusText = (event: PorterAppEvent) => {
     switch (event.status) {
@@ -41,64 +42,31 @@ const PreDeployEventCard: React.FC<Props> = ({ event, appData }) => {
     }
   };
 
-  const getPodWithCorrectTimestamp = (
-    getJobPodsResponses: any[],
-    start_range: dayjs.Dayjs,
-    end_range: dayjs.Dayjs,
-  ) => {
-    let filteredObjects = getJobPodsResponses.filter(obj =>
-      obj.data &&
-      obj.data.some((d: any) => {
-        return (
-          d?.metadata?.creationTimestamp &&
-          dayjs(d.metadata.creationTimestamp) >= start_range &&
-          dayjs(d.metadata.creationTimestamp) <= end_range
-        );
-      })
-    );
-
-    if (filteredObjects.length === 0) {
-      return undefined;
-    }
-
-    return filteredObjects[filteredObjects.length - 1]
-  }
-
   const getPredeployLogs = async () => {
     setLogModalVisible(true);
     try {
-      // get the pod name
-      const filters = {
-        namespace: appData.releaseChart.namespace,
-        match_prefix: appData.releaseChart.name,
-        start_range: dayjs(event.metadata.start_time).toISOString(),
-        end_range: dayjs(event.metadata.end_time).toISOString(),
-      };
-      const logPodValuesResp = await api.getLogPodValues("<TOKEN>", filters, {
-        project_id: appData.app.project_id,
-        cluster_id: appData.app.cluster_id,
-      });
-      const logPodValues = logPodValuesResp.data;
-
-      if (logPodValues != null && logPodValues.length > 0) {
-        // wheeeee
-        const podNames = logPodValues.map((v: string) => v.split('-hook')[0] + '-hook');
-        const getJobPodsResponses = await Promise.all(podNames.map((podName: string) => api.getJobPods(
-          "<token>",
-          {},
-          {
-            id: appData.app.project_id,
-            name: podName,
-            cluster_id: appData.app.cluster_id,
-            namespace: appData.releaseChart.namespace,
-          },
-        )));
-        const latestPod = getPodWithCorrectTimestamp(getJobPodsResponses, dayjs(event.metadata.start_time), dayjs(event.metadata.end_time));
-        if (latestPod != null) {
-          setPods(latestPod.data);
+      const logResp = await api.getLogsWithinTimeRange(
+        "<token>",
+        {
+          chart_name: appData.releaseChart.name,
+          namespace: appData.releaseChart.namespace,
+          start_range: dayjs(event.metadata.start_time).toISOString(),
+          end_range: dayjs(event.metadata.end_time).toISOString(),
+          limit: 1000,
+        },
+        {
+          project_id: appData.app.project_id,
+          cluster_id: appData.app.cluster_id,
         }
-      }
-
+      )
+      const updatedLogs = logResp.data.logs.map((l: { line: string; timestamp: string; }, index: number) =>
+      ({
+        line: Anser.ansiToJson(l.line),
+        lineNumber: index + 1,
+        timestamp: l.timestamp,
+      }));
+
+      setLogs(updatedLogs);
     } catch (error) {
       console.log(error);
     }
@@ -143,10 +111,9 @@ const PreDeployEventCard: React.FC<Props> = ({ event, appData }) => {
           )}
           {logModalVisible && (
             <LogsModal
-              selectedPod={pods[0]}
-              podError={!pods[0] ? "Pod no longer exists." : ""}
-              setModalVisible={setLogModalVisible}
+              logs={logs}
               logsName={"pre-deploy"}
+              setModalVisible={setLogModalVisible}
             />
           )}
           <Spacer inline x={1} />

dashboard/src/main/home/app-dashboard/expanded-app/status/GHALogsModal.tsx

@@ -88,6 +88,7 @@ const GHALogsModal: React.FC<Props> = ({
     if (!logs) {
       return <Loading />;
     }
+    console.log(logs)
     return (
       <>
         <ExpandedIncidentLogs logs={logs} />

dashboard/src/main/home/app-dashboard/expanded-app/status/LogsModal.tsx

@@ -6,16 +6,13 @@ import Text from "components/porter/Text";
 import danger from "assets/danger.svg";
 
 import ExpandedIncidentLogs from "./ExpandedIncidentLogs";
-import { SelectedPodType } from "./types";
-import { useLogs } from "./useLogs";
 
 interface LogsModalProps {
-    selectedPod: SelectedPodType;
-    podError: string;
-    logsName: string;
+    logs: Log[];
     setModalVisible: (x: boolean) => void;
+    logsName: string;
 }
-const LogsModal: React.FC<LogsModalProps> = ({ selectedPod, setModalVisible, logsName }) => {
+const LogsModal: React.FC<LogsModalProps> = ({ logs, logsName, setModalVisible }) => {
     const scrollToBottomRef = useRef<HTMLDivElement>(null);
     const scrollToBottom = () => {
         if (scrollToBottomRef.current) {
@@ -29,27 +26,13 @@ const LogsModal: React.FC<LogsModalProps> = ({ selectedPod, setModalVisible, log
         scrollToBottom();
     }, [scrollToBottomRef]);
 
-    const { logs } = useLogs(selectedPod, scrollToBottom);
-
-    const renderLogs = (): Log[] => {
-        if (!Array.isArray(logs) || logs?.length === 0) {
-            return (
-                []
-            );
-        }
-
-        return logs.map((log, i) => ({
-            line: log,
-            lineNumber: i + 1,
-        }));
-    };
 
     return (
         <Modal closeModal={() => setModalVisible(false)} width={"800px"}>
             <TitleSection icon={danger}>
                 <Text size={16}>Logs for {logsName}</Text>
             </TitleSection>
-            <ExpandedIncidentLogs logs={renderLogs()} />
+            <ExpandedIncidentLogs logs={logs} />
         </Modal>
     );
 };

dashboard/src/shared/api.tsx

@@ -211,6 +211,27 @@ const deletePorterApp = baseApi<
   return `/api/projects/${project_id}/clusters/${cluster_id}/stacks/${name}`;
 });
 
+const getLogsWithinTimeRange = baseApi<
+  {
+    chart_name: string;
+    limit: number;
+    start_range?: string;
+    end_range?: string;
+    search_param?: string;
+    revision?: string;
+    namespace?: string;
+    pod_selector?: string;
+  },
+  {
+    project_id: number;
+    cluster_id: number;
+  }
+>(
+  "GET",
+  ({ project_id, cluster_id }) =>
+    `/api/projects/${project_id}/clusters/${cluster_id}/stacks/logs`
+);
+
 const getFeedEvents = baseApi<
   {},
   {
@@ -221,9 +242,8 @@ const getFeedEvents = baseApi<
   }
 >("GET", (pathParams) => {
   let { project_id, cluster_id, stack_name, page } = pathParams;
-  return `/api/projects/${project_id}/clusters/${cluster_id}/stacks/${stack_name}/events?page=${
-    page || 1
-  }`;
+  return `/api/projects/${project_id}/clusters/${cluster_id}/stacks/${stack_name}/events?page=${page || 1
+    }`;
 });
 
 const createEnvironment = baseApi<
@@ -648,11 +668,9 @@ const detectBuildpack = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${
-    pathParams.git_repo_id
-  }/repos/${pathParams.kind}/${pathParams.owner}/${
-    pathParams.name
-  }/${encodeURIComponent(pathParams.branch)}/buildpack/detect`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
+    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
+    }/${encodeURIComponent(pathParams.branch)}/buildpack/detect`;
 });
 
 const detectGitlabBuildpack = baseApi<
@@ -683,11 +701,9 @@ const getBranchContents = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${
-    pathParams.git_repo_id
-  }/repos/${pathParams.kind}/${pathParams.owner}/${
-    pathParams.name
-  }/${encodeURIComponent(pathParams.branch)}/contents`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
+    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
+    }/${encodeURIComponent(pathParams.branch)}/contents`;
 });
 
 const getProcfileContents = baseApi<
@@ -703,11 +719,9 @@ const getProcfileContents = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${
-    pathParams.git_repo_id
-  }/repos/${pathParams.kind}/${pathParams.owner}/${
-    pathParams.name
-  }/${encodeURIComponent(pathParams.branch)}/procfile`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
+    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
+    }/${encodeURIComponent(pathParams.branch)}/procfile`;
 });
 
 const getPorterYamlContents = baseApi<
@@ -723,11 +737,9 @@ const getPorterYamlContents = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${
-    pathParams.git_repo_id
-  }/repos/${pathParams.kind}/${pathParams.owner}/${
-    pathParams.name
-  }/${encodeURIComponent(pathParams.branch)}/porteryaml`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
+    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
+    }/${encodeURIComponent(pathParams.branch)}/porteryaml`;
 });
 
 const getGitlabProcfileContents = baseApi<
@@ -1581,11 +1593,9 @@ const getEnvGroup = baseApi<
     version?: number;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.id}/clusters/${
-    pathParams.cluster_id
-  }/namespaces/${pathParams.namespace}/envgroup?name=${pathParams.name}${
-    pathParams.version ? "&version=" + pathParams.version : ""
-  }`;
+  return `/api/projects/${pathParams.id}/clusters/${pathParams.cluster_id
+    }/namespaces/${pathParams.namespace}/envgroup?name=${pathParams.name}${pathParams.version ? "&version=" + pathParams.version : ""
+    }`;
 });
 
 const getConfigMap = baseApi<
@@ -2587,7 +2597,7 @@ const removeStackEnvGroup = baseApi<
     `/api/v1/projects/${project_id}/clusters/${cluster_id}/namespaces/${namespace}/stacks/${stack_id}/remove_env_group/${env_group_name}`
 );
 
-const getGithubStatus = baseApi<{}, {}>("GET", ({}) => `/api/status/github`);
+const getGithubStatus = baseApi<{}, {}>("GET", ({ }) => `/api/status/github`);
 
 const createSecretAndOpenGitHubPullRequest = baseApi<
   {
@@ -2642,10 +2652,12 @@ export default {
   createPasswordResetVerify,
   createPasswordResetFinalize,
   createProject,
+  // PORTER APP
   getPorterApps,
   getPorterApp,
   createPorterApp,
   deletePorterApp,
+  getLogsWithinTimeRange,
   createConfigMap,
   deleteCluster,
   deleteConfigMap,