Merge branch 'meehawk/logs-improvement' of github.com-meehawk:porter-dev/porter into dev

dashboard/src/main/home/cluster-dashboard/expanded-chart/logs-section/LogsSection.tsx

@@ -15,6 +15,7 @@ import api from "shared/api";
 import { Direction, useLogs } from "./useAgentLogs";
 import Anser from "anser";
 import DateTimePicker from "components/date-time-picker/DateTimePicker";
+import dayjs from "dayjs";
 
 type Props = {
   currentChart?: any;
@@ -40,7 +41,7 @@ const LogsSection: React.FC<Props> = ({
   const [enteredSearchText, setEnteredSearchText] = useState("");
   const [selectedDate, setSelectedDate] = useState<Date | undefined>(undefined);
 
-  const { logs, refresh, moveCursor } = useLogs(
+  const { logs, refresh, moveCursor, paginationInfo } = useLogs(
     podFilter,
     currentChart.namespace,
     enteredSearchText,
@@ -79,6 +80,9 @@ const LogsSection: React.FC<Props> = ({
       return (
         <Log key={[log.lineNumber, i].join(".")}>
           <span className="line-number">{log.lineNumber}.</span>
+          <span className="line-timestamp">
+            {dayjs(log.timestamp).format("MMM D, YYYY HH:mm:ss")}
+          </span>
           {log.line.map((ansi, j) => {
             if (ansi.clearLine) {
               return null;
@@ -158,7 +162,11 @@ const LogsSection: React.FC<Props> = ({
         </FlexRow>
         <StyledLogsSection isFullscreen={isFullscreen}>
           <LoadMoreButton
-            active={selectedDate && logs.length !== 0}
+            active={
+              selectedDate &&
+              logs.length !== 0 &&
+              paginationInfo.previousCursor !== null
+            }
             role="button"
             onClick={() => moveCursor(Direction.backward)}
             ref={scrollToBottomRef}
@@ -436,6 +444,14 @@ const Log = styled.div`
   & > * {
     padding-block: 5px;
   }
+  & > .line-timestamp {
+    height: 100%;
+    color: #949effff;
+    opacity: 0.5;
+    font-family: monospace;
+    min-width: 45px;
+    padding-inline-end: 5px;
+  }
   & > .line-number {
     height: 100%;
     background: #202538;
@@ -449,6 +465,9 @@ const Log = styled.div`
 `;
 
 const LogSpan = styled.span`
+  display: inline-block;
+  word-wrap: break-word;
+  max-width: 80%;
   font-family: monospace, sans-serif;
   font-size: 12px;
   font-weight: ${(props: { ansi: Anser.AnserJsonEntry }) =>

dashboard/src/main/home/cluster-dashboard/expanded-chart/logs-section/useAgentLogs.ts

@@ -5,9 +5,11 @@ import { useContext, useEffect, useRef, useState } from "react";
 import api from "shared/api";
 import { Context } from "shared/Context";
 import { useWebsockets, NewWebsocketOptions } from "shared/hooks/useWebsockets";
+import { isJSON } from "shared/util";
 
 const MAX_LOGS = 5000;
 const MAX_BUFFER_LOGS = 1000;
+const QUERY_LIMIT = 1000;
 
 export enum Direction {
   forward = "forward",
@@ -27,18 +29,30 @@ interface LogLine {
 }
 
 const parseLogs = (logs: string[] = []): Log[] => {
-  return logs.filter(Boolean).map((logLine: string, idx) => {
-    const parsedLine: LogLine = JSON.parse(logLine);
-    // TODO Move log parsing to the render method
-    const ansiLog = Anser.ansiToJson(parsedLine.log);
-    return {
-      line: ansiLog,
-      lineNumber: idx + 1,
-      timestamp: parsedLine.time,
-    };
-  });
+  return logs
+    .filter(Boolean)
+    .filter(isJSON)
+    .map((logLine: string, idx) => {
+      try {
+        const parsedLine: LogLine = JSON.parse(logLine);
+        // TODO Move log parsing to the render method
+        const ansiLog = Anser.ansiToJson(parsedLine.log);
+        return {
+          line: ansiLog,
+          lineNumber: idx + 1,
+          timestamp: parsedLine.time,
+        };
+      } catch (err) {
+        console.error(err, logLine);
+      }
+    });
 };
 
+interface PaginationInfo {
+  previousCursor: string | null;
+  nextCursor: string | null;
+}
+
 export const useLogs = (
   currentPod: string,
   namespace: string,
@@ -50,6 +64,10 @@ export const useLogs = (
   const logsBufferRef = useRef<Log[]>([]);
   const { currentCluster, currentProject } = useContext(Context);
   const [logs, setLogs] = useState<Log[]>([]);
+  const [paginationInfo, setPaginationInfo] = useState<PaginationInfo>({
+    previousCursor: null,
+    nextCursor: null,
+  });
 
   // if we are live:
   // - start date is initially set to 2 weeks ago
@@ -152,7 +170,9 @@ export const useLogs = (
         console.log("Opened websocket:", websocketKey);
       },
       onmessage: (evt: MessageEvent) => {
-        const newLogs = parseLogs(evt?.data?.split("\n"));
+        const newLogs = parseLogs(
+          evt?.data?.split("}\n").map((line: string) => line + "}")
+        );
 
         pushLogs(newLogs);
       },
@@ -165,18 +185,23 @@ export const useLogs = (
     openWebsocket(websocketKey);
   };
 
-  const queryLogs = (startDate: Date, endDate: Date, direction: Direction) => {
+  const queryLogs = (
+    startDate: string,
+    endDate: string,
+    direction: Direction,
+    limit: number = QUERY_LIMIT
+  ) => {
     return api
       .getLogs(
         "<token>",
         {
           pod_selector: currentPod,
-          namespace: namespace,
+          namespace,
           search_param: searchParam,
-          start_range: startDate.toISOString(),
-          end_range: endDate.toISOString(),
-          limit: 1000,
-          direction: direction,
+          start_range: startDate,
+          end_range: endDate,
+          limit,
+          direction,
         },
         {
           cluster_id: currentCluster.id,
@@ -188,7 +213,11 @@ export const useLogs = (
           res.data.logs?.filter(Boolean).map((logLine: any) => logLine.line)
         );
 
-        return newLogs;
+        return {
+          logs: newLogs,
+          previousCursor: res.data.backward_continue_time,
+          nextCursor: res.data.forward_continue_time,
+        };
       });
   };
 
@@ -203,12 +232,17 @@ export const useLogs = (
     const endDate = dayjs(setDate);
     const twoWeeksAgo = endDate.subtract(14, "days");
 
-    const initialLogs = await queryLogs(
-      twoWeeksAgo.toDate(),
-      endDate.toDate(),
+    const { logs: initialLogs, previousCursor, nextCursor } = await queryLogs(
+      twoWeeksAgo.toISOString(),
+      endDate.toISOString(),
       Direction.forward
     );
 
+    setPaginationInfo({
+      previousCursor,
+      nextCursor,
+    });
+
     updateLogs(initialLogs);
 
     closeWebsocket(websocketKey);
@@ -224,17 +258,24 @@ export const useLogs = (
     if (direction === Direction.backward) {
       // we query by setting the endDate equal to the previous startDate, and setting the direction
       // to "backward"
-      const refDate = dayjs(logs[0]?.timestamp);
-      const twoWeeksAgo = refDate.subtract(14, "days");
+      const refDate = paginationInfo.previousCursor ?? dayjs().toISOString();
+      const twoWeeksAgo = dayjs(refDate).subtract(14, "days");
 
-      const newLogs = await queryLogs(
-        twoWeeksAgo.toDate(),
-        refDate.toDate(),
+      const { logs: newLogs, previousCursor } = await queryLogs(
+        twoWeeksAgo.toISOString(),
+        refDate,
         Direction.backward
       );
 
-      // TODO For backwards query, we want to add to the front of the current logs rather than append at the end
-      updateLogs(newLogs);
+      updateLogs(
+        paginationInfo.previousCursor ? newLogs.slice(0, -1) : newLogs,
+        direction
+      );
+
+      setPaginationInfo((paginationInfo) => ({
+        ...paginationInfo,
+        previousCursor,
+      }));
     } else {
       if (isLive) {
         return;
@@ -242,17 +283,22 @@ export const useLogs = (
 
       // we query by setting the startDate equal to the previous endDate, setting the endDate equal to the
       // current time, and setting the direction to "forward"
-      const refDate = logs.length
-        ? dayjs(logs.at(-1).timestamp)
-        : dayjs(setDate);
+      const refDate = paginationInfo.nextCursor ?? dayjs(setDate).toISOString();
       const currDate = dayjs();
-      const newLogs = await queryLogs(
-        refDate.toDate(),
-        currDate.toDate(),
+
+      const { logs: newLogs, nextCursor } = await queryLogs(
+        refDate,
+        currDate.toISOString(),
         Direction.forward
       );
 
-      updateLogs(newLogs);
+      // If previously we had next cursor set, it is likely that the log might have a duplicate entry so we ignore the first line
+      updateLogs(paginationInfo.nextCursor ? newLogs.slice(1) : newLogs);
+
+      setPaginationInfo((paginationInfo) => ({
+        ...paginationInfo,
+        nextCursor,
+      }));
     }
   };
 
@@ -296,5 +342,6 @@ export const useLogs = (
     logs,
     refresh,
     moveCursor,
+    paginationInfo,
   };
 };

dashboard/src/shared/util.ts

@@ -0,0 +1,8 @@
+export const isJSON = (value: string): boolean => {
+  try {
+    JSON.parse(value);
+    return true;
+  } catch (err) {
+    return false;
+  }
+};