implement initial logging integration

api/server/handlers/namespace/stream_pod_logs_loki.go

@@ -0,0 +1,73 @@
+package namespace
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/porter-dev/porter/api/server/authz"
+	"github.com/porter-dev/porter/api/server/handlers"
+	"github.com/porter-dev/porter/api/server/shared"
+	"github.com/porter-dev/porter/api/server/shared/apierrors"
+	"github.com/porter-dev/porter/api/server/shared/config"
+	"github.com/porter-dev/porter/api/server/shared/websocket"
+	"github.com/porter-dev/porter/api/types"
+	"github.com/porter-dev/porter/internal/models"
+)
+
+type StreamPodLogsLokiHandler struct {
+	handlers.PorterHandlerReadWriter
+	authz.KubernetesAgentGetter
+}
+
+func NewStreamPodLogsLokiHandler(
+	config *config.Config,
+	decoderValidator shared.RequestDecoderValidator,
+	writer shared.ResultWriter,
+) *StreamPodLogsLokiHandler {
+	return &StreamPodLogsLokiHandler{
+		PorterHandlerReadWriter: handlers.NewDefaultPorterHandler(config, decoderValidator, writer),
+		KubernetesAgentGetter:   authz.NewOutOfClusterAgentGetter(config),
+	}
+}
+
+func (c *StreamPodLogsLokiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	request := &types.GetLogRequest{}
+
+	if ok := c.DecodeAndValidate(w, r, request); !ok {
+		return
+	}
+
+	safeRW := r.Context().Value(types.RequestCtxWebsocketKey).(*websocket.WebsocketSafeReadWriter)
+
+	cluster, _ := r.Context().Value(types.ClusterScope).(*models.Cluster)
+
+	agent, err := c.GetAgent(r, cluster, "")
+
+	if err != nil {
+		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		return
+	}
+
+	if request.StartRange == nil {
+		dayAgo := time.Now().Add(-24 * time.Hour)
+		request.StartRange = &dayAgo
+	}
+
+	startTime, err := request.StartRange.MarshalText()
+
+	if err != nil {
+		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		return
+	}
+
+	err = agent.StreamPorterAgentLokiLog([]string{
+		fmt.Sprintf("pod=%s", request.PodSelector),
+		fmt.Sprintf("namespace=%s", request.Namespace),
+	}, string(startTime), request.SearchParam, 0, safeRW)
+
+	if err != nil {
+		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
+		return
+	}
+}

api/server/router/namespace.go

@@ -420,6 +420,40 @@ func getNamespaceRoutes(
 		Router:   r,
 	})
 
+	// GET /api/projects/{project_id}/clusters/{cluster_id}/namespaces/{namespace}/logs/loki -> namespace.NewStreamPodLogsLokiHandler
+	streamPodLogsLokiEndpoint := factory.NewAPIEndpoint(
+		&types.APIRequestMetadata{
+			Verb:   types.APIVerbGet,
+			Method: types.HTTPVerbGet,
+			Path: &types.Path{
+				Parent: basePath,
+				RelativePath: fmt.Sprintf(
+					"%s/logs/loki",
+					relPath,
+				),
+			},
+			Scopes: []types.PermissionScope{
+				types.UserScope,
+				types.ProjectScope,
+				types.ClusterScope,
+				types.NamespaceScope,
+			},
+			IsWebsocket: true,
+		},
+	)
+
+	streamPodLogsLokiHandler := namespace.NewStreamPodLogsLokiHandler(
+		config,
+		factory.GetDecoderValidator(),
+		factory.GetResultWriter(),
+	)
+
+	routes = append(routes, &router.Route{
+		Endpoint: streamPodLogsLokiEndpoint,
+		Handler:  streamPodLogsLokiHandler,
+		Router:   r,
+	})
+
 	// GET /api/projects/{project_id}/clusters/{cluster_id}/namespaces/{namespace}/jobs/stream -> namespace.NewStreamJobRunsHandler
 	streamJobRunsEndpoint := factory.NewAPIEndpoint(
 		&types.APIRequestMetadata{

api/types/incident.go

@@ -97,6 +97,7 @@ type GetLogRequest struct {
 	Limit       uint       `schema:"limit"`
 	StartRange  *time.Time `schema:"start_range"`
 	EndRange    *time.Time `schema:"end_range"`
+	SearchParam string     `schema:"search_param"`
 	PodSelector string     `schema:"pod_selector" form:"required"`
 	Namespace   string     `schema:"namespace" form:"required"`
 }

dashboard/src/main/home/cluster-dashboard/expanded-chart/logs-section/LogsSection.tsx

@@ -1,10 +1,15 @@
-import React, { useEffect, useState } from "react";
+import React, { useContext, useEffect, useState } from "react";
 
 import styled from "styled-components";
 import RadioFilter from "components/RadioFilter";
 
 import filterOutline from "assets/filter-outline.svg";
 import downArrow from "assets/down-arrow.svg";
+import { Context } from "shared/Context";
+import api from "shared/api";
+import { useLogs } from "./useAgentLogs";
+import Anser from "anser";
+import { flatMap } from "lodash";
 
 type Props = {
   currentChart?: any;
@@ -12,18 +17,60 @@ type Props = {
   setIsFullscreen: (x: boolean) => void;
 };
 
-const LogsSection: React.FC<Props> = ({ 
+const LogsSection: React.FC<Props> = ({
   currentChart,
   isFullscreen,
-  setIsFullscreen
+  setIsFullscreen,
 }) => {
-  const [podFilter, setPodFilter] = useState("pod-a");
+  const { currentProject, currentCluster } = useContext(Context);
+  const [podFilter, setPodFilter] = useState();
+  const [podFilterOpts, setPodFilterOpts] = useState<string[]>();
   const [scrollToBottom, setScrollToBottom] = useState(true);
 
+  // TODO: don't hardcode namespace
+  const { logs, refresh } = useLogs(podFilter, currentChart.namespace);
+
   useEffect(() => {
+    api
+      .getLogPodValues(
+        "<TOKEN>",
+        {
+          match_prefix: currentChart.name,
+        },
+        {
+          project_id: currentProject.id,
+          cluster_id: currentCluster.id,
+        }
+      )
+      .then((res) => {
+        console.log(res.data);
+        setPodFilterOpts(res.data);
+        setPodFilter(res.data[0]);
+      });
+
     console.log(currentChart);
   }, []);
 
+  const renderLogs = () => {
+    return logs?.map((log, i) => {
+      return (
+        <Log key={i}>
+          {log.map((ansi, j) => {
+            if (ansi.clearLine) {
+              return null;
+            }
+
+            return (
+              <LogSpan key={i + "." + j} ansi={ansi}>
+                {ansi.content.replace(/ /g, "\u00a0")}
+              </LogSpan>
+            );
+          })}
+        </Log>
+      );
+    });
+  };
+
   const renderContents = () => {
     return (
       <>
@@ -34,12 +81,8 @@ const LogsSection: React.FC<Props> = ({
                 <i className="material-icons">search</i>
                 <SearchInput
                   value=""
-                  onChange={(e: any) => {
-
-                  }}
-                  onKeyPress={({ key }) => {
-
-                  }}
+                  onChange={(e: any) => {}}
+                  onKeyPress={({ key }) => {}}
                   placeholder="Search logs . . ."
                 />
               </SearchBarWrapper>
@@ -48,24 +91,12 @@ const LogsSection: React.FC<Props> = ({
               icon={filterOutline}
               selected={podFilter}
               setSelected={setPodFilter}
-              options={[
-                {
-                  value: 'pod-a',
-                  label: 'Pod A'
-                },
-                {
-                  value: 'pod-b',
-                  label: 'Pod B'
-                },
-                {
-                  value: 'pod-c',
-                  label: 'Pod C'
-                },
-                {
-                  value: 'pod-d',
-                  label: 'Pod D'
-                },
-              ]}
+              options={podFilterOpts?.map((name) => {
+                return {
+                  value: name,
+                  label: name,
+                };
+              })}
               name="Filter logs"
             />
           </Flex>
@@ -81,50 +112,46 @@ const LogsSection: React.FC<Props> = ({
               <i className="material-icons">autorenew</i>
               Refresh
             </Button>
-            {
-              !isFullscreen && (
-                <>
-                  <Spacer />
-                  <Icon onClick={() => setIsFullscreen(true)}>
-                    <i className="material-icons">open_in_full</i>
-                  </Icon>
-                </>
-              )
-            }
+            {!isFullscreen && (
+              <>
+                <Spacer />
+                <Icon onClick={() => setIsFullscreen(true)}>
+                  <i className="material-icons">open_in_full</i>
+                </Icon>
+              </>
+            )}
           </Flex>
         </FlexRow>
         <StyledLogsSection isFullscreen={isFullscreen}>
-          <Message>
+          {renderLogs()}
+          {/* <Message>
+            
             No matching logs found.
             <Highlight onClick={() => {}}>
               <i className="material-icons">autorenew</i>
               Refresh
             </Highlight>
-          </Message>
+          </Message> */}
         </StyledLogsSection>
       </>
     );
-  }
+  };
 
   return (
     <>
-      {
-        isFullscreen ? (
-          <Fullscreen>
-            <AbsoluteTitle>
-              <BackButton onClick={() => setIsFullscreen(false)}>
-                <i className="material-icons">navigate_before</i>
-              </BackButton>
-              Logs ({currentChart.name})
-            </AbsoluteTitle>
-            {renderContents()}
-          </Fullscreen>
-        ) : (
-          <>
-            {renderContents()}
-          </>
-        )
-      }
+      {isFullscreen ? (
+        <Fullscreen>
+          <AbsoluteTitle>
+            <BackButton onClick={() => setIsFullscreen(false)}>
+              <i className="material-icons">navigate_before</i>
+            </BackButton>
+            Logs ({currentChart.name})
+          </AbsoluteTitle>
+          {renderContents()}
+        </Fullscreen>
+      ) : (
+        <>{renderContents()}</>
+      )}
     </>
   );
 };
@@ -278,8 +305,8 @@ const FlexRow = styled.div<{ isFullscreen?: boolean }>`
   align-items: center;
   justify-content: space-between;
   flex-wrap: wrap;
-  margin-top: ${props => props.isFullscreen ? "10px" : ""};
-  padding: ${props => props.isFullscreen ? "0 20px" : ""};
+  margin-top: ${(props) => (props.isFullscreen ? "10px" : "")};
+  padding: ${(props) => (props.isFullscreen ? "0 20px" : "")};
 `;
 
 const SearchBarWrapper = styled.div`
@@ -310,7 +337,7 @@ const SearchRow = styled.div`
   align-items: center;
   height: 30px;
   margin-right: 15px;
-  background: #26292E;
+  background: #26292e;
   border-radius: 5px;
   border: 1px solid #aaaabb33;
 `;
@@ -323,19 +350,22 @@ const SearchRowWrapper = styled(SearchRow)`
 const StyledLogsSection = styled.div<{ isFullscreen: boolean }>`
   width: 100%;
   min-height: 400px;
-  height: ${props => props.isFullscreen ? "calc(100vh - 125px)" : "calc(100vh - 460px)"};
+  height: ${(props) =>
+    props.isFullscreen ? "calc(100vh - 125px)" : "calc(100vh - 460px)"};
   display: flex;
   flex-direction: column;
   position: relative;
   font-size: 13px;
-  border-radius: ${props => props.isFullscreen ? "" : "8px"};
-  border: ${props => props.isFullscreen ? "" : "1px solid #ffffff33"};
-  border-top: ${props => props.isFullscreen ? "1px solid #ffffff33" : ""};
+  border-radius: ${(props) => (props.isFullscreen ? "" : "8px")};
+  border: ${(props) => (props.isFullscreen ? "" : "1px solid #ffffff33")};
+  border-top: ${(props) => (props.isFullscreen ? "1px solid #ffffff33" : "")};
   padding: 18px 22px;
   background: #121318;
   animation: floatIn 0.3s;
   animation-timing-function: ease-out;
   animation-fill-mode: forwards;
+  overflow-y: auto;
+  overflow-wrap: break-word;
   @keyframes floatIn {
     from {
       opacity: 0;
@@ -346,4 +376,20 @@ const StyledLogsSection = styled.div<{ isFullscreen: boolean }>`
       transform: translateY(0px);
     }
   }
-`;
+`;
+
+const Log = styled.div`
+  font-family: monospace;
+  user-select: text;
+`;
+
+const LogSpan = styled.span`
+  font-family: monospace, sans-serif;
+  font-size: 12px;
+  font-weight: ${(props: { ansi: Anser.AnserJsonEntry }) =>
+    props.ansi?.decoration && props.ansi?.decoration == "bold" ? "700" : "400"};
+  color: ${(props: { ansi: Anser.AnserJsonEntry }) =>
+    props.ansi?.fg ? `rgb(${props.ansi?.fg})` : "white"};
+  background-color: ${(props: { ansi: Anser.AnserJsonEntry }) =>
+    props.ansi?.bg ? `rgb(${props.ansi?.bg})` : "transparent"};
+`;

dashboard/src/main/home/cluster-dashboard/expanded-chart/logs-section/useAgentLogs.ts

@@ -0,0 +1,115 @@
+import Anser from "anser";
+import { flatMap } from "lodash";
+import { useContext, useEffect, useMemo, useRef, useState } from "react";
+import api from "shared/api";
+import { Context } from "shared/Context";
+import { useWebsockets, NewWebsocketOptions } from "shared/hooks/useWebsockets";
+
+const MAX_LOGS = 250;
+
+export const useLogs = (
+  currentPod: string,
+  namespace: string,
+  searchParam: string,
+  scroll?: (smooth: boolean) => void
+) => {
+  const { currentCluster, currentProject } = useContext(Context);
+  const [logs, setLogs] = useState<Anser.AnserJsonEntry[][]>([]);
+  const [initialized, setInitialized] = useState(false);
+
+  const {
+    newWebsocket,
+    openWebsocket,
+    closeAllWebsockets,
+    getWebsocket,
+    closeWebsocket,
+  } = useWebsockets();
+
+  useEffect(() => {
+    if (!currentPod) {
+      return;
+    }
+    api
+      .getLogs(
+        "<token>",
+        {
+          pod_selector: currentPod,
+          namespace: namespace,
+        },
+        {
+          cluster_id: currentCluster.id,
+          project_id: currentProject.id,
+        }
+      )
+      .then((res) => {
+        console.log(res.data);
+        var initLogs: Anser.AnserJsonEntry[][] = [];
+        res.data.logs?.forEach((logLine: any) => {
+          var parsedLine = JSON.parse(logLine.line);
+
+          let ansiLog = Anser.ansiToJson(parsedLine.log);
+          initLogs.push(ansiLog);
+        });
+
+        setLogs(initLogs.reverse());
+        setInitialized(true);
+        refresh();
+      });
+  }, [currentPod, namespace]);
+
+  const setupWebsocket = (websocketKey: string) => {
+    const endpoint = `/api/projects/${currentProject.id}/clusters/${currentCluster.id}/namespaces/${namespace}/logs/loki?pod_selector=${currentPod}&namespace=${namespace}`;
+
+    const config: NewWebsocketOptions = {
+      onopen: () => {
+        console.log("Opened websocket:", websocketKey);
+      },
+      onmessage: (evt: MessageEvent) => {
+        let newLogs: Anser.AnserJsonEntry[][] = [];
+
+        evt.data.split("\n").forEach((logLine: string) => {
+          if (logLine) {
+            var parsedLine = JSON.parse(logLine);
+
+            let ansiLog = Anser.ansiToJson(parsedLine.log);
+            newLogs.push(ansiLog);
+          }
+        });
+
+        setLogs((prevLogs) => {
+          return prevLogs.concat(newLogs);
+        });
+      },
+      onclose: () => {
+        console.log("Closed websocket:", websocketKey);
+      },
+    };
+
+    newWebsocket(websocketKey, endpoint, config);
+    openWebsocket(websocketKey);
+  };
+
+  const refresh = () => {
+    const websocketKey = `${currentPod}-${namespace}-websocket`;
+    closeWebsocket(websocketKey);
+
+    setupWebsocket(websocketKey);
+  };
+
+  useEffect(() => {
+    if (!currentPod) {
+      return;
+    }
+
+    if (!initialized) {
+      return;
+    }
+
+    refresh();
+  }, [currentPod, namespace]);
+
+  return {
+    logs,
+    refresh,
+  };
+};

dashboard/src/shared/api.tsx

@@ -2005,6 +2005,41 @@ const getGitlabFolderContent = baseApi<
     `/api/projects/${project_id}/integrations/gitlab/${integration_id}/repos/${repo_owner}/${repo_name}/${branch}/contents`
 );
 
+const getLogPodValues = baseApi<
+  {
+    match_prefix?: string;
+    start_range?: string;
+    end_range?: string;
+  },
+  {
+    project_id: number;
+    cluster_id: number;
+  }
+>(
+  "GET",
+  ({ project_id, cluster_id }) =>
+    `/api/projects/${project_id}/clusters/${cluster_id}/logs/pod_values`
+);
+
+const getLogs = baseApi<
+  {
+    limit?: number;
+    start_range?: string;
+    end_range?: string;
+    pod_selector: string;
+    namespace: string;
+    search_param?: string;
+  },
+  {
+    project_id: number;
+    cluster_id: number;
+  }
+>(
+  "GET",
+  ({ project_id, cluster_id }) =>
+    `/api/projects/${project_id}/clusters/${cluster_id}/logs`
+);
+
 // STACKS
 
 const createStack = baseApi<
@@ -2367,6 +2402,8 @@ export default {
   getGitlabRepos,
   getGitlabBranches,
   getGitlabFolderContent,
+  getLogPodValues,
+  getLogs,
 
   // STACKS
   listStacks,

internal/kubernetes/agent.go

@@ -1758,6 +1758,7 @@ func (a *Agent) StreamHelmReleases(namespace string, chartList []string, selecto
 func (a *Agent) StreamPorterAgentLokiLog(
 	labels []string,
 	startTime string,
+	searchParam string,
 	limit uint32,
 	rw *websocket.WebsocketSafeReadWriter,
 ) error {
@@ -1805,7 +1806,7 @@ func (a *Agent) StreamPorterAgentLokiLog(
 			defer wg.Done()
 
 			podList, err := a.Clientset.CoreV1().Pods("porter-agent-system").List(context.Background(), metav1.ListOptions{
-				LabelSelector: "app.kubernetes.io/instance=porter-agent",
+				LabelSelector: "control-plane=controller-manager",
 			})
 
 			if err != nil {
@@ -1835,8 +1836,6 @@ func (a *Agent) StreamPorterAgentLokiLog(
 				SubResource("exec")
 
 			cmd := []string{
-				"sh",
-				"-c",
 				"/porter/agent-cli",
 				"--start",
 				startTime,
@@ -1846,6 +1845,10 @@ func (a *Agent) StreamPorterAgentLokiLog(
 				cmd = append(cmd, "--label", label)
 			}
 
+			if searchParam != "" {
+				cmd = append(cmd, "--search", searchParam)
+			}
+
 			if limit > 0 {
 				cmd = append(cmd, "--limit", fmt.Sprintf("%d", limit))
 			}