group logs by higher level service, not pod (#3106)

* group logs by higher level service, not pod

* if empty namespace don't create websocket

* remove console log

* remove console log

* disable service filter for live streaming, tweak filter icon ui

* add All to dropdown

* add back white filter icon

* undo change to use effects in agent logs

* undim All

* undim All

---------

Co-authored-by: David Townley <davidtownley@Davids-MacBook-Air.local>
Co-authored-by: Justin Rhee <jusrhee@Justins-MacBook-Air.local>
Co-authored-by: jusrhee <justin@porter.run>

dashboard/src/assets/filter-outline-white.svg

@@ -0,0 +1,3 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="white" xmlns="http://www.w3.org/2000/svg">
+      <path fill-rule="evenodd" clip-rule="evenodd" d="M9.29332 22L14.0696 19.7519V13.8603L21.5593 6.26456C21.8416 5.97995 22 5.58933 22 5.18027V3.51754C22 2.67869 21.3417 2 20.5295 2H3.47049C2.65826 2 2 2.67869 2 3.51754V5.2183C2 5.60431 2.14169 5.97534 2.39719 6.2565L9.29332 13.8603V22Z" stroke="white" stroke-width="1.5" stroke-linecap="round" strokeLinejoin="round"/>
+</svg>

dashboard/src/assets/filter-outline.svg

@@ -1,3 +1,3 @@
 <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M9.29332 22L14.0696 19.7519V13.8603L21.5593 6.26456C21.8416 5.97995 22 5.58933 22 5.18027V3.51754C22 2.67869 21.3417 2 20.5295 2H3.47049C2.65826 2 2 2.67869 2 3.51754V5.2183C2 5.60431 2.14169 5.97534 2.39719 6.2565L9.29332 13.8603V22Z" stroke="white" stroke-width="1.5" stroke-linecap="round" strokeLinejoin="round"/>
+    <path fill-rule="evenodd" clip-rule="evenodd" d="M9.29332 22L14.0696 19.7519V13.8603L21.5593 6.26456C21.8416 5.97995 22 5.58933 22 5.18027V3.51754C22 2.67869 21.3417 2 20.5295 2H3.47049C2.65826 2 2 2.67869 2 3.51754V5.2183C2 5.60431 2.14169 5.97534 2.39719 6.2565L9.29332 13.8603V22Z" stroke="white" stroke-width="1.5" stroke-linecap="round" strokeLinejoin="round"/>
 </svg>

dashboard/src/components/RadioFilter.tsx

@@ -95,7 +95,7 @@ const RadioFilter: React.FC<Props> = (props) => {
         <TextAlt>{props.name}</TextAlt>
         <Bar />
         <Selected>
-          {props.selected
+          {props.selected != null
             ? props.selected === ""
               ? "All"
               : getLabel(props.selected)
@@ -126,6 +126,15 @@ const Selected = styled.div`
   max-width: 120px;
 `;
 
+const DimmedText = styled.div`
+  overflow: hidden;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  word-break: anywhere;
+  margin-right: 10px;
+  color: #999;
+`;
+
 const Text = styled.div`
   overflow: hidden;
   white-space: nowrap;

dashboard/src/main/home/app-dashboard/expanded-app/ExpandedApp.tsx

@@ -69,9 +69,12 @@ const icons = [
 ];
 
 const ExpandedApp: React.FC<Props> = ({ ...props }) => {
-  const { currentCluster, currentProject, setCurrentError, featurePreview } = useContext(
-    Context
-  );
+  const {
+    currentCluster,
+    currentProject,
+    setCurrentError,
+    featurePreview,
+  } = useContext(Context);
   const [isLoading, setIsLoading] = useState(true);
   const [deleting, setDeleting] = useState(false);
   const [appData, setAppData] = useState(null);
@@ -360,8 +363,12 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
               const timestampPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+Z/;
 
               lines.forEach((line, index) => {
-                const lineWithoutTimestamp = line.replace(timestampPattern, "").trimStart();
-                const anserLine: AnserJsonEntry[] = Anser.ansiToJson(lineWithoutTimestamp);
+                const lineWithoutTimestamp = line
+                  .replace(timestampPattern, "")
+                  .trimStart();
+                const anserLine: AnserJsonEntry[] = Anser.ansiToJson(
+                  lineWithoutTimestamp
+                );
                 if (lineWithoutTimestamp.toLowerCase().includes("error")) {
                   anserLine[0].fg = "238,75,43";
                 }
@@ -648,7 +655,7 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
         return (
           <>
             {/* pre-deploy stuff - only if this is from github! */}
-            {!isLoading && appData?.app?.git_repo_id != null &&
+            {!isLoading && appData?.app?.git_repo_id != null && (
               <>
                 <Text size={16}>Pre-deploy job</Text>
                 <Spacer y={0.5} />
@@ -671,7 +678,7 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
                 />
                 <Spacer y={0.5} />
               </>
-            }
+            )}
             <Text size={16}>Application services</Text>
             <Spacer y={0.5} />
             {!isLoading && services.length === 0 && (
@@ -751,7 +758,7 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
           />
         );
       case "logs":
-        return <LogSection currentChart={appData.chart} />;
+        return <LogSection currentChart={appData.chart} services={services} />;
       case "metrics":
         return <MetricsSection currentChart={appData.chart} />;
       case "status":
@@ -778,9 +785,10 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
                   <Container row>
                     <PlaceholderIcon src={notFound} />
                     <Text color="helper">
-                      No pre-deploy jobs were found. You can add a pre-deploy job in the Overview tab to
-                      perform an operation before your application services
-                      deploy each time, like a database migration.
+                      No pre-deploy jobs were found. You can add a pre-deploy
+                      job in the Overview tab to perform an operation before
+                      your application services deploy, like a database
+                      migration.
                     </Text>
                   </Container>
                 </Fieldset>
@@ -956,7 +964,6 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
                           />
                         )}
                       </>
-
                       <Spacer inline width="5px" />
                     </Banner>
                   ) : bannerLoading ? (
@@ -1003,7 +1010,7 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
                     shouldUpdate={
                       appData.chart.latest_version &&
                       appData.chart.latest_version !==
-                      appData.chart.chart.metadata.version
+                        appData.chart.chart.metadata.version
                     }
                     latestVersion={appData.chart.latest_version}
                     upgradeVersion={appUpgradeVersion}
@@ -1035,23 +1042,27 @@ const ExpandedApp: React.FC<Props> = ({ ...props }) => {
                 </Banner>
               </AnimateHeight>
               <TabSelector
-                options={
-                  [
-                    { label: "Overview", value: "overview" },
-                    featurePreview && { label: "Activity", value: "activity" },
-                    hasBuiltImage && { label: "Events", value: "events" },
-                    hasBuiltImage && { label: "Logs", value: "logs" },
-                    hasBuiltImage && { label: "Metrics", value: "metrics" },
-                    hasBuiltImage && { label: "Debug", value: "status" },
-                    appData.app.git_repo_id && { label: "Pre-deploy logs", value: "pre-deploy" },
-                    {
-                      label: "Environment",
-                      value: "environment-variables",
-                    },
-                    appData.app.git_repo_id && { label: "Build settings", value: "build-settings" },
-                    { label: "Settings", value: "settings" },
-                  ].filter(x => x)
-                }
+                options={[
+                  { label: "Overview", value: "overview" },
+                  featurePreview && { label: "Activity", value: "activity" },
+                  hasBuiltImage && { label: "Events", value: "events" },
+                  hasBuiltImage && { label: "Logs", value: "logs" },
+                  hasBuiltImage && { label: "Metrics", value: "metrics" },
+                  hasBuiltImage && { label: "Debug", value: "status" },
+                  appData.app.git_repo_id && {
+                    label: "Pre-deploy logs",
+                    value: "pre-deploy",
+                  },
+                  {
+                    label: "Environment",
+                    value: "environment-variables",
+                  },
+                  appData.app.git_repo_id && {
+                    label: "Build settings",
+                    value: "build-settings",
+                  },
+                  { label: "Settings", value: "settings" },
+                ].filter((x) => x)}
                 currentTab={tab}
                 setCurrentTab={(tab: string) => {
                   if (buttonStatus !== "") {

dashboard/src/main/home/app-dashboard/expanded-app/LogSection.tsx

@@ -11,6 +11,7 @@ import RadioFilter from "components/RadioFilter";
 
 import spinner from "assets/loading.gif";
 import filterOutline from "assets/filter-outline.svg";
+import filterOutlineWhite from "assets/filter-outline-white.svg";
 import time from "assets/time.svg";
 import { Context } from "shared/Context";
 import api from "shared/api";
@@ -29,22 +30,24 @@ import Text from "components/porter/Text";
 import Spacer from "components/porter/Spacer";
 import Container from "components/porter/Container";
 import Button from "components/porter/Button";
+import { Service } from "../new-app-flow/serviceTypes";
 
 type Props = {
   currentChart?: ChartType;
+  services?: Service[];
 };
 
 type PodFilter = {
   podName: string;
-  podNamespace: string;
+  podType: string;
 };
 
-const LogSection: React.FC<Props> = ({ currentChart }) => {
+const LogSection: React.FC<Props> = ({ currentChart, services }) => {
   const scrollToBottomRef = useRef<HTMLDivElement | undefined>(undefined);
   const { currentProject, currentCluster } = useContext(Context);
   const [podFilter, setPodFilter] = useState<PodFilter>({
     podName: "",
-    podNamespace: "",
+    podType: "",
   });
   const [podFilterOpts, setPodFilterOpts] = useState<PodFilter[]>([]);
   const [scrollToBottomEnabled, setScrollToBottomEnabled] = useState(true);
@@ -66,9 +69,12 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
     }, 5000);
   };
 
+  const namespace = currentChart == null ? "" : currentChart.namespace;
+
   const { logs, refresh, moveCursor, paginationInfo } = useLogs(
     podFilter.podName,
-    podFilter.podNamespace,
+    podFilter.podType,
+    namespace,
     enteredSearchText,
     notify,
     currentChart,
@@ -77,63 +83,17 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
   );
 
   const refreshPodLogsValues = async () => {
-    const filters = {
-      namespace: currentChart.namespace,
-      revision: currentChart.version.toString(),
-      match_prefix: currentChart.name,
-    };
-
-    try {
-      const logPodValuesResp = await api.getLogPodValues("<TOKEN>", filters, {
-        project_id: currentProject.id,
-        cluster_id: currentCluster.id,
+    if (currentChart == null || services == null) {
+      setPodFilterOpts([]);
+    } else {
+      const podList = services.map((service: Service) => {
+        return {
+          podName: service.name,
+          podType:
+            service.type.valueOf() == "worker" ? "wkr" : service.type.valueOf(),
+        };
       });
-
-      if (logPodValuesResp.data?.length != 0) {
-        setPodFilterOpts(
-          _.uniq(logPodValuesResp.data ?? []).map((podName: any) => {
-            return { podName: podName, podNamespace: currentChart.namespace };
-          })
-        );
-
-        // only set pod filter if the current pod is not found in the resulting data
-        if (!podFilter || !logPodValuesResp.data?.includes(podFilter)) {
-          setPodFilter({
-            podName: logPodValuesResp.data[0],
-            podNamespace: currentChart.namespace,
-          });
-        }
-        return;
-      }
-
-      // if we're on the latest revision and no pod values were returned, query for all release pods
-      if (currentChart.info.status == "deployed") {
-        const allReleasePodsResp = await api.getAllReleasePods(
-          "<TOKEN>",
-          {},
-          {
-            id: currentProject.id,
-            name: currentChart.name,
-            namespace: currentChart.namespace,
-            cluster_id: currentCluster.id,
-          }
-        );
-
-        let podList = allReleasePodsResp.data.map((pod: any) => {
-          return {
-            podName: pod.metadata.name,
-            podNamespace: pod.metadata.namespace,
-          };
-        });
-
-        setPodFilterOpts(podList);
-
-        if (!podFilter || !podList.includes(podFilter)) {
-          setPodFilter(podList[0]);
-        }
-      }
-    } catch (err) {
-      console.log(err);
+      setPodFilterOpts(podList);
     }
   };
 
@@ -146,6 +106,26 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
     }
   }, [isLoading, logs, scrollToBottomRef, scrollToBottomEnabled]);
 
+  useEffect(() => {
+    if (podFilter.podName != "") {
+      setSelectedDateIfUndefined();
+      return;
+    }
+  }, [podFilter]);
+
+  useEffect(() => {
+    if (selectedDate == null) {
+      resetPodFilter();
+      return;
+    }
+  }, [selectedDate]);
+
+  const resetPodFilter = () => {
+    if (podFilter.podName != "" || podFilter.podType != "") {
+      setPodFilter({ podName: "", podType: "" });
+    }
+  };
+
   const renderLogs = () => {
     return logs?.map((log, i) => {
       return (
@@ -178,11 +158,16 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
   };
 
   const setPodFilterWithPodName = (podName: string) => {
+    if (podName == "All") {
+      resetPodFilter();
+      return;
+    }
+
     const filtered = podFilterOpts.filter((pod) => pod.podName == podName);
     if (filtered.length > 0) {
       setPodFilter(filtered[0]);
     } else {
-      setPodFilter({ podName: "", podNamespace: "" });
+      resetPodFilter();
     }
   };
 
@@ -207,6 +192,14 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
   };
 
   const renderContents = () => {
+    const radioOptions = podFilterOpts?.map((pod) => {
+      return {
+        value: pod.podName,
+        label: pod.podName,
+      };
+    });
+    radioOptions.unshift({ value: "All", label: "All" });
+
     return (
       <>
         <FlexRow>
@@ -223,15 +216,12 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
               resetSearch={resetSearch}
             />
             <RadioFilter
-              icon={filterOutline}
+              icon={
+                podFilter.podName == "" ? filterOutline : filterOutlineWhite
+              }
               selected={podFilter.podName}
               setSelected={setPodFilterWithPodName}
-              options={podFilterOpts?.map((pod) => {
-                return {
-                  value: pod.podName,
-                  label: pod.podName,
-                };
-              })}
+              options={radioOptions}
               name="Filter logs"
             />
           </Flex>
@@ -305,6 +295,7 @@ const LogSection: React.FC<Props> = ({ currentChart }) => {
   useEffect(() => {
     // determine if the agent is installed properly - if not, start by render upgrade screen
     checkForAgent();
+    resetSearch();
   }, []);
 
   useEffect(() => {

dashboard/src/main/home/app-dashboard/expanded-app/useAgentLogs.ts

@@ -71,11 +71,12 @@ interface PaginationInfo {
 }
 
 export const useLogs = (
-  currentPod: string,
+  currentPodName: string,
+  currentPodType: string,
   namespace: string,
   searchParam: string,
   notify: (message: string) => void,
-  currentChart: ChartType,
+  currentChart: ChartType | undefined,
   setLoading: (isLoading: boolean) => void,
   // if setDate is set, results are not live
   setDate?: Date
@@ -91,6 +92,11 @@ export const useLogs = (
     nextCursor: null,
   });
 
+  const currentPod =
+    currentPodName == ""
+      ? currentChart?.name
+      : `${currentChart?.name}-${currentPodName}-${currentPodType}`;
+
   // if we are live:
   // - start date is initially set to 2 weeks ago
   // - the query has an end date set to current date
@@ -185,12 +191,15 @@ export const useLogs = (
   };
 
   const setupWebsocket = (websocketKey: string) => {
+    if (namespace == "") {
+      return;
+    }
+
     const websocketBaseURL = `/api/projects/${currentProject.id}/clusters/${currentCluster.id}/namespaces/${namespace}/logs/loki`;
 
     const q = new URLSearchParams({
-      pod_selector: currentPod,
-      // TODO: re-enable namespace when we properly install stack apps to namespace
-      // namespace,
+      pod_selector: currentPod + "-.*",
+      namespace,
       search_param: searchParam,
       revision: currentChart.version.toString(),
     }).toString();
@@ -236,7 +245,7 @@ export const useLogs = (
       .getLogs(
         "<token>",
         {
-          pod_selector: currentPod,
+          pod_selector: currentPod + "-.*",
           namespace,
           revision: currentChart.version.toString(),
           search_param: searchParam,

dashboard/src/main/home/cluster-dashboard/expanded-chart/logs-section/LogsSection.tsx

@@ -107,16 +107,6 @@ const LogsSection: React.FC<Props> = ({
 }) => {
   const scrollToBottomRef = useRef<HTMLDivElement | undefined>(undefined);
   const { currentProject, currentCluster } = useContext(Context);
-  const [podFilter, setPodFilter] = useState(
-    initData.podName || overridingPodName
-  );
-  const [podFilterOpts, setPodFilterOpts] = useState<string[]>(
-    initData?.podName
-      ? _.compact([initData.podName])
-      : overridingPodName
-      ? _.compact([overridingPodName])
-      : []
-  );
   const [scrollToBottomEnabled, setScrollToBottomEnabled] = useState(true);
   const [searchText, setSearchText] = useState("");
   const [enteredSearchText, setEnteredSearchText] = useState("");
@@ -135,8 +125,8 @@ const LogsSection: React.FC<Props> = ({
   };
 
   const { logs, refresh, moveCursor, paginationInfo } = useLogs(
-    podFilter,
-    currentChart.namespace,
+    "",
+    "",
     enteredSearchText,
     notify,
     currentChart,
@@ -144,74 +134,6 @@ const LogsSection: React.FC<Props> = ({
     selectedDate
   );
 
-  const refreshPodLogsValues = async () => {
-    if (overridingPodName) {
-      return;
-    }
-
-    const filters = {
-      namespace: currentChart.namespace,
-      revision: initData.revision ?? currentChart.version.toString(),
-      match_prefix: currentChart.name,
-    };
-
-    // if the current chart is set to a blue-green deployment, we don't set a revision, but instead
-    // we set the match prefix to the current chart and the active image tag.
-    if (currentChart.config.bluegreen?.enabled) {
-      filters.revision = null;
-
-      if (currentChart?.name.includes("web")) {
-        filters.match_prefix = `${currentChart.name}-${currentChart.config.bluegreen?.activeImageTag}`;
-      } else {
-        filters.match_prefix = `${currentChart.name}-web-${currentChart.config.bluegreen?.activeImageTag}`;
-      }
-    }
-
-    const logPodValuesResp = await api.getLogPodValues("<TOKEN>", filters, {
-      project_id: currentProject.id,
-      cluster_id: currentCluster.id,
-    });
-
-    if (logPodValuesResp.data?.length != 0) {
-      setPodFilterOpts(_.uniq(logPodValuesResp.data ?? []));
-
-      // only set pod filter if the current pod is not found in the resulting data
-      if (!logPodValuesResp.data?.includes(podFilter)) {
-        setPodFilter(logPodValuesResp.data[0]);
-      }
-
-      return;
-    }
-
-    // if we're on the latest revision and no pod values were returned, query for all release pods
-    if (currentChart.info.status == "deployed") {
-      const allReleasePodsResp = await api.getAllReleasePods(
-        "<TOKEN>",
-        {},
-        {
-          id: currentProject.id,
-          name: currentChart.name,
-          namespace: currentChart.namespace,
-          cluster_id: currentCluster.id,
-        }
-      );
-
-      let podList = allReleasePodsResp.data.map((pod: any) => {
-        return pod.metadata.name;
-      });
-
-      setPodFilterOpts(podList);
-
-      if (!podFilter || !podList.includes(podFilter)) {
-        setPodFilter(podList[0]);
-      }
-    }
-  };
-
-  useEffect(() => {
-    refreshPodLogsValues();
-  }, [initData]);
-
   useEffect(() => {
     if (!loading && scrollToBottomRef.current && scrollToBottomEnabled) {
       scrollToBottomRef.current.scrollIntoView({
@@ -222,10 +144,6 @@ const LogsSection: React.FC<Props> = ({
   }, [loading, logs, scrollToBottomRef, scrollToBottomEnabled]);
 
   useEffect(() => {
-    if (initData.podName) {
-      setPodFilter(initData.podName);
-    }
-
     if (initData.timestamp) {
       setSelectedDate(dayjs(initData.timestamp).toDate());
     }
@@ -306,18 +224,6 @@ const LogsSection: React.FC<Props> = ({
               setSelectedDate={setSelectedDate}
               resetSearch={resetSearch}
             />
-            <RadioFilter
-              icon={filterOutline}
-              selected={podFilter}
-              setSelected={setPodFilter}
-              options={podFilterOpts?.map((name) => {
-                return {
-                  value: name,
-                  label: name,
-                };
-              })}
-              name="Filter logs"
-            />
           </Flex>
           <Flex>
             <Button onClick={() => setScrollToBottomEnabled((s) => !s)}>

dashboard/src/main/home/cluster-dashboard/expanded-chart/logs-section/useAgentLogs.ts

@@ -91,6 +91,9 @@ export const useLogs = (
     nextCursor: null,
   });
 
+  currentPod = currentChart.name;
+  namespace = currentChart.namespace;
+
   // if we are live:
   // - start date is initially set to 2 weeks ago
   // - the query has an end date set to current date
@@ -185,11 +188,15 @@ export const useLogs = (
   };
 
   const setupWebsocket = (websocketKey: string) => {
+    if (namespace == "") {
+      return;
+    }
+
     const websocketBaseURL = `/api/projects/${currentProject.id}/clusters/${currentCluster.id}/namespaces/${namespace}/logs/loki`;
 
     const q = new URLSearchParams({
-      pod_selector: currentPod,
-      namespace,
+      pod_selector: currentPod + "-.*",
+      namespace: namespace,
       search_param: searchParam,
       revision: currentChart.version.toString(),
     }).toString();
@@ -235,8 +242,8 @@ export const useLogs = (
       .getLogs(
         "<token>",
         {
-          pod_selector: currentPod,
-          namespace,
+          pod_selector: currentPod + "-.*",
+          namespace: namespace,
           revision: currentChart.version.toString(),
           search_param: searchParam,
           start_range: startDate,