POR-1742 read and update env for a revision (#3588)

dashboard/src/lib/hooks/useAppValidation.ts

@@ -70,16 +70,23 @@ export const useAppValidation = ({
       if (!deploymentTargetID) {
         throw new Error("No deployment target selected");
       }
-      const variables = data.app.env
+
+      const { env } = data.app;
+      const variables = env
+        .filter((e) => !e.hidden && !e.deleted)
+        .reduce((acc: Record<string, string>, item) => {
+          acc[item.key] = item.value;
+          return acc;
+        }, {});
+      const secrets = env
         .filter((e) => !e.deleted)
-        .reduce((acc: Record<string, string>, curr) => {
-          acc[curr.key] = curr.value;
+        .reduce((acc: Record<string, string>, item) => {
+          if (item.hidden) {
+            acc[item.key] = item.value;
+          }
           return acc;
         }, {});
-      const envVariableDeletions = removedEnvKeys(
-        variables,
-        prevRevision?.env || {}
-      );
+
       const proto = clientAppToProto(data);
       const commit_sha = await match(data.source)
         .with({ type: "github" }, async (src) => {
@@ -110,7 +117,7 @@ export const useAppValidation = ({
           commit_sha,
           deletions: {
             service_names: data.deletions.serviceNames.map((s) => s.name),
-            env_variable_names: envVariableDeletions,
+            env_variable_names: [],
           },
         },
         {
@@ -129,7 +136,7 @@ export const useAppValidation = ({
         atob(validAppData.validate_b64_app_proto)
       );
 
-      return { validatedAppProto: validatedAppProto, env: data.app.env };
+      return { validatedAppProto: validatedAppProto, variables, secrets };
     },
     [deploymentTargetID, currentProject, currentCluster]
   );

dashboard/src/lib/porter-apps/index.ts

@@ -15,7 +15,7 @@ import {
 } from "./services";
 import { Build, PorterApp, Service } from "@porter-dev/api-contracts";
 import { match } from "ts-pattern";
-import { valueExists } from "shared/util";
+import { KeyValueType } from "main/home/cluster-dashboard/env-groups/EnvGroupArray";
 
 // buildValidator is used to validate inputs for build setting fields
 export const buildValidator = z.discriminatedUnion("method", [
@@ -274,10 +274,17 @@ const clientBuildFromProto = (proto?: Build): BuildOptions | undefined => {
     .exhaustive();
 };
 
-export function clientAppFromProto(
-  proto: PorterApp,
-  overrides: DetectedServices | null
-): ClientPorterApp {
+export function clientAppFromProto({
+  proto,
+  overrides,
+  variables = {},
+  secrets = {},
+}: {
+  proto: PorterApp;
+  overrides: DetectedServices | null;
+  variables?: Record<string, string>;
+  secrets?: Record<string, string>;
+}): ClientPorterApp {
   const services = Object.entries(proto.services)
     .map(([name, service]) => serializedServiceFromProto({ name, service }))
     .map((svc) => {
@@ -295,6 +302,23 @@ export function clientAppFromProto(
     });
 
   const predeployList = [];
+  const parsedEnv: KeyValueType[] = [
+    ...Object.entries(variables).map(([key, value]) => ({
+      key,
+      value,
+      hidden: false,
+      locked: false,
+      deleted: false,
+    })),
+    ...Object.entries(secrets).map(([key, value]) => ({
+      key,
+      value,
+      hidden: true,
+      locked: false,
+      deleted: false,
+    })),
+  ];
+
   if (proto.predeploy) {
     predeployList.push(
       deserializeService({
@@ -314,7 +338,7 @@ export function clientAppFromProto(
       },
       services,
       predeploy: predeployList,
-      env: [],
+      env: parsedEnv,
       envGroups: proto.envGroups.map((eg) => ({
         name: eg.name,
         version: eg.version,
@@ -349,7 +373,7 @@ export function clientAppFromProto(
     },
     services,
     predeploy,
-    env: [],
+    env: parsedEnv,
     envGroups: proto.envGroups.map((eg) => ({
       name: eg.name,
       version: eg.version,

dashboard/src/lib/revisions/types.ts

@@ -16,6 +16,13 @@ export const appRevisionValidator = z.object({
   id: z.string(),
   created_at: z.string(),
   updated_at: z.string(),
+  env: z.object({
+    name: z.string(),
+    latest_version: z.number(),
+    variables: z.record(z.string(), z.string()).optional(),
+    secrets: z.record(z.string(), z.string()).optional(),
+    created_at: z.string(),
+  }),
 });
 
 export type AppRevision = z.infer<typeof appRevisionValidator>;

dashboard/src/main/home/app-dashboard/app-view/AppDataContainer.tsx

@@ -29,6 +29,8 @@ import MetricsTab from "./tabs/MetricsTab";
 import RevisionsList from "../validate-apply/revisions-list/RevisionsList";
 import Activity from "./tabs/Activity";
 import EventFocusView from "./tabs/activity-feed/events/focus-views/EventFocusView";
+import { z } from "zod";
+import { PorterApp } from "@porter-dev/api-contracts";
 
 // commented out tabs are not yet implemented
 // will be included as support is available based on data from app revisions rather than helm releases
@@ -104,7 +106,12 @@ const AppDataContainer: React.FC<AppDataContainerProps> = ({ tabParam }) => {
     reValidateMode: "onSubmit",
     resolver: zodResolver(porterAppFormValidator),
     defaultValues: {
-      app: clientAppFromProto(latestProto, servicesFromYaml),
+      app: clientAppFromProto({
+        proto: latestProto,
+        overrides: servicesFromYaml,
+        variables: latestRevision.env.variables,
+        secrets: latestRevision.env.secrets,
+      }),
       source: latestSource,
       deletions: {
         serviceNames: [],
@@ -151,11 +158,52 @@ const AppDataContainer: React.FC<AppDataContainerProps> = ({ tabParam }) => {
 
   const onSubmit = handleSubmit(async (data) => {
     try {
-      const { validatedAppProto } = await validateApp(data, latestProto);
+      const { validatedAppProto, variables, secrets } = await validateApp(
+        data,
+        latestProto
+      );
+
+      // updates the default env group associated with this app to store app specific env vars
+      const res = await api.updateAppEnvironmentGroup(
+        "<token>",
+        {
+          deployment_target_id: deploymentTargetId,
+          variables,
+          secrets,
+          remove_missing: true,
+        },
+        {
+          project_id: projectId,
+          cluster_id: clusterId,
+          app_name: porterApp.name,
+        }
+      );
+
+      const updatedEnvGroup = z
+        .object({
+          env_group_name: z.string(),
+          env_group_version: z.coerce.bigint(),
+        })
+        .parse(res.data);
+
+      const protoWithUpdatedEnv = new PorterApp({
+        ...validatedAppProto,
+        envGroups: validatedAppProto.envGroups.map((envGroup) => {
+          if (envGroup.name === updatedEnvGroup.env_group_name) {
+            return {
+              ...envGroup,
+              version: updatedEnvGroup.env_group_version,
+            };
+          }
+
+          return envGroup;
+        }),
+      });
+
       await api.applyApp(
         "<token>",
         {
-          b64_app_proto: btoa(validatedAppProto.toJsonString()),
+          b64_app_proto: btoa(protoWithUpdatedEnv.toJsonString()),
           deployment_target_id: deploymentTargetId,
         },
         {
@@ -201,12 +249,17 @@ const AppDataContainer: React.FC<AppDataContainerProps> = ({ tabParam }) => {
 
       // redirect to the default tab after save
       history.push(`/apps/${porterApp.name}/${DEFAULT_TAB}`);
-    } catch (err) { }
+    } catch (err) {}
   });
 
   useEffect(() => {
     reset({
-      app: clientAppFromProto(latestProto, servicesFromYaml),
+      app: clientAppFromProto({
+        proto: latestProto,
+        overrides: servicesFromYaml,
+        variables: latestRevision.env.variables,
+        secrets: latestRevision.env.secrets,
+      }),
       source: latestSource,
       deletions: {
         serviceNames: [],
@@ -265,11 +318,11 @@ const AppDataContainer: React.FC<AppDataContainerProps> = ({ tabParam }) => {
             { label: "Environment", value: "environment" },
             ...(latestProto.build
               ? [
-                {
-                  label: "Build Settings",
-                  value: "build-settings",
-                },
-              ]
+                  {
+                    label: "Build Settings",
+                    value: "build-settings",
+                  },
+                ]
               : []),
             { label: "Settings", value: "settings" },
           ]}

dashboard/src/main/home/app-dashboard/create-app/CreateApp.tsx

@@ -44,7 +44,6 @@ import { useAppValidation } from "lib/hooks/useAppValidation";
 import { useQuery } from "@tanstack/react-query";
 import { z } from "zod";
 import PorterYamlModal from "./PorterYamlModal";
-import { KeyValueType } from "main/home/cluster-dashboard/env-groups/EnvGroupArray";
 
 type CreateAppProps = {} & RouteComponentProps;
 
@@ -67,6 +66,13 @@ const CreateApp: React.FC<CreateAppProps> = ({ history }) => {
   ] = React.useState<PorterApp | null>(null);
   const [isDeploying, setIsDeploying] = React.useState(false);
   const [deployError, setDeployError] = React.useState("");
+  const [{ variables, secrets }, setFinalizedAppEnv] = React.useState<{
+    variables: Record<string, string>;
+    secrets: Record<string, string>;
+  }>({
+    variables: {},
+    secrets: {},
+  });
 
   const { data: porterApps = [] } = useQuery<string[]>(
     ["getPorterApps", currentProject?.id, currentCluster?.id],
@@ -140,7 +146,6 @@ const CreateApp: React.FC<CreateAppProps> = ({ history }) => {
   const build = watch("app.build");
   const image = watch("source.image");
   const services = watch("app.services");
-  const env = watch("app.env");
 
   const {
     detectedServices: servicesFromYaml,
@@ -158,15 +163,21 @@ const CreateApp: React.FC<CreateAppProps> = ({ history }) => {
   const onSubmit = handleSubmit(async (data) => {
     try {
       setDeployError("");
-      const { validatedAppProto } = await validateApp(data);
+      const { validatedAppProto, variables, secrets } = await validateApp(data);
       setValidatedAppProto(validatedAppProto);
+      setFinalizedAppEnv({ variables, secrets });
 
       if (source.type === "github") {
         setShowGHAModal(true);
         return;
       }
 
-      await createAndApply({ app: validatedAppProto, source, env });
+      await createAndApply({
+        app: validatedAppProto,
+        source,
+        variables,
+        secrets,
+      });
     } catch (err) {
       if (axios.isAxiosError(err) && err.response?.data?.error) {
         setDeployError(err.response?.data?.error);
@@ -182,11 +193,13 @@ const CreateApp: React.FC<CreateAppProps> = ({ history }) => {
     async ({
       app,
       source,
-      env,
+      variables,
+      secrets,
     }: {
       app: PorterApp | null;
       source: SourceOptions;
-      env: KeyValueType[];
+      variables: Record<string, string>;
+      secrets: Record<string, string>;
     }) => {
       setIsDeploying(true);
       // log analytics event that we started form submission
@@ -213,20 +226,6 @@ const CreateApp: React.FC<CreateAppProps> = ({ history }) => {
           }
         );
 
-        const variables = env
-          .filter((e) => !e.hidden && !e.deleted)
-          .reduce((acc: Record<string, string>, item) => {
-            acc[item.key] = item.value;
-            return acc;
-          }, {});
-        const secrets = env
-          .filter((e) => !e.deleted)
-          .reduce((acc: Record<string, string>, item) => {
-            if (item.hidden) {
-              acc[item.key] = item.value;
-            }
-            return acc;
-          }, {});
         const envGroupResponse = await api.updateEnvironmentGroupV2(
           "<token>",
           {
@@ -622,7 +621,12 @@ const CreateApp: React.FC<CreateAppProps> = ({ history }) => {
           projectId={currentProject.id}
           clusterId={currentCluster.id}
           deployPorterApp={() =>
-            createAndApply({ app: validatedAppProto, source, env })
+            createAndApply({
+              app: validatedAppProto,
+              source,
+              variables,
+              secrets,
+            })
           }
           deploymentError={deployError}
           porterYamlPath={source.porter_yaml_path}

dashboard/src/main/home/app-dashboard/validate-apply/revisions-list/RevisionTableContents.tsx

@@ -23,6 +23,8 @@ type RevisionTableContentsProps = {
     SetStateAction<{
       app: PorterApp;
       revision: number;
+      variables: Record<string, string>;
+      secrets: Record<string, string>;
     } | null>
   >;
 };
@@ -184,10 +186,12 @@ const RevisionTableContents: React.FC<RevisionTableContentsProps> = ({
                     }
                     onClick={() => {
                       reset({
-                        app: clientAppFromProto(
-                          revision.app_proto,
-                          servicesFromYaml
-                        ),
+                        app: clientAppFromProto({
+                          proto: revision.app_proto,
+                          overrides: servicesFromYaml,
+                          variables: revision.env.variables,
+                          secrets: revision.env.secrets,
+                        }),
                         source: latestSource,
                         deletions: {
                           serviceNames: [],
@@ -229,6 +233,8 @@ const RevisionTableContents: React.FC<RevisionTableContentsProps> = ({
                           setRevertData({
                             app: revision.app_proto,
                             revision: revision.revision_number,
+                            variables: revision.env.variables ?? {},
+                            secrets: revision.env.secrets ?? {},
                           });
                         }}
                       >

dashboard/src/main/home/app-dashboard/validate-apply/revisions-list/RevisionsList.tsx

@@ -44,6 +44,8 @@ const RevisionsList: React.FC<Props> = ({
   const [revertData, setRevertData] = useState<{
     app: PorterApp;
     revision: number;
+    variables: Record<string, string>;
+    secrets: Record<string, string>;
   } | null>(null);
 
   const res = useQuery(
@@ -76,7 +78,14 @@ const RevisionsList: React.FC<Props> = ({
       return;
     }
 
-    setValue("app", clientAppFromProto(revertData.app, servicesFromYaml));
+    setValue(
+      "app",
+      clientAppFromProto({
+        proto: revertData.app,
+        overrides: servicesFromYaml,
+        
+      })
+    );
     setRevertData(null);
 
     void onSubmit();

dashboard/src/shared/api.tsx

@@ -310,8 +310,9 @@ const getFeedEvents = baseApi<
   }
 >("GET", (pathParams) => {
   let { project_id, cluster_id, stack_name, page } = pathParams;
-  return `/api/projects/${project_id}/clusters/${cluster_id}/applications/${stack_name}/events?page=${page || 1
-    }`;
+  return `/api/projects/${project_id}/clusters/${cluster_id}/applications/${stack_name}/events?page=${
+    page || 1
+  }`;
 });
 
 const createEnvironment = baseApi<
@@ -736,9 +737,11 @@ const detectBuildpack = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
-    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
-    }/${encodeURIComponent(pathParams.branch)}/buildpack/detect`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${
+    pathParams.git_repo_id
+  }/repos/${pathParams.kind}/${pathParams.owner}/${
+    pathParams.name
+  }/${encodeURIComponent(pathParams.branch)}/buildpack/detect`;
 });
 
 const detectGitlabBuildpack = baseApi<
@@ -769,9 +772,11 @@ const getBranchContents = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
-    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
-    }/${encodeURIComponent(pathParams.branch)}/contents`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${
+    pathParams.git_repo_id
+  }/repos/${pathParams.kind}/${pathParams.owner}/${
+    pathParams.name
+  }/${encodeURIComponent(pathParams.branch)}/contents`;
 });
 
 const getProcfileContents = baseApi<
@@ -787,9 +792,11 @@ const getProcfileContents = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
-    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
-    }/${encodeURIComponent(pathParams.branch)}/procfile`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${
+    pathParams.git_repo_id
+  }/repos/${pathParams.kind}/${pathParams.owner}/${
+    pathParams.name
+  }/${encodeURIComponent(pathParams.branch)}/procfile`;
 });
 
 const getPorterYamlContents = baseApi<
@@ -805,9 +812,11 @@ const getPorterYamlContents = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
-    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
-    }/${encodeURIComponent(pathParams.branch)}/porteryaml`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${
+    pathParams.git_repo_id
+  }/repos/${pathParams.kind}/${pathParams.owner}/${
+    pathParams.name
+  }/${encodeURIComponent(pathParams.branch)}/porteryaml`;
 });
 
 const parsePorterYaml = baseApi<
@@ -843,9 +852,11 @@ const getBranchHead = baseApi<
     branch: string;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.project_id}/gitrepos/${pathParams.git_repo_id
-    }/repos/${pathParams.kind}/${pathParams.owner}/${pathParams.name
-    }/${encodeURIComponent(pathParams.branch)}/head`;
+  return `/api/projects/${pathParams.project_id}/gitrepos/${
+    pathParams.git_repo_id
+  }/repos/${pathParams.kind}/${pathParams.owner}/${
+    pathParams.name
+  }/${encodeURIComponent(pathParams.branch)}/head`;
 });
 
 const validatePorterApp = baseApi<
@@ -868,21 +879,21 @@ const validatePorterApp = baseApi<
 
 const createApp = baseApi<
   | {
-    name: string;
-    type: "github";
-    git_repo_id: number;
-    git_branch: string;
-    git_repo_name: string;
-    porter_yaml_path: string;
-  }
+      name: string;
+      type: "github";
+      git_repo_id: number;
+      git_branch: string;
+      git_repo_name: string;
+      porter_yaml_path: string;
+    }
   | {
-    name: string;
-    type: "docker-registry";
-    image: {
-      repository: string;
-      tag: string;
-    };
-  },
+      name: string;
+      type: "docker-registry";
+      image: {
+        repository: string;
+        tag: string;
+      };
+    },
   {
     project_id: number;
     cluster_id: number;
@@ -891,6 +902,22 @@ const createApp = baseApi<
   return `/api/projects/${pathParams.project_id}/clusters/${pathParams.cluster_id}/apps/create`;
 });
 
+const updateAppEnvironmentGroup = baseApi<
+  {
+    deployment_target_id: string;
+    variables: Record<string, string>;
+    secrets: Record<string, string>;
+    remove_missing: boolean;
+  },
+  {
+    project_id: number;
+    cluster_id: number;
+    app_name: string;
+  }
+>("POST", (pathParams) => {
+  return `/api/projects/${pathParams.project_id}/clusters/${pathParams.cluster_id}/apps/${pathParams.app_name}/update-environment`;
+});
+
 const applyApp = baseApi<
   {
     deployment_target_id: string;
@@ -932,12 +959,14 @@ const listAppRevisions = baseApi<
 });
 
 const getLatestAppRevisions = baseApi<
-  {}, {
+  {},
+  {
     project_id: number;
     cluster_id: number;
-  }>("GET", ({ project_id, cluster_id }) => {
-    return `/api/projects/${project_id}/clusters/${cluster_id}/apps/revisions`;
-  })
+  }
+>("GET", ({ project_id, cluster_id }) => {
+  return `/api/projects/${project_id}/clusters/${cluster_id}/apps/revisions`;
+});
 
 const getGitlabProcfileContents = baseApi<
   {
@@ -1842,9 +1871,11 @@ const getEnvGroup = baseApi<
     version?: number;
   }
 >("GET", (pathParams) => {
-  return `/api/projects/${pathParams.id}/clusters/${pathParams.cluster_id
-    }/namespaces/${pathParams.namespace}/envgroup?name=${pathParams.name}${pathParams.version ? "&version=" + pathParams.version : ""
-    }`;
+  return `/api/projects/${pathParams.id}/clusters/${
+    pathParams.cluster_id
+  }/namespaces/${pathParams.namespace}/envgroup?name=${pathParams.name}${
+    pathParams.version ? "&version=" + pathParams.version : ""
+  }`;
 });
 
 const getConfigMap = baseApi<
@@ -2901,7 +2932,7 @@ const removeStackEnvGroup = baseApi<
     `/api/v1/projects/${project_id}/clusters/${cluster_id}/namespaces/${namespace}/stacks/${stack_id}/remove_env_group/${env_group_name}`
 );
 
-const getGithubStatus = baseApi<{}, {}>("GET", ({ }) => `/api/status/github`);
+const getGithubStatus = baseApi<{}, {}>("GET", ({}) => `/api/status/github`);
 
 const createSecretAndOpenGitHubPullRequest = baseApi<
   {
@@ -3052,6 +3083,7 @@ export default {
   getBranchHead,
   validatePorterApp,
   createApp,
+  updateAppEnvironmentGroup,
   applyApp,
   getLatestRevision,
   listAppRevisions,

internal/kubernetes/environment_groups/list.go

@@ -29,9 +29,9 @@ type EnvironmentGroup struct {
 	// Version is the environment group version which can be found in the labels (LabelKey_EnvironmentGroupVersion) of the ConfigMap. This is NOT included in the configmap name
 	Version int `json:"latest_version"`
 	// Variables are non-secret values for the EnvironmentGroup. This usually will be a configmap
-	Variables map[string]string `json:"variables"`
+	Variables map[string]string `json:"variables,omitempty"`
 	// SecretVariables are secret values for the EnvironmentGroup. This usually will be a Secret on the kubernetes cluster
-	SecretVariables map[string][]byte `json:"variables_secrets,omitempty"`
+	SecretVariables map[string][]byte `json:"secrets,omitempty"`
 	// CreatedAt is only used for display purposes and is in UTC Unix time
 	CreatedAtUTC time.Time `json:"created_at"`
 }