Merge branch 'master' into nafees/gcr-tags-fix

cli/cmd/apply.go

@@ -173,6 +173,7 @@ type ApplicationConfig struct {
 
 	Build struct {
 		ForceBuild bool
+		ForcePush  bool
 		Method     string
 		Context    string
 		Dockerfile string
@@ -506,7 +507,7 @@ func (d *Driver) updateApplication(resource *models.Resource, client *api.Client
 			return nil, err
 		}
 
-		err = updateAgent.Push()
+		err = updateAgent.Push(appConf.Build.ForcePush)
 
 		if err != nil {
 			return nil, err

cli/cmd/deploy.go

@@ -206,6 +206,7 @@ var dockerfile string
 var method string
 var stream bool
 var buildFlagsEnv []string
+var forcePush bool
 
 func init() {
 	buildFlagsEnv = []string{}
@@ -288,6 +289,20 @@ func init() {
 		"stream update logs to porter dashboard",
 	)
 
+	updateCmd.PersistentFlags().BoolVar(
+		&forceBuild,
+		"force-build",
+		false,
+		"set this to force build an image (images tagged with \"latest\" have this set by default)",
+	)
+
+	updateCmd.PersistentFlags().BoolVar(
+		&forcePush,
+		"force-push",
+		false,
+		"set this to force push an image (images tagged with \"latest\" have this set by default)",
+	)
+
 	updateCmd.AddCommand(updateGetEnvCmd)
 
 	updateGetEnvCmd.PersistentFlags().StringVar(
@@ -297,13 +312,6 @@ func init() {
 		"file destination for .env files",
 	)
 
-	updateCmd.PersistentFlags().BoolVar(
-		&forceBuild,
-		"force-build",
-		false,
-		"set this to force build an image",
-	)
-
 	updateCmd.AddCommand(updateBuildCmd)
 	updateCmd.AddCommand(updatePushCmd)
 	updateCmd.AddCommand(updateConfigCmd)
@@ -523,7 +531,7 @@ func updatePushWithAgent(updateAgent *deploy.DeployAgent) error {
 		})
 	}
 
-	if err := updateAgent.Push(); err != nil {
+	if err := updateAgent.Push(forcePush); err != nil {
 		if stream {
 			updateAgent.StreamEvent(types.SubEvent{
 				EventID: "push",

cli/cmd/deploy/create.go

@@ -273,15 +273,11 @@ func (c *CreateAgent) CreateFromDocker(
 		return "", err
 	}
 
-	imageExists, err := agent.CheckIfImageExists(imageURL, imageTag)
+	imageExists := agent.CheckIfImageExists(imageURL, imageTag)
 
-	if err != nil {
-		return "", err
-	}
-
-	if imageExists && imageTag != "default" && !forceBuild {
+	if imageExists && imageTag != "latest" && !forceBuild {
 		fmt.Printf("%s:%s already exists in the registry, so skipping build\n", imageURL, imageTag)
-	} else { // image does not exist or has tag default so we (re)build one
+	} else { // image does not exist or has tag "latest" so we (re)build one
 		env, err := GetEnvForRelease(c.Client, mergedValues, opts.ProjectID, opts.ClusterID, opts.Namespace)
 
 		if err != nil {

cli/cmd/deploy/deploy.go

@@ -140,6 +140,8 @@ func NewDeployAgent(client *client.Client, app string, opts *DeployOpts) (*Deplo
 	err = coalesceEnvGroups(deployAgent.client, deployAgent.opts.ProjectID, deployAgent.opts.ClusterID,
 		deployAgent.opts.Namespace, deployAgent.opts.EnvGroups, deployAgent.release.Config)
 
+	deployAgent.imageExists = deployAgent.agent.CheckIfImageExists(deployAgent.imageRepo, deployAgent.tag)
+
 	return deployAgent, err
 }
 
@@ -228,14 +230,6 @@ func (d *DeployAgent) Build(overrideBuildConfig *types.BuildConfig, forceBuild b
 		d.tag = currentTag
 	}
 
-	imageExists, err := d.agent.CheckIfImageExists(d.imageRepo, d.tag)
-
-	if err != nil {
-		return err
-	}
-
-	d.imageExists = imageExists
-
 	// we do not want to re-build an image
 	// FIXME: what if overrideBuildConfig == nil but the image stays the same?
 	if overrideBuildConfig == nil && d.imageExists && d.tag != "latest" && !forceBuild {
@@ -245,6 +239,8 @@ func (d *DeployAgent) Build(overrideBuildConfig *types.BuildConfig, forceBuild b
 
 	// if build is not local, fetch remote source
 	var basePath string
+	var err error
+
 	buildCtx := d.opts.LocalPath
 
 	if !d.opts.Local {
@@ -323,7 +319,12 @@ func (d *DeployAgent) Build(overrideBuildConfig *types.BuildConfig, forceBuild b
 }
 
 // Push pushes a local image to the remote repository linked in the release
-func (d *DeployAgent) Push() error {
+func (d *DeployAgent) Push(forcePush bool) error {
+	if d.imageExists && !forcePush && d.tag != "latest" {
+		fmt.Printf("%s:%s has been pushed already, so skipping push\n", d.imageRepo, d.tag)
+		return nil
+	}
+
 	return d.agent.PushImage(fmt.Sprintf("%s:%s", d.imageRepo, d.tag))
 }
 

cli/cmd/docker/agent.go

@@ -173,11 +173,11 @@ func getRegistryRepositoryPair(imageRepo string) ([]string, error) {
 }
 
 // CheckIfImageExists checks if the image exists in the registry
-func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) (bool, error) {
+func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) bool {
 	registryToken, err := a.getContainerRegistryToken(imageRepo)
 
 	if err != nil {
-		return false, err
+		return false
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
@@ -187,13 +187,13 @@ func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) (bool, error) {
 		gcrRegRepo, err := getRegistryRepositoryPair(imageRepo)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		named, err := reference.ParseNamed(imageRepo)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(
@@ -201,7 +201,7 @@ func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) (bool, error) {
 		), nil)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		req.Header.Add("Content-Type", "application/json")
@@ -210,7 +210,7 @@ func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) (bool, error) {
 		resp, err := http.DefaultClient.Do(req)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		defer resp.Body.Close()
@@ -222,21 +222,21 @@ func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) (bool, error) {
 		err = json.NewDecoder(resp.Body).Decode(&tags)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		for _, tag := range tags.Tags {
 			if tag == imageTag {
-				return true, nil
+				return true
 			}
 		}
 
-		return false, nil
+		return false
 	} else if strings.Contains(imageRepo, "registry.digitalocean.com") {
 		doRegRepo, err := getRegistryRepositoryPair(imageRepo)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		doClient := godo.NewFromToken(registryToken)
@@ -246,37 +246,37 @@ func (a *Agent) CheckIfImageExists(imageRepo, imageTag string) (bool, error) {
 		)
 
 		if err != nil {
-			return false, err
+			return false
 		}
 
 		for _, manifest := range manifests {
 			for _, tag := range manifest.Tags {
 				if tag == imageTag {
-					return true, nil
+					return true
 				}
 			}
 		}
 
-		return false, nil
+		return false
 	}
 
 	image := imageRepo + ":" + imageTag
 	encodedRegistryAuth, err := a.getEncodedRegistryAuth(image)
 
 	if err != nil {
-		return false, err
+		return false
 	}
 
 	_, err = a.client.DistributionInspect(context.Background(), image, encodedRegistryAuth)
 
 	if err == nil {
-		return true, nil
+		return true
 	} else if strings.Contains(err.Error(), "image not found") ||
 		strings.Contains(err.Error(), "does not exist in the registry") {
-		return false, nil
+		return false
 	}
 
-	return false, err
+	return false
 }
 
 // PullImage pulls an image specified by the image string

dashboard/src/main/home/cluster-dashboard/env-groups/ExpandedEnvGroup.tsx

@@ -236,43 +236,114 @@ export const ExpandedEnvGroupFC = ({
         setTimeout(() => setButtonStatus(""), 1000);
       }
     } else {
-      const configMapSecretVariables = fillWithDeletedVariables(
-        originalEnvVars.filter((variable) => {
-          return variable.value.includes("PORTERSECRET");
-        }),
-        variables.filter((variable) => {
-          return variable.value.includes("PORTERSECRET") || variable.hidden;
-        })
-      ).reduce(
-        (acc, variable) => ({
-          ...acc,
-          [variable.key]: variable.value,
-        }),
-        {}
+      // SEPARATE THE TWO KINDS OF VARIABLES
+      let secret = variables.filter(
+        (variable) =>
+          variable.hidden && !variable.value.includes("PORTERSECRET")
       );
 
-      const configMapVariables = fillWithDeletedVariables(
-        originalEnvVars,
-        variables.filter(
-          (variable) =>
-            !variable.hidden || !variable.value?.includes("PORTERSECRET")
-        )
-      ).reduce(
-        (acc, variable) => ({
-          ...acc,
-          [variable.key]: variable.value,
-        }),
-        {}
+      let normal = variables.filter(
+        (variable) =>
+          !variable.hidden && !variable.value.includes("PORTERSECRET")
       );
 
+      // Filter variables that weren't updated
+      normal = normal.reduce((acc, variable) => {
+        const originalVar = originalEnvVars.find(
+          (orgVar) => orgVar.key === variable.key
+        );
+
+        // Remove variables that weren't updated
+        if (variable.value === originalVar?.value) {
+          return acc;
+        }
+
+        // add the variable that's going to be updated
+        return [...acc, variable];
+      }, []);
+
+      secret = secret.reduce((acc, variable) => {
+        const originalVar = originalEnvVars.find(
+          (orgVar) => orgVar.key === variable.key
+        );
+
+        // Remove variables that weren't updated
+        if (variable.value === originalVar?.value) {
+          return acc;
+        }
+
+        // add the variable that's going to be updated
+        return [...acc, variable];
+      }, []);
+
+      // Check through the original env vars to see if there's a missing variable, if it is, then means it was removed
+      const removedNormal = originalEnvVars.reduce((acc, orgVar) => {
+        if (orgVar.value.includes("PORTERSECRET")) {
+          return acc;
+        }
+
+        const variableFound = variables.find(
+          (variable) => orgVar.key === variable.key
+        );
+        if (variableFound) {
+          return acc;
+        }
+        return [
+          ...acc,
+          {
+            key: orgVar.key,
+            value: null,
+          },
+        ];
+      }, []);
+
+      const removedSecret = originalEnvVars.reduce((acc, orgVar) => {
+        if (!orgVar.value.includes("PORTERSECRET")) {
+          return acc;
+        }
+
+        const variableFound = variables.find(
+          (variable) => orgVar.key === variable.key
+        );
+        if (variableFound) {
+          return acc;
+        }
+        return [
+          ...acc,
+          {
+            key: orgVar.key,
+            value: null,
+          },
+        ];
+      }, []);
+
+      normal = [...normal, ...removedNormal];
+      secret = [...secret, ...removedSecret];
+
+      const normalObject = normal.reduce((acc, val) => {
+        return {
+          ...acc,
+          [val.key]: val.value,
+        };
+      }, {});
+
+      const secretObject = secret.reduce((acc, val) => {
+        return {
+          ...acc,
+          [val.key]: val.value,
+        };
+      }, {});
+
+      console.log({ normalObject, secretObject });
+
       try {
         const updatedEnvGroup = await api
           .updateConfigMap(
             "<token>",
             {
               name,
-              variables: configMapVariables,
-              secret_variables: configMapSecretVariables,
+              variables: normalObject,
+              secret_variables: secretObject,
             },
             {
               id: currentProject.id,

dashboard/src/main/home/provisioner/AWSFormSection.tsx

@@ -295,9 +295,7 @@ const AWSFormSectionFC: React.FC<PropsType> = (props) => {
       hosting: "aws",
     });
 
-    window.open(
-      "https://docs.getporter.dev/docs/getting-started-with-porter-on-aws"
-    );
+    window.open("https://docs.porter.run/getting-started/provisioning-on-aws");
   };
 
   return (

dashboard/src/main/home/provisioner/GCPFormSection.tsx

@@ -279,7 +279,7 @@ const GCPFormSectionFC: React.FC<PropsType> = (props) => {
       hosting: "gcp",
     });
 
-    window.open("https://docs.getporter.dev/docs/getting-started-on-gcp");
+    window.open("https://docs.porter.run/getting-started/provisioning-on-gcp");
   };
 
   return (

internal/helm/agent.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/porter-dev/porter/internal/helm/loader"
@@ -12,6 +13,7 @@ import (
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/release"
+	"helm.sh/helm/v3/pkg/storage/driver"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/helm/pkg/chartutil"
@@ -207,7 +209,59 @@ func (a *Agent) UpgradeReleaseByValues(
 	res, err := cmd.Run(conf.Name, ch, conf.Values)
 
 	if err != nil {
-		return nil, fmt.Errorf("Upgrade failed: %v", err)
+		// refer: https://github.com/helm/helm/blob/release-3.8/pkg/action/action.go#L62
+		// issue tracker: https://github.com/helm/helm/issues/4558
+		if err.Error() == "another operation (install/upgrade/rollback) is in progress" {
+			secretList, err := a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace).List(
+				context.Background(),
+				v1.ListOptions{
+					LabelSelector: fmt.Sprintf("owner=helm,status in (pending-install, pending-upgrade, pending-rollback),name=%s", rel.Name),
+				},
+			)
+
+			if err != nil {
+				return nil, fmt.Errorf("Upgrade failed: %w", err)
+			}
+
+			if len(secretList.Items) > 0 {
+				mostRecentSecret := secretList.Items[0]
+
+				for i := 1; i < len(secretList.Items); i += 1 {
+					oldVersion, _ := strconv.Atoi(mostRecentSecret.Labels["version"])
+					newVersion, _ := strconv.Atoi(secretList.Items[i].Labels["version"])
+
+					if oldVersion < newVersion {
+						mostRecentSecret = secretList.Items[i]
+					}
+				}
+
+				if time.Since(mostRecentSecret.CreationTimestamp.Time) >= time.Minute {
+					helmSecrets := driver.NewSecrets(a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace))
+
+					rel.Info.Status = release.StatusFailed
+
+					err = helmSecrets.Update(mostRecentSecret.GetName(), rel)
+
+					if err != nil {
+						return nil, fmt.Errorf("Upgrade failed: %w", err)
+					}
+
+					// retry upgrade
+					res, err = cmd.Run(conf.Name, ch, conf.Values)
+
+					if err != nil {
+						return nil, fmt.Errorf("Upgrade failed: %w", err)
+					}
+
+					return res, nil
+				} else {
+					// ask the user to wait for about a minute before retrying for the above fix to kick in
+					return nil, fmt.Errorf("another operation (install/upgrade/rollback) is in progress. If this error persists, please wait for 60 seconds to force an upgrade")
+				}
+			}
+		}
+
+		return nil, fmt.Errorf("Upgrade failed: %w", err)
 	}
 
 	return res, nil

internal/kubernetes/envgroup/create.go

@@ -52,11 +52,8 @@ func ConvertV1ToV2EnvGroup(agent *kubernetes.Agent, name, namespace string) (*v1
 		return nil, err
 	}
 
-	// delete the old configmap and secret
-	if err := agent.DeleteLinkedSecret(name, namespace); err != nil {
-		return nil, err
-	}
-
+	// delete the old configmap
+	// note: we keep the old secret to ensure existing secret references are kept intact
 	if err := agent.DeleteConfigMap(name, namespace); err != nil {
 		return nil, err
 	}