Merge pull request #1685 from porter-dev/nafees/pending-helm-error

[POR-322] Try to circumvent Helm pending upgrade error

internal/helm/agent.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/porter-dev/porter/internal/helm/loader"
@@ -12,6 +13,7 @@ import (
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/release"
+	"helm.sh/helm/v3/pkg/storage/driver"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/helm/pkg/chartutil"
@@ -207,7 +209,59 @@ func (a *Agent) UpgradeReleaseByValues(
 	res, err := cmd.Run(conf.Name, ch, conf.Values)
 
 	if err != nil {
-		return nil, fmt.Errorf("Upgrade failed: %v", err)
+		// refer: https://github.com/helm/helm/blob/release-3.8/pkg/action/action.go#L62
+		// issue tracker: https://github.com/helm/helm/issues/4558
+		if err.Error() == "another operation (install/upgrade/rollback) is in progress" {
+			secretList, err := a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace).List(
+				context.Background(),
+				v1.ListOptions{
+					LabelSelector: fmt.Sprintf("owner=helm,status in (pending-install, pending-upgrade, pending-rollback),name=%s", rel.Name),
+				},
+			)
+
+			if err != nil {
+				return nil, fmt.Errorf("Upgrade failed: %w", err)
+			}
+
+			if len(secretList.Items) > 0 {
+				mostRecentSecret := secretList.Items[0]
+
+				for i := 1; i < len(secretList.Items); i += 1 {
+					oldVersion, _ := strconv.Atoi(mostRecentSecret.Labels["version"])
+					newVersion, _ := strconv.Atoi(secretList.Items[i].Labels["version"])
+
+					if oldVersion < newVersion {
+						mostRecentSecret = secretList.Items[i]
+					}
+				}
+
+				if time.Since(mostRecentSecret.CreationTimestamp.Time) >= time.Minute {
+					helmSecrets := driver.NewSecrets(a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace))
+
+					rel.Info.Status = release.StatusFailed
+
+					err = helmSecrets.Update(mostRecentSecret.GetName(), rel)
+
+					if err != nil {
+						return nil, fmt.Errorf("Upgrade failed: %w", err)
+					}
+
+					// retry upgrade
+					res, err = cmd.Run(conf.Name, ch, conf.Values)
+
+					if err != nil {
+						return nil, fmt.Errorf("Upgrade failed: %w", err)
+					}
+
+					return res, nil
+				} else {
+					// ask the user to wait for about a minute before retrying for the above fix to kick in
+					return nil, fmt.Errorf("another operation (install/upgrade/rollback) is in progress. If this error persists, please wait for 60 seconds to force an upgrade")
+				}
+			}
+		}
+
+		return nil, fmt.Errorf("Upgrade failed: %w", err)
 	}
 
 	return res, nil