package helm import ( "bytes" "context" "fmt" "runtime/debug" "strconv" "strings" "time" "github.com/porter-dev/porter/internal/telemetry" "github.com/pkg/errors" "github.com/porter-dev/porter/internal/helm/loader" "github.com/stefanmcshane/helm/pkg/action" "github.com/stefanmcshane/helm/pkg/chart" "github.com/stefanmcshane/helm/pkg/release" "github.com/stefanmcshane/helm/pkg/storage/driver" "golang.org/x/oauth2" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/helm/pkg/chartutil" "github.com/porter-dev/porter/api/types" "github.com/porter-dev/porter/internal/kubernetes" "github.com/porter-dev/porter/internal/models" "github.com/porter-dev/porter/internal/repository" ) // Agent is a Helm agent for performing helm operations type Agent struct { ActionConfig *action.Configuration K8sAgent *kubernetes.Agent // The namespace struct attribute is unexported to avoid cases // where a developer might change this, thinking that it will // apply to all api interactions. RESTClientGetter has an immutable // copy of the value, so change this won't impact those requests. namespace string } // Namespace returns the configured namespace func (a *Agent) Namespace() string { return a.namespace } // ListReleases lists releases based on a ListFilter func (a *Agent) ListReleases( ctx context.Context, namespace string, filter *types.ReleaseListFilter, ) ([]*release.Release, error) { ctx, span := telemetry.NewSpan(ctx, "helm-list-releases") defer span.End() stringFilter := strings.Join(filter.StatusFilter, ",") telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "namespace", Value: namespace}, telemetry.AttributeKV{Key: "filter", Value: stringFilter}, ) lsel := fmt.Sprintf("owner=helm,status in (%s)", stringFilter) secretList, err := a.K8sAgent.Clientset.CoreV1().Secrets(namespace).List( ctx, v1.ListOptions{ LabelSelector: lsel, }, ) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting secret list") } // before decoding to helm release, only keep the latest releases for each chart latestMap := make(map[string]corev1.Secret) for _, secret := range secretList.Items { relName, relNameExists := secret.Labels["name"] if !relNameExists { continue } id := fmt.Sprintf("%s/%s", secret.Namespace, relName) if currLatest, exists := latestMap[id]; exists { // get version currVersionStr, currVersionExists := currLatest.Labels["version"] versionStr, versionExists := secret.Labels["version"] if versionExists && currVersionExists { currVersion, currErr := strconv.Atoi(currVersionStr) version, err := strconv.Atoi(versionStr) if currErr == nil && err == nil && currVersion < version { latestMap[id] = secret } } } else { latestMap[id] = secret } } chartList := []string{} res := make([]*release.Release, 0) for _, secret := range latestMap { rel, isErr, err := kubernetes.ParseSecretToHelmRelease(secret, chartList) if !isErr && err == nil { res = append(res, rel) } } return res, nil } // GetRelease returns the info of a release. func (a *Agent) GetRelease( ctx context.Context, name string, version int, getDeps bool, ) (*release.Release, error) { ctx, span := telemetry.NewSpan(ctx, "helm-get-release") defer span.End() // This span is one of most frequent spans. We need to sample this. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "name", Value: name}, telemetry.AttributeKV{Key: "namespace", Value: a.Namespace()}, telemetry.AttributeKV{Key: "version", Value: version}, telemetry.AttributeKV{Key: "getDeps", Value: getDeps}, ) if version == 0 && a.Namespace() != "" { version = a.getLatestReleaseVersion(ctx, name) telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "computed-version", Value: version}) } // Namespace is already known by the RESTClientGetter. cmd := action.NewGet(a.ActionConfig) cmd.Version = version release, err := cmd.Run(name) if err != nil { return nil, telemetry.Error(ctx, span, err, "error running get release") } if getDeps && release.Chart != nil && release.Chart.Metadata != nil { for _, dep := range release.Chart.Metadata.Dependencies { // only search for dependency if it passes the condition specified in Chart.yaml if dep.Enabled { depExists := false for _, currDep := range release.Chart.Dependencies() { // we just case on name for now -- there might be edge cases we're missing // but this will cover 99% of cases if dep != nil && currDep != nil && dep.Name == currDep.Name() { depExists = true break } } if !depExists { depChart, err := loader.LoadChartPublic(ctx, dep.Repository, dep.Name, dep.Version) if err != nil { return nil, telemetry.Error(ctx, span, err, fmt.Sprintf("Error retrieving chart dependency %s/%s-%s", dep.Repository, dep.Name, dep.Version)) } release.Chart.AddDependency(depChart) } } } } return release, err } // getLatestReleaseVersion retrieves the actual number for the last helm release // for a given name. // // If we use helm's built-in method of retrieving the last // release, it will retrieve _all_ releases and then only return the last one. // That works ~fine except in cases where you have hundreds of releases, // in which case any api calls will take significantly longer. // // Instead, we retrieve all non-supersceded versions, then grab the highest // version in that list. In the worst case, this would also retrieve every // release. In the best case, it only retrieves a single release (the latest). // // The ideal case would be if we could sort on the server side, in which case // we could Limit results by 1 and not worry about the helm release status label, // but Kubernetes only supports ordering results client-side. func (a *Agent) getLatestReleaseVersion(ctx context.Context, name string) int { ctx, span := telemetry.NewSpan(ctx, "helm-get-latest-release-version") defer span.End() // This span is one of most frequent spans. We need to sample this. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "name", Value: name}, telemetry.AttributeKV{Key: "namespace", Value: a.Namespace()}, ) helmStatuses := []string{ string(release.StatusDeployed), string(release.StatusFailed), string(release.StatusPendingInstall), string(release.StatusPendingRollback), string(release.StatusPendingUpgrade), string(release.StatusUninstalled), string(release.StatusUninstalling), string(release.StatusUnknown), } var labelSelectors []string labelSelectors = append(labelSelectors, fmt.Sprintf("name in (%s)", name)) labelSelectors = append(labelSelectors, "owner in (helm)") labelSelectors = append(labelSelectors, fmt.Sprintf("status in (%s)", strings.Join(helmStatuses, ","))) listOptions := v1.ListOptions{ LabelSelector: strings.Join(labelSelectors, ","), } client, _ := a.ActionConfig.KubernetesClientSet() secretListResp, err := client.CoreV1().Secrets(a.Namespace()).List(ctx, listOptions) version := 0 if err == nil { for _, secret := range secretListResp.Items { versionString, ok := secret.Labels["version"] if !ok { continue // missing version label, not a helm release } // errors in conversion will implicitly set version to 0 // which is the original value secretVersion, _ := strconv.Atoi(versionString) if secretVersion > version { version = secretVersion } } } return version } // DeleteReleaseRevision deletes a specific revision of a release func (a *Agent) DeleteReleaseRevision( ctx context.Context, name string, version int, ) error { ctx, span := telemetry.NewSpan(ctx, "helm-delete-release-history") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "name", Value: name}, telemetry.AttributeKV{Key: "version", Value: version}, ) _, err := a.ActionConfig.Releases.Delete(name, version) return err } // GetReleaseHistory returns a list of charts for a specific release func (a *Agent) GetReleaseHistory( ctx context.Context, name string, ) ([]*release.Release, error) { ctx, span := telemetry.NewSpan(ctx, "helm-get-release-history") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "name", Value: name}, ) cmd := action.NewHistory(a.ActionConfig) return cmd.Run(name) } type UpgradeReleaseConfig struct { Name string Values map[string]interface{} Cluster *models.Cluster Repo repository.Repository Registries []*models.Registry // Optional, if chart should be overriden Chart *chart.Chart // Optional, if chart is part of a Porter Stack StackName string StackRevision uint } // UpgradeRelease upgrades a specific release with new values.yaml func (a *Agent) UpgradeRelease( ctx context.Context, conf *UpgradeReleaseConfig, values string, doAuth *oauth2.Config, disablePullSecretsInjection bool, ignoreDependencies bool, ) (*release.Release, error) { ctx, span := telemetry.NewSpan(ctx, "helm-upgrade-release") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "project-id", Value: conf.Cluster.ProjectID}, telemetry.AttributeKV{Key: "cluster-id", Value: conf.Cluster.ID}, telemetry.AttributeKV{Key: "name", Value: conf.Name}, telemetry.AttributeKV{Key: "stack-name", Value: conf.StackName}, telemetry.AttributeKV{Key: "stack-revision", Value: conf.StackRevision}, ) valuesYaml, err := chartutil.ReadValues([]byte(values)) if err != nil { return nil, telemetry.Error(ctx, span, err, "Values could not be parsed") } conf.Values = valuesYaml return a.UpgradeReleaseByValues(ctx, conf, doAuth, disablePullSecretsInjection, ignoreDependencies) } // UpgradeReleaseByValues upgrades a release by unmarshaled yaml values func (a *Agent) UpgradeReleaseByValues( ctx context.Context, conf *UpgradeReleaseConfig, doAuth *oauth2.Config, disablePullSecretsInjection bool, ignoreDependencies bool, ) (*release.Release, error) { ctx, span := telemetry.NewSpan(ctx, "helm-upgrade-release-by-values") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "project-id", Value: conf.Cluster.ProjectID}, telemetry.AttributeKV{Key: "cluster-id", Value: conf.Cluster.ID}, telemetry.AttributeKV{Key: "name", Value: conf.Name}, telemetry.AttributeKV{Key: "stack-name", Value: conf.StackName}, telemetry.AttributeKV{Key: "stack-revision", Value: conf.StackRevision}, ) // grab the latest release rel, err := a.GetRelease(ctx, conf.Name, 0, !ignoreDependencies) if err != nil { return nil, telemetry.Error(ctx, span, err, "Could not get release to be upgraded") } ch := rel.Chart if conf.Chart != nil { ch = conf.Chart } cmd := action.NewUpgrade(a.ActionConfig) cmd.Namespace = rel.Namespace cmd.PostRenderer, err = NewPorterPostrenderer( conf.Cluster, conf.Repo, a.K8sAgent, rel.Namespace, conf.Registries, doAuth, disablePullSecretsInjection, ) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting porter postrenderer") } if conf.StackName != "" && conf.StackRevision > 0 { conf.Values["stack"] = map[string]interface{}{ "enabled": true, "name": conf.StackName, "revision": conf.StackRevision, } } res, err := cmd.Run(conf.Name, ch, conf.Values) if err != nil { // refer: https://github.com/helm/helm/blob/release-3.8/pkg/action/action.go#L62 // issue tracker: https://github.com/helm/helm/issues/4558 if err.Error() == "another operation (install/upgrade/rollback) is in progress" { secretList, err := a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace).List( context.Background(), v1.ListOptions{ LabelSelector: fmt.Sprintf("owner=helm,status in (pending-install, pending-upgrade, pending-rollback),name=%s", rel.Name), }, ) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting secret list") } if len(secretList.Items) > 0 { mostRecentSecret := secretList.Items[0] for i := 1; i < len(secretList.Items); i += 1 { oldVersion, _ := strconv.Atoi(mostRecentSecret.Labels["version"]) newVersion, _ := strconv.Atoi(secretList.Items[i].Labels["version"]) if oldVersion < newVersion { mostRecentSecret = secretList.Items[i] } } if time.Since(mostRecentSecret.CreationTimestamp.Time) >= time.Minute { helmSecrets := driver.NewSecrets(a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace)) rel.Info.Status = release.StatusFailed err = helmSecrets.Update(mostRecentSecret.GetName(), rel) if err != nil { return nil, telemetry.Error(ctx, span, err, "error updating helm secrets") } // retry upgrade res, err = cmd.Run(conf.Name, ch, conf.Values) if err != nil { return nil, telemetry.Error(ctx, span, err, "error running upgrade after updating helm secrets") } return res, nil } else { // ask the user to wait for about a minute before retrying for the above fix to kick in return nil, telemetry.Error(ctx, span, err, "another operation (install/upgrade/rollback) is in progress. If this error persists, please wait for 60 seconds to force an upgrade") } } } else if strings.Contains(err.Error(), "current release manifest contains removed kubernetes api(s)") || strings.Contains(err.Error(), "resource mapping not found for name") { // ref: https://helm.sh/docs/topics/kubernetes_apis/#updating-api-versions-of-a-release-manifest // in this case, we manually update the secret containing the new manifests secretList, err := a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace).List( context.Background(), v1.ListOptions{ LabelSelector: fmt.Sprintf("owner=helm,name=%s", rel.Name), }, ) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting secret list") } if len(secretList.Items) > 0 { mostRecentSecret := secretList.Items[0] for i := 1; i < len(secretList.Items); i += 1 { oldVersion, _ := strconv.Atoi(mostRecentSecret.Labels["version"]) newVersion, _ := strconv.Atoi(secretList.Items[i].Labels["version"]) if oldVersion < newVersion { mostRecentSecret = secretList.Items[i] } } // run the equivalent of `helm template` to get the manifest string for the new release installCmd := action.NewInstall(a.ActionConfig) installCmd.ReleaseName = conf.Name installCmd.Namespace = rel.Namespace installCmd.DryRun = true installCmd.Replace = true installCmd.ClientOnly = false installCmd.IncludeCRDs = true newRelDryRun, err := installCmd.Run(ch, conf.Values) if err != nil { return nil, telemetry.Error(ctx, span, err, "error running install cmd") } oldManifestBuffer := bytes.NewBufferString(rel.Manifest) newManifestBuffer := bytes.NewBufferString(newRelDryRun.Manifest) versionMapper := &DeprecatedAPIVersionMapper{} updatedManifestBuffer, err := versionMapper.Run(oldManifestBuffer, newManifestBuffer) if err != nil { return nil, telemetry.Error(ctx, span, err, "error running version mapper") } rel.Manifest = updatedManifestBuffer.String() helmSecrets := driver.NewSecrets(a.K8sAgent.Clientset.CoreV1().Secrets(rel.Namespace)) err = helmSecrets.Update(mostRecentSecret.GetName(), rel) if err != nil { return nil, telemetry.Error(ctx, span, err, "error updating helm secret") } res, err := cmd.Run(conf.Name, ch, conf.Values) if err != nil { return nil, telemetry.Error(ctx, span, err, "error running upgrade after updating helm secrets") } return res, nil } } return nil, telemetry.Error(ctx, span, err, "error running upgrade") } return res, nil } // InstallChartConfig is the config required to install a chart type InstallChartConfig struct { Chart *chart.Chart Name string Namespace string Values map[string]interface{} Cluster *models.Cluster Repo repository.Repository Registries []*models.Registry } // InstallChartFromValuesBytes reads the raw values and calls Agent.InstallChart func (a *Agent) InstallChartFromValuesBytes( ctx context.Context, conf *InstallChartConfig, values []byte, doAuth *oauth2.Config, disablePullSecretsInjection bool, ) (*release.Release, error) { ctx, span := telemetry.NewSpan(ctx, "helm-install-chart-from-values-bytes") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "project-id", Value: conf.Cluster.ProjectID}, telemetry.AttributeKV{Key: "cluster-id", Value: conf.Cluster.ID}, telemetry.AttributeKV{Key: "chart-name", Value: conf.Name}, telemetry.AttributeKV{Key: "chart-namespace", Value: conf.Namespace}, ) valuesYaml, err := chartutil.ReadValues(values) if err != nil { return nil, telemetry.Error(ctx, span, err, "Values could not be parsed") } conf.Values = valuesYaml return a.InstallChart(ctx, conf, doAuth, disablePullSecretsInjection) } // InstallChart installs a new chart func (a *Agent) InstallChart( ctx context.Context, conf *InstallChartConfig, doAuth *oauth2.Config, disablePullSecretsInjection bool, ) (*release.Release, error) { defer func() { if r := recover(); r != nil { fmt.Println("stacktrace from panic: \n" + string(debug.Stack())) } }() ctx, span := telemetry.NewSpan(ctx, "helm-install-chart") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "project-id", Value: conf.Cluster.ProjectID}, telemetry.AttributeKV{Key: "cluster-id", Value: conf.Cluster.ID}, telemetry.AttributeKV{Key: "chart-name", Value: conf.Name}, telemetry.AttributeKV{Key: "chart-namespace", Value: conf.Namespace}, ) cmd := action.NewInstall(a.ActionConfig) if cmd.Version == "" && cmd.Devel { cmd.Version = ">0.0.0-0" } cmd.ReleaseName = conf.Name cmd.Namespace = conf.Namespace cmd.Timeout = 300 * time.Second if err := checkIfInstallable(conf.Chart); err != nil { return nil, telemetry.Error(ctx, span, err, "error checking if installable") } var err error cmd.PostRenderer, err = NewPorterPostrenderer( conf.Cluster, conf.Repo, a.K8sAgent, conf.Namespace, conf.Registries, doAuth, disablePullSecretsInjection, ) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting post renderer") } if req := conf.Chart.Metadata.Dependencies; req != nil { for _, dep := range req { depChart, err := loader.LoadChartPublic(ctx, dep.Repository, dep.Name, dep.Version) if err != nil { return nil, telemetry.Error(ctx, span, err, fmt.Sprintf("error retrieving chart dependency %s/%s-%s", dep.Repository, dep.Name, dep.Version)) } conf.Chart.AddDependency(depChart) } } return cmd.Run(conf.Chart, conf.Values) } // UpgradeInstallChart installs a new chart if it doesn't exist, otherwise it upgrades it func (a *Agent) UpgradeInstallChart( ctx context.Context, conf *InstallChartConfig, doAuth *oauth2.Config, disablePullSecretsInjection bool, ) (*release.Release, error) { defer func() { if r := recover(); r != nil { fmt.Println("stacktrace from panic: \n" + string(debug.Stack())) } }() ctx, span := telemetry.NewSpan(ctx, "helm-upgrade-install-chart") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "project-id", Value: conf.Cluster.ProjectID}, telemetry.AttributeKV{Key: "cluster-id", Value: conf.Cluster.ID}, telemetry.AttributeKV{Key: "chart-name", Value: conf.Name}, telemetry.AttributeKV{Key: "chart-namespace", Value: conf.Namespace}, ) cmd := action.NewUpgrade(a.ActionConfig) cmd.Install = true if cmd.Version == "" && cmd.Devel { cmd.Version = ">0.0.0-0" } cmd.Namespace = conf.Namespace cmd.Timeout = 300 * time.Second if err := checkIfInstallable(conf.Chart); err != nil { return nil, telemetry.Error(ctx, span, err, "error checking if installable") } var err error cmd.PostRenderer, err = NewPorterPostrenderer( conf.Cluster, conf.Repo, a.K8sAgent, conf.Namespace, conf.Registries, doAuth, disablePullSecretsInjection, ) if err != nil { return nil, telemetry.Error(ctx, span, err, "error getting post renderer") } if req := conf.Chart.Metadata.Dependencies; req != nil { for _, dep := range req { depChart, err := loader.LoadChartPublic(ctx, dep.Repository, dep.Name, dep.Version) if err != nil { return nil, telemetry.Error(ctx, span, err, fmt.Sprintf("error retrieving chart dependency %s/%s-%s", dep.Repository, dep.Name, dep.Version)) } conf.Chart.AddDependency(depChart) } } release, err := cmd.RunWithContext(ctx, conf.Name, conf.Chart, conf.Values) if err != nil { return nil, telemetry.Error(ctx, span, err, "error running helm upgrade") } return release, nil } // UninstallChart uninstalls a chart func (a *Agent) UninstallChart( ctx context.Context, name string, ) (*release.UninstallReleaseResponse, error) { ctx, span := telemetry.NewSpan(ctx, "helm-uninstall-chart") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "chart-name", Value: name}) cmd := action.NewUninstall(a.ActionConfig) return cmd.Run(name) } // RollbackRelease rolls a release back to a specified revision/version func (a *Agent) RollbackRelease( ctx context.Context, name string, version int, ) error { ctx, span := telemetry.NewSpan(ctx, "helm-rollback-release") defer span.End() telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "name", Value: name}, telemetry.AttributeKV{Key: "version", Value: version}, ) cmd := action.NewRollback(a.ActionConfig) cmd.Version = version return cmd.Run(name) } // ------------------------ Helm agent helper functions ------------------------ // // checkIfInstallable validates if a chart can be installed // Application chart type is only installable func checkIfInstallable(ch *chart.Chart) error { switch ch.Metadata.Type { case "", "application": return nil } return errors.Errorf("%s charts are not installable", ch.Metadata.Type) }