package helm import ( "bytes" "context" "fmt" "io" "net/url" "regexp" "sort" "strings" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/porter-dev/porter/internal/kubernetes" "github.com/porter-dev/porter/internal/models" "github.com/porter-dev/porter/internal/repository" "github.com/stefanmcshane/helm/pkg/postrender" "golang.org/x/oauth2" "gopkg.in/yaml.v2" "github.com/docker/distribution/reference" ) type PorterPostrenderer struct { DockerSecretsPostRenderer *DockerSecretsPostRenderer EnvironmentVariablePostrenderer *EnvironmentVariablePostrenderer } func NewPorterPostrenderer( cluster *models.Cluster, repo repository.Repository, agent *kubernetes.Agent, namespace string, regs []*models.Registry, doAuth *oauth2.Config, disablePullSecretsInjection bool, ) (postrender.PostRenderer, error) { var dockerSecretsPostrenderer *DockerSecretsPostRenderer var err error if !disablePullSecretsInjection && cluster != nil && agent != nil && regs != nil && len(regs) > 0 { dockerSecretsPostrenderer, err = NewDockerSecretsPostRenderer(cluster, repo, agent, namespace, regs, doAuth) if err != nil { return nil, err } } envVarPostrenderer, err := NewEnvironmentVariablePostrenderer() if err != nil { return nil, err } return &PorterPostrenderer{ DockerSecretsPostRenderer: dockerSecretsPostrenderer, EnvironmentVariablePostrenderer: envVarPostrenderer, }, nil } func (p *PorterPostrenderer) Run( renderedManifests *bytes.Buffer, ) (modifiedManifests *bytes.Buffer, err error) { if p.DockerSecretsPostRenderer != nil { renderedManifests, err = p.DockerSecretsPostRenderer.Run(renderedManifests) if err != nil { return nil, err } } renderedManifests, err = p.EnvironmentVariablePostrenderer.Run(renderedManifests) return renderedManifests, err } // DockerSecretsPostRenderer is a Helm post-renderer that adds image pull secrets to // pod specs that would otherwise be unable to pull an image. // // The post-renderer currently looks for two types of registries: GCR and ECR (TODO: DOCR // and Dockerhub). It also detects if the image pull secret is necessary: if GCR image pulls // occur in a GKE cluster in the same project, or if ECR image pulls exist in an EKS cluster // in the same organization + region, an image pull is not necessary. type DockerSecretsPostRenderer struct { Cluster *models.Cluster Repo repository.Repository Agent *kubernetes.Agent Namespace string DOAuth *oauth2.Config registries map[string]*models.Registry podSpecs []resource resources []resource } // while manifests are map[string]interface{} at the top level, // nested keys will be of type map[interface{}]interface{} type resource map[interface{}]interface{} func NewDockerSecretsPostRenderer( cluster *models.Cluster, repo repository.Repository, agent *kubernetes.Agent, namespace string, regs []*models.Registry, doAuth *oauth2.Config, ) (*DockerSecretsPostRenderer, error) { // Registries is a map of registry URLs to registry ids registries := make(map[string]*models.Registry) for _, reg := range regs { regURL := reg.URL if !strings.Contains(regURL, "http") { regURL = "https://" + regURL } parsedRegURL, err := url.Parse(regURL) if err != nil { continue } addReg := parsedRegURL.Host if parsedRegURL.Path != "" { addReg += "/" + strings.Trim(parsedRegURL.Path, "/") } registries[addReg] = reg } return &DockerSecretsPostRenderer{ Cluster: cluster, Repo: repo, Agent: agent, Namespace: namespace, DOAuth: doAuth, registries: registries, podSpecs: make([]resource, 0), resources: make([]resource, 0), }, nil } func (d *DockerSecretsPostRenderer) Run( renderedManifests *bytes.Buffer, ) (modifiedManifests *bytes.Buffer, err error) { bufCopy := bytes.NewBuffer(renderedManifests.Bytes()) linkedRegs, err := d.getRegistriesToLink(bufCopy) // if we encountered an error here, we'll render the manifests anyway // without modification if err != nil { return renderedManifests, nil } // Check to see if the resources loaded into the postrenderer contain a configmap // with a manifest that needs secrets generation as well. If this is the case, create and // run another postrenderer for this specific manifest. for i, res := range d.resources { kindVal, hasKind := res["kind"] if !hasKind { continue } kind, ok := kindVal.(string) if !ok { continue } if kind == "ConfigMap" { labelVal := getNestedResource(res, "metadata", "labels") if labelVal == nil { continue } porterLabelVal, exists := labelVal["getporter.dev/manifest"] if !exists { continue } if labelValStr, ok := porterLabelVal.(string); ok && labelValStr == "true" { data := getNestedResource(res, "data") manifestData, exists := data["manifest"] if !exists { continue } manifestDataStr, ok := manifestData.(string) if !ok { continue } dCopy := &DockerSecretsPostRenderer{ Cluster: d.Cluster, Repo: d.Repo, Agent: d.Agent, Namespace: d.Namespace, DOAuth: d.DOAuth, registries: d.registries, podSpecs: make([]resource, 0), resources: make([]resource, 0), } newData, err := dCopy.Run(bytes.NewBufferString(manifestDataStr)) if err != nil { continue } data["manifest"] = string(newData.Bytes()) d.resources[i] = res } } } // create the necessary secrets secrets, err := d.Agent.CreateImagePullSecrets( d.Repo, d.Namespace, linkedRegs, d.DOAuth, ) if err != nil { return renderedManifests, nil } d.updatePodSpecs(secrets) modifiedManifests = bytes.NewBuffer([]byte{}) encoder := yaml.NewEncoder(modifiedManifests) defer encoder.Close() for _, resource := range d.resources { // if the resource is empty, we skip encoding it to prevent errors. Helm/k8s expects empty resources to take the form "{}", // while this library writes an empty string, causing problems during installation. if len(resource) != 0 { err = encoder.Encode(resource) if err != nil { return nil, err } } } return modifiedManifests, nil } func (d *DockerSecretsPostRenderer) getRegistriesToLink(renderedManifests *bytes.Buffer) (map[string]*models.Registry, error) { // create a map of registry names to registries: these are the registries // that a secret will be generated for, if it does not exist linkedRegs := make(map[string]*models.Registry) var err error d.resources, err = decodeRenderedManifests(renderedManifests) if err != nil { return linkedRegs, err } // read the pod specs into the post-renderer object d.getPodSpecs(d.resources) for _, podSpec := range d.podSpecs { // get all images images := d.getImageList(podSpec) // read the image url for _, image := range images { regName, err := getRegNameFromImageRef(image) if err != nil { continue } // check if the integration is native to the cluster/registry combination isNative := d.isRegistryNative(regName) if isNative { continue } reg, exists := d.registries[regName] if !exists { continue } // if the registry exists, add it to the map linkedRegs[regName] = reg } } return linkedRegs, nil } func decodeRenderedManifests( renderedManifests *bytes.Buffer, ) ([]resource, error) { resArr := make([]resource, 0) // use the yaml decoder to parse the multi-document yaml. decoder := yaml.NewDecoder(renderedManifests) for { res := make(resource) err := decoder.Decode(&res) if err == io.EOF { break } if err != nil { return resArr, err } if len(res) != 0 { resArr = append(resArr, res) } } return resArr, nil } func (d *DockerSecretsPostRenderer) getPodSpecs(resources []resource) { for _, res := range resources { kindVal, hasKind := res["kind"] if !hasKind { continue } kind, ok := kindVal.(string) if !ok { continue } // manifests of list type will have an items field, items should // be recursively parsed if itemsVal, isList := res["items"]; isList { if items, ok := itemsVal.([]interface{}); ok { // convert items to resource resArr := make([]resource, 0) for _, item := range items { if arrVal, ok := item.(resource); ok { resArr = append(resArr, arrVal) } } d.getPodSpecs(resArr) } continue } // otherwise, get the pod spec based on the type of resource podSpec := getPodSpecFromResource(kind, res) if podSpec == nil { continue } d.podSpecs = append(d.podSpecs, podSpec) } return } func (d *DockerSecretsPostRenderer) updatePodSpecs(secrets map[string]string) { for _, podSpec := range d.podSpecs { containersVal, hasContainers := podSpec["containers"] if !hasContainers { continue } containers, ok := containersVal.([]interface{}) if !ok { continue } imagePullSecrets := make([]map[string]interface{}, 0) if existingPullSecrets, ok := podSpec["imagePullSecrets"]; ok { if existing, ok := existingPullSecrets.([]map[string]interface{}); ok { imagePullSecrets = existing } } for _, container := range containers { _container, ok := container.(resource) if !ok { continue } image, ok := _container["image"].(string) if !ok { continue } regName, err := getRegNameFromImageRef(image) if err != nil { continue } if secretName, ok := secrets[regName]; ok && secretName != "" { imagePullSecrets = append(imagePullSecrets, map[string]interface{}{ "name": secretName, }) } } if len(imagePullSecrets) > 0 { podSpec["imagePullSecrets"] = imagePullSecrets } } } func (d *DockerSecretsPostRenderer) getImageList(podSpec resource) []string { images := make([]string, 0) containersVal, hasContainers := podSpec["containers"] if !hasContainers { return images } containers, ok := containersVal.([]interface{}) if !ok { return images } for _, container := range containers { _container, ok := container.(resource) if !ok { continue } image, ok := _container["image"].(string) if !ok { continue } images = append(images, image) } return images } var ecrPattern = regexp.MustCompile(`(^[a-zA-Z0-9][a-zA-Z0-9-_]*)\.dkr\.ecr(\-fips)?\.([a-zA-Z0-9][a-zA-Z0-9-_]*)\.amazonaws\.com(\.cn)?`) func (d *DockerSecretsPostRenderer) isRegistryNative(regName string) bool { ctx := context.Background() isNative := false if strings.Contains(regName, "gcr") && d.Cluster.AuthMechanism == models.GCP { // TODO (POR-33): fix architecture for clusters and re-add the code below // // get the project id of the cluster // gcpInt, err := d.Repo.GCPIntegration().ReadGCPIntegration(d.Cluster.ProjectID, d.Cluster.GCPIntegrationID) // if err != nil { // return false // } // gkeProjectID, err := integrations.GCPProjectIDFromJSON(gcpInt.GCPKeyData) // if err != nil { // return false // } // // parse the project id of the gcr url // if regNameArr := strings.Split(regName, "/"); len(regNameArr) >= 2 { // gcrProjectID := regNameArr[1] // isNative = gcrProjectID == gkeProjectID // } } else if strings.Contains(regName, "ecr") && d.Cluster.AuthMechanism == models.AWS { matches := ecrPattern.FindStringSubmatch(regName) if len(matches) < 3 { return false } eksAccountID := matches[1] eksRegion := matches[3] awsInt, err := d.Repo.AWSIntegration().ReadAWSIntegration(d.Cluster.ProjectID, d.Cluster.AWSIntegrationID) if err != nil { return false } err = awsInt.PopulateAWSArn(ctx) if err != nil { return false } parsedARN, err := arn.Parse(awsInt.AWSArn) if err != nil { return false } isNative = parsedARN.AccountID == eksAccountID && parsedARN.Region == eksRegion } return isNative } // EnvironmentVariablePostrenderer removes duplicated environment variables, giving preference to synced // env vars type EnvironmentVariablePostrenderer struct { podSpecs []resource resources []resource } func NewEnvironmentVariablePostrenderer() (*EnvironmentVariablePostrenderer, error) { return &EnvironmentVariablePostrenderer{ podSpecs: make([]resource, 0), resources: make([]resource, 0), }, nil } func (e *EnvironmentVariablePostrenderer) Run( renderedManifests *bytes.Buffer, ) (modifiedManifests *bytes.Buffer, err error) { e.resources, err = decodeRenderedManifests(renderedManifests) if err != nil { return nil, err } // Check to see if the resources loaded into the postrenderer contain a configmap // with a manifest that needs env var cleanup as well. If this is the case, create and // run another postrenderer for this specific manifest. for i, res := range e.resources { kindVal, hasKind := res["kind"] if !hasKind { continue } kind, ok := kindVal.(string) if !ok { continue } if kind == "ConfigMap" { labelVal := getNestedResource(res, "metadata", "labels") if labelVal == nil { continue } porterLabelVal, exists := labelVal["getporter.dev/manifest"] if !exists { continue } if labelValStr, ok := porterLabelVal.(string); ok && labelValStr == "true" { data := getNestedResource(res, "data") manifestData, exists := data["manifest"] if !exists { continue } manifestDataStr, ok := manifestData.(string) if !ok { continue } dCopy := &EnvironmentVariablePostrenderer{ podSpecs: make([]resource, 0), resources: make([]resource, 0), } newData, err := dCopy.Run(bytes.NewBufferString(manifestDataStr)) if err != nil { continue } data["manifest"] = string(newData.Bytes()) e.resources[i] = res } } } e.getPodSpecs(e.resources) e.updatePodSpecs() modifiedManifests = bytes.NewBuffer([]byte{}) encoder := yaml.NewEncoder(modifiedManifests) defer encoder.Close() for _, resource := range e.resources { err = encoder.Encode(resource) if err != nil { return nil, err } } return modifiedManifests, nil } func (e *EnvironmentVariablePostrenderer) getPodSpecs(resources []resource) { for _, res := range resources { kindVal, hasKind := res["kind"] if !hasKind { continue } kind, ok := kindVal.(string) if !ok { continue } // manifests of list type will have an items field, items should // be recursively parsed if itemsVal, isList := res["items"]; isList { if items, ok := itemsVal.([]interface{}); ok { // convert items to resource resArr := make([]resource, 0) for _, item := range items { if arrVal, ok := item.(resource); ok { resArr = append(resArr, arrVal) } } e.getPodSpecs(resArr) } continue } // otherwise, get the pod spec based on the type of resource podSpec := getPodSpecFromResource(kind, res) if podSpec == nil { continue } e.podSpecs = append(e.podSpecs, podSpec) } return } func (e *EnvironmentVariablePostrenderer) updatePodSpecs() error { // for each pod spec, remove duplicate env variables for _, podSpec := range e.podSpecs { containersVal, hasContainers := podSpec["containers"] if !hasContainers { continue } containers, ok := containersVal.([]interface{}) if !ok { continue } newContainers := make([]interface{}, 0) for _, container := range containers { envVars := make(map[string]interface{}) _container, ok := container.(resource) if !ok { continue } // read container env variables envInter, ok := _container["env"] if !ok { newContainers = append(newContainers, _container) continue } env, ok := envInter.([]interface{}) if !ok { newContainers = append(newContainers, _container) continue } for _, envVar := range env { envVarMap, ok := envVar.(resource) if !ok { continue } envVarName, ok := envVarMap["name"] if !ok { continue } envVarNameStr, ok := envVarName.(string) if !ok { continue } // check if the env var already exists, if it does perform reconciliation if currVal, exists := envVars[envVarNameStr]; exists { currValMap, ok := currVal.(resource) if !ok { continue } // if the current value has a valueFrom field, this should override the existing env var if _, currValFromFieldExists := currValMap["valueFrom"]; currValFromFieldExists { continue } else { envVars[envVarNameStr] = envVarMap } } else { envVars[envVarNameStr] = envVarMap } } // flatten env var map to array envVarArr := make([]interface{}, 0) for _, envVar := range envVars { envVarArr = append(envVarArr, envVar) } // Sort the slices according to a stable ordering. This is hacky and inefficient. sort.SliceStable(envVarArr, func(i, j int) bool { return fmt.Sprintf("%v", envVarArr[i]) > fmt.Sprintf("%v", envVarArr[j]) }) _container["env"] = envVarArr newContainers = append(newContainers, _container) } podSpec["containers"] = newContainers } return nil } // HELPERS func getPodSpecFromResource(kind string, res resource) resource { switch kind { case "Pod": return getNestedResource(res, "spec") case "DaemonSet", "Deployment", "Job", "ReplicaSet", "ReplicationController", "StatefulSet": return getNestedResource(res, "spec", "template", "spec") case "PodTemplate": return getNestedResource(res, "template", "spec") case "CronJob": return getNestedResource(res, "spec", "jobTemplate", "spec", "template", "spec") } return nil } func getNestedResource(res resource, keys ...string) resource { curr := res var ok bool for _, key := range keys { curr, ok = curr[key].(resource) if !ok { return nil } } return curr } func getRegNameFromImageRef(image string) (string, error) { named, err := reference.ParseNormalizedNamed(image) if err != nil { return "", err } domain := reference.Domain(named) path := reference.Path(named) var regName string // if registry is dockerhub, leave the image name as-is if strings.Contains(domain, "docker.io") { regName = "index.docker.io/" + path } else if strings.Contains(domain, "pkg.dev") { pathSlice := strings.Split(path, "/") // a GAR image path can either be PROJECT-ID/REPOSITORY/IMAGE or DOMAIN/PROJECT-ID/REPOSITORY/IMAGE // // see: https://cloud.google.com/artifact-registry/docs/docker/names#domain if len(pathSlice) == 3 { regName = fmt.Sprintf("%s/%s", domain, pathSlice[0]) } else if len(pathSlice) == 4 { regName = fmt.Sprintf("%s/%s/%s", domain, pathSlice[0], pathSlice[1]) } else { return "", fmt.Errorf("invalid GAR image: %s", image) } } else { regName = domain if pathArr := strings.Split(path, "/"); len(pathArr) > 1 { regName += "/" + strings.Join(pathArr[:len(pathArr)-1], "/") } } return regName, nil } type DeprecatedAPIVersionMapper struct { } type APIVersionKind struct { oldAPIVersion, newAPIVersion, oldKind, newKind string } func (d *DeprecatedAPIVersionMapper) Run( oldRenderedManifests *bytes.Buffer, newRenderedManifests *bytes.Buffer, ) (modifiedManifests *bytes.Buffer, err error) { oldResources, err := decodeRenderedManifests(oldRenderedManifests) if err != nil { return nil, err } newResources, err := decodeRenderedManifests(newRenderedManifests) if err != nil { return nil, err } newNameResourceMap := make(map[string]resource) for _, newRes := range newResources { name, ok := getResourceName(newRes) if !ok { continue } newKind, _, ok := getKindAndAPIVersion(newRes) if !ok { continue } uniqueName := fmt.Sprintf("%s-%s", strings.ToLower(newKind), name) newNameResourceMap[uniqueName] = newRes } nameMap := make(map[string]APIVersionKind) for _, oldRes := range oldResources { oldName, ok := getResourceName(oldRes) if !ok { continue } oldKind, oldAPIVersion, ok := getKindAndAPIVersion(oldRes) if !ok { continue } uniqueName := fmt.Sprintf("%s-%s", strings.ToLower(oldKind), oldName) newRes, exists := newNameResourceMap[uniqueName] if !exists { continue } newKind, newAPIVersion, ok := getKindAndAPIVersion(newRes) if !ok { continue } nameMap[oldName] = APIVersionKind{ oldAPIVersion: oldAPIVersion, newAPIVersion: newAPIVersion, oldKind: oldKind, newKind: newKind, } // if the API versions don't match, update the old api version to the new api version if oldAPIVersion != newAPIVersion { oldRes["apiVersion"] = newAPIVersion } } modifiedManifests = bytes.NewBuffer([]byte{}) encoder := yaml.NewEncoder(modifiedManifests) defer encoder.Close() for _, resource := range oldResources { err = encoder.Encode(resource) if err != nil { return nil, err } } return modifiedManifests, nil } func getResourceName(res resource) (string, bool) { metadataVal, hasMetadataVal := res["metadata"] if !hasMetadataVal { return "", false } metadata, ok := metadataVal.(resource) if !ok { return "", false } nameVal, ok := metadata["name"] if !ok { return "", false } name, ok := nameVal.(string) return name, ok } func getKindAndAPIVersion(res resource) (kind string, apiVersion string, ok bool) { kindVal, hasKindVal := res["kind"] if !hasKindVal { return "", "", false } kind, ok = kindVal.(string) if !ok { return "", "", false } apiVersionVal, hasAPIVersionVal := res["apiVersion"] if !hasAPIVersionVal { return "", "", false } apiVersion, ok = apiVersionVal.(string) if !ok { return "", "", false } return kind, apiVersion, true }