Add support for shimmed CNB buildpacks to porter backend (#3718)

cli/cmd/pack/pack.go

@@ -2,8 +2,8 @@ package pack
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -73,80 +73,13 @@ func (a *Agent) Build(ctx context.Context, opts *docker.BuildOpts, buildConfig *
 			if bp == "" {
 				continue
 			}
-			u, err := url.Parse(bp)
-			if err == nil && u.Scheme != "" && u.Scheme != "urn" {
-				// could be a git repository containing the buildpack
-				if !strings.HasSuffix(u.Path, ".zip") && u.Host != "github.com" && u.Host != "www.github.com" {
-					return fmt.Errorf("please provide either a github.com URL or a ZIP file URL")
-				}
-
-				urlPaths := strings.Split(u.Path[1:], "/")
-				dstDir := filepath.Join(homedir.HomeDir(), ".porter")
-				bpCustomName := regexp.MustCompile("/|-").ReplaceAllString(u.Path[1:], "_")
-
-				var zipFileName string
-				if strings.HasSuffix(bpCustomName, ".zip") {
-					zipFileName = bpCustomName
-				} else {
-					zipFileName = fmt.Sprintf("%s.zip", bpCustomName)
-				}
-				downloader := &github.ZIPDownloader{
-					ZipFolderDest:       dstDir,
-					AssetFolderDest:     dstDir,
-					ZipName:             zipFileName,
-					RemoveAfterDownload: true,
-				}
-
-				if zipFileName != bpCustomName {
-					// try to download the repo ZIP from github
-					githubClient := githubApi.NewClient(nil)
-					rel, _, err := githubClient.Repositories.GetLatestRelease(
-						ctx,
-						urlPaths[0],
-						urlPaths[1],
-					)
-					if err == nil {
-						bp = rel.GetZipballURL()
-					} else {
-						// default to the current default branch
-						repo, _, err := githubClient.Repositories.Get(
-							ctx,
-							urlPaths[0],
-							urlPaths[1],
-						)
-						if err != nil {
-							return fmt.Errorf("could not fetch git repo details")
-						}
-						bp = fmt.Sprintf("%s/archive/refs/heads/%s.zip", bp, repo.GetDefaultBranch())
-					}
-				}
-
-				err = downloader.DownloadToFile(bp)
-				if err != nil {
-					return err
-				}
-
-				err = downloader.UnzipToDir()
-				if err != nil {
-					return err
-				}
-
-				dstFiles, err := ioutil.ReadDir(dstDir)
-				if err != nil {
-					return err
-				}
-
-				var bpRealName string
-				for _, info := range dstFiles {
-					if info.Mode().IsDir() && strings.Contains(info.Name(), urlPaths[1]) {
-						bpRealName = filepath.Join(dstDir, info.Name())
-					}
-				}
-
-				buildOpts.Buildpacks = append(buildOpts.Buildpacks, bpRealName)
-			} else {
-				buildOpts.Buildpacks = append(buildOpts.Buildpacks, bp)
+
+			bpRealName, err := getBuildpackName(ctx, bp)
+			if err != nil {
+				return err
 			}
+
+			buildOpts.Buildpacks = append(buildOpts.Buildpacks, bpRealName)
 		}
 		// FIXME: use all the config vars
 	}
@@ -157,3 +90,93 @@ func (a *Agent) Build(ctx context.Context, opts *docker.BuildOpts, buildConfig *
 
 	return sharedPackClient.Build(ctx, buildOpts)
 }
+
+func getBuildpackName(ctx context.Context, bp string) (string, error) {
+	if bp == "" {
+		return "", errors.New("please specify a buildpack name")
+	}
+
+	u, err := url.Parse(bp)
+	if err != nil {
+		return bp, nil
+	}
+
+	// if there is no scheme, it's likely something like `heroku/nodejs`
+	// if the scheme is `urn`, it's like something like `urn:cnb:registry:heroku/nodejs`
+	if u.Scheme == "" || u.Scheme == "urn" {
+		return bp, nil
+	}
+
+	// pass cnb-shimmed buildpacks as is
+	if u.Host == "cnb-shim.herokuapp.com" {
+		return bp, nil
+	}
+
+	var bpRealName string
+	// could be a git repository containing the buildpack
+	if !strings.HasSuffix(u.Path, ".zip") && u.Host != "github.com" && u.Host != "www.github.com" {
+		return bpRealName, errors.New("please provide either a github.com URL or a ZIP file URL")
+	}
+
+	urlPaths := strings.Split(u.Path[1:], "/")
+	dstDir := filepath.Join(homedir.HomeDir(), ".porter")
+	bpCustomName := regexp.MustCompile("/|-").ReplaceAllString(u.Path[1:], "_")
+
+	var zipFileName string
+	if strings.HasSuffix(bpCustomName, ".zip") {
+		zipFileName = bpCustomName
+	} else {
+		zipFileName = fmt.Sprintf("%s.zip", bpCustomName)
+	}
+	downloader := &github.ZIPDownloader{
+		ZipFolderDest:       dstDir,
+		AssetFolderDest:     dstDir,
+		ZipName:             zipFileName,
+		RemoveAfterDownload: true,
+	}
+
+	if zipFileName != bpCustomName {
+		// try to download the repo ZIP from github
+		githubClient := githubApi.NewClient(nil)
+		rel, _, err := githubClient.Repositories.GetLatestRelease(
+			ctx,
+			urlPaths[0],
+			urlPaths[1],
+		)
+		if err == nil {
+			bp = rel.GetZipballURL()
+		} else {
+			// default to the current default branch
+			repo, _, err := githubClient.Repositories.Get(
+				ctx,
+				urlPaths[0],
+				urlPaths[1],
+			)
+			if err != nil {
+				return bpRealName, errors.New("could not fetch git repo details")
+			}
+			bp = fmt.Sprintf("%s/archive/refs/heads/%s.zip", bp, repo.GetDefaultBranch())
+		}
+	}
+
+	if err := downloader.DownloadToFile(bp); err != nil {
+		return bpRealName, fmt.Errorf("failed to download buildpack: %w", err)
+	}
+
+	if err := downloader.UnzipToDir(); err != nil {
+		return bpRealName, fmt.Errorf("failed to extract buildpack: %w", err)
+	}
+
+	dstFiles, err := os.ReadDir(dstDir)
+	if err != nil {
+		return bpRealName, fmt.Errorf("failed to list files in extracted buildpack: %w", err)
+	}
+
+	for _, info := range dstFiles {
+		if info.Type().IsDir() && strings.Contains(info.Name(), urlPaths[1]) {
+			bpRealName = filepath.Join(dstDir, info.Name())
+		}
+	}
+
+	return bpRealName, nil
+}

cli/cmd/pack/pack_test.go

@@ -0,0 +1,111 @@
+package pack
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/client-go/util/homedir"
+)
+
+type BuildpackNameTestResult struct {
+	name string
+	err  error
+}
+
+func setupPackTest(tb testing.TB) func(tb testing.TB) {
+	porterHome := filepath.Join(homedir.HomeDir(), ".porter")
+	if err := os.MkdirAll(porterHome, 0o750); err != nil {
+		tb.Errorf("unable to initialize porter home folder for tests: %s", err.Error())
+	}
+
+	return func(tb testing.TB) {
+		dstFiles, err := os.ReadDir(porterHome)
+		if err != nil {
+			return
+		}
+
+		for _, info := range dstFiles {
+			if !info.Type().IsDir() {
+				continue
+			}
+
+			if err := os.RemoveAll(filepath.Join(porterHome, info.Name())); err != nil {
+				tb.Errorf("unable to delete porter home subfolder for tests: %s", err.Error())
+			}
+		}
+	}
+}
+
+func TestGetBuildpackName(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected BuildpackNameTestResult
+	}{
+		{
+			"empty buildpack name",
+			"",
+			BuildpackNameTestResult{"", errors.New("please specify a buildpack name")},
+		},
+		{
+			"cnb short name",
+			"heroku/nodejs",
+			BuildpackNameTestResult{"heroku/nodejs", nil},
+		},
+		{
+			"cnb urn",
+			"urn:cnb:registry:heroku/nodejs",
+			BuildpackNameTestResult{"urn:cnb:registry:heroku/nodejs", nil},
+		},
+		{
+			"cnb shim",
+			"https://cnb-shim.herokuapp.com/v1/heroku/nodejs?version=0.0.0&name=Node.js",
+			BuildpackNameTestResult{"https://cnb-shim.herokuapp.com/v1/heroku/nodejs?version=0.0.0&name=Node.js", nil},
+		},
+		{
+			"invalid tgz",
+			"https://example.com/tar.tgz",
+			BuildpackNameTestResult{"", errors.New("please provide either a github.com URL or a ZIP file URL")},
+		},
+		{
+			"github repo",
+			"https://github.com/heroku/buildpacks-nodejs/archive/fa2dc153e4683181608307ecb3922eaaeb43d92c.zip",
+			BuildpackNameTestResult{filepath.Join(homedir.HomeDir(), ".porter", "buildpacks-nodejs-fa2dc153e4683181608307ecb3922eaaeb43d92c"), nil},
+		},
+		{
+			"github repo zip",
+			"https://github.com/heroku/buildpacks-nodejs/archive/refs/tags/v1.1.6.zip",
+			BuildpackNameTestResult{filepath.Join(homedir.HomeDir(), ".porter", "buildpacks-nodejs-1.1.6"), nil},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			teardownPackTest := setupPackTest(t)
+			defer teardownPackTest(t)
+
+			ctx := context.Background()
+			actual, err := getBuildpackName(ctx, tt.input)
+			if actual != tt.expected.name {
+				t.Errorf("got %s, want %s", actual, tt.expected.name)
+			}
+
+			if err != nil && tt.expected.err == nil {
+				t.Errorf("got unexpected error: %s", err.Error())
+			}
+
+			if err == nil && tt.expected.err != nil {
+				t.Errorf("missing expected error %s", tt.expected.err)
+			}
+
+			if err != nil && tt.expected.err != nil {
+				if err.Error() != tt.expected.err.Error() {
+					t.Errorf("wrong error: %v", err)
+				}
+			}
+		})
+	}
+}