porter/server/api/git_repo_handler.go

package api

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/porter-dev/porter/internal/models"
	"golang.org/x/oauth2"

	"github.com/bradleyfalzon/ghinstallation"
	"github.com/go-chi/chi"
	"github.com/google/go-github/github"
)

// HandleListProjectGitRepos returns a list of git repos for a project
func (app *App) HandleListProjectGitRepos(w http.ResponseWriter, r *http.Request) {
	tok, err := app.getGithubAppOauthTokenFromRequest(r)

	if err != nil {
		app.Logger.Warn().Err(err).
			Str("info", "github app oauth token error").
			Msg("")

		json.NewEncoder(w).Encode(make([]*models.GitRepoExternal, 0))
		return
	}

	client := github.NewClient(app.GithubProjectConf.Client(oauth2.NoContext, tok))

	accountIds := make([]int64, 0)

	AuthUser, _, err := client.Users.Get(context.Background(), "")

	if err != nil {

		app.handleErrorInternal(err, w)
		return
	}

	accountIds = append(accountIds, *AuthUser.ID)

	opts := &github.ListOptions{
		PerPage: 100,
		Page:    1,
	}

	for {
		orgs, pages, err := client.Organizations.List(context.Background(), "", opts)

		if err != nil {
			res := HandleListGithubAppAccessResp{
				HasAccess: false,
			}
			json.NewEncoder(w).Encode(res)
			return
		}

		for _, org := range orgs {
			accountIds = append(accountIds, *org.ID)
		}

		if pages.NextPage == 0 {
			break
		}
	}

	installationData, err := app.Repo.GithubAppInstallation.ReadGithubAppInstallationByAccountIDs(accountIds)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	installationIds := make([]int64, 0)

	for _, v := range installationData {
		installationIds = append(installationIds, v.InstallationID)
	}

	json.NewEncoder(w).Encode(installationIds)
}

// Repo represents a GitHub or Gitab repository
type Repo struct {
	FullName string
	Kind     string
}

// DirectoryItem represents a file or subfolder in a repository
type DirectoryItem struct {
	Path string
	Type string
}

// AutoBuildpack represents an automatically detected buildpack
type AutoBuildpack struct {
	Valid bool   `json:"valid"`
	Name  string `json:"name"`
}

// HandleListRepos retrieves a list of repo names
func (app *App) HandleListRepos(w http.ResponseWriter, r *http.Request) {

	client, err := app.githubAppClientFromRequest(r)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	// figure out number of repositories
	opt := &github.ListOptions{
		PerPage: 100,
	}

	allRepos, resp, err := client.Apps.ListRepos(context.Background(), opt)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	// make workers to get pages concurrently
	const WCOUNT = 5
	numPages := resp.LastPage + 1
	var workerErr error
	var mu sync.Mutex
	var wg sync.WaitGroup

	worker := func(cp int) {
		defer wg.Done()

		for cp < numPages {
			cur_opt := &github.ListOptions{
				Page:    cp,
				PerPage: 100,
			}

			repos, _, err := client.Apps.ListRepos(context.Background(), cur_opt)

			if err != nil {
				mu.Lock()
				workerErr = err
				mu.Unlock()
				return
			}

			mu.Lock()
			allRepos = append(allRepos, repos...)
			mu.Unlock()

			cp += WCOUNT
		}
	}

	var numJobs int
	if numPages > WCOUNT {
		numJobs = WCOUNT
	} else {
		numJobs = numPages
	}

	wg.Add(numJobs)

	// page 1 is already loaded so we start with 2
	for i := 1; i <= numJobs; i++ {
		go worker(i + 1)
	}

	wg.Wait()

	if workerErr != nil {
		app.handleErrorInternal(workerErr, w)
		return
	}

	res := make([]Repo, 0)

	for _, repo := range allRepos {
		res = append(res, Repo{
			FullName: repo.GetFullName(),
			Kind:     "github",
		})
	}

	json.NewEncoder(w).Encode(res)
}

// HandleGetBranches retrieves a list of branch names for a specified repo
func (app *App) HandleGetBranches(w http.ResponseWriter, r *http.Request) {

	client, err := app.githubAppClientFromRequest(r)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	owner := chi.URLParam(r, "owner")
	name := chi.URLParam(r, "name")

	// List all branches for a specified repo
	allBranches, resp, err := client.Repositories.ListBranches(context.Background(), owner, name, &github.ListOptions{
		PerPage: 100,
	})

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	// make workers to get branches concurrently
	const WCOUNT = 5
	numPages := resp.LastPage + 1
	var workerErr error
	var mu sync.Mutex
	var wg sync.WaitGroup

	worker := func(cp int) {
		defer wg.Done()

		for cp < numPages {
			opts := &github.ListOptions{
				Page:    cp,
				PerPage: 100,
			}

			branches, _, err := client.Repositories.ListBranches(context.Background(), owner, name, opts)

			if err != nil {
				mu.Lock()
				workerErr = err
				mu.Unlock()
				return
			}

			mu.Lock()
			allBranches = append(allBranches, branches...)
			mu.Unlock()

			cp += WCOUNT
		}
	}

	var numJobs int
	if numPages > WCOUNT {
		numJobs = WCOUNT
	} else {
		numJobs = numPages
	}

	wg.Add(numJobs)

	// page 1 is already loaded so we start with 2
	for i := 1; i <= numJobs; i++ {
		go worker(i + 1)
	}

	wg.Wait()

	if workerErr != nil {
		app.handleErrorInternal(workerErr, w)
		return
	}

	res := make([]string, 0)
	for _, b := range allBranches {
		res = append(res, b.GetName())
	}

	json.NewEncoder(w).Encode(res)
}

// HandleDetectBuildpack attempts to figure which buildpack will be auto used based on directory contents
func (app *App) HandleDetectBuildpack(w http.ResponseWriter, r *http.Request) {

	client, err := app.githubAppClientFromRequest(r)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	queryParams, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		app.handleErrorFormDecoding(err, ErrReleaseDecode, w)
		return
	}

	owner := chi.URLParam(r, "owner")
	name := chi.URLParam(r, "name")
	branch := chi.URLParam(r, "branch")

	repoContentOptions := github.RepositoryContentGetOptions{}
	repoContentOptions.Ref = branch
	_, directoryContents, _, err := client.Repositories.GetContents(context.Background(), owner, name, queryParams["dir"][0], &repoContentOptions)
	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	var BREQS = map[string]string{
		"requirements.txt": "Python",
		"Gemfile":          "Ruby",
		"package.json":     "Node.js",
		"pom.xml":          "Java",
		"composer.json":    "PHP",
	}

	res := AutoBuildpack{
		Valid: true,
	}
	matches := 0

	for i := range directoryContents {
		name := *directoryContents[i].Name

		bname, ok := BREQS[name]
		if ok {
			matches++
			res.Name = bname
		}
	}

	if matches != 1 {
		res.Valid = false
		res.Name = ""
	}

	json.NewEncoder(w).Encode(res)
}

// HandleGetBranchContents retrieves the contents of a specific branch and subdirectory
func (app *App) HandleGetBranchContents(w http.ResponseWriter, r *http.Request) {

	client, err := app.githubAppClientFromRequest(r)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	queryParams, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		app.handleErrorFormDecoding(err, ErrReleaseDecode, w)
		return
	}

	owner := chi.URLParam(r, "owner")
	name := chi.URLParam(r, "name")
	branch, err := url.QueryUnescape(chi.URLParam(r, "branch"))

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	repoContentOptions := github.RepositoryContentGetOptions{}
	repoContentOptions.Ref = branch
	_, directoryContents, _, err := client.Repositories.GetContents(context.Background(), owner, name, queryParams["dir"][0], &repoContentOptions)
	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	res := []DirectoryItem{}
	for i := range directoryContents {
		d := DirectoryItem{}
		d.Path = *directoryContents[i].Path
		d.Type = *directoryContents[i].Type
		res = append(res, d)
	}

	// Ret2: recursively traverse all dirs to create config bundle (case on type == dir)
	// https://api.github.com/repos/porter-dev/porter/contents?ref=frontend-graph
	json.NewEncoder(w).Encode(res)
}

type GetProcfileContentsResp map[string]string

var procfileRegex = regexp.MustCompile("^([A-Za-z0-9_]+):\\s*(.+)$")

// HandleGetProcfileContents retrieves the contents of a procfile in a github repo
func (app *App) HandleGetProcfileContents(w http.ResponseWriter, r *http.Request) {

	client, err := app.githubAppClientFromRequest(r)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	owner := chi.URLParam(r, "owner")
	name := chi.URLParam(r, "name")
	branch, err := url.QueryUnescape(chi.URLParam(r, "branch"))

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	queryParams, err := url.ParseQuery(r.URL.RawQuery)

	if err != nil {
		app.handleErrorFormDecoding(err, ErrReleaseDecode, w)
		return
	}

	resp, _, _, err := client.Repositories.GetContents(
		context.TODO(),
		owner,
		name,
		queryParams["path"][0],
		&github.RepositoryContentGetOptions{
			Ref: branch,
		},
	)

	if err != nil {
		http.NotFound(w, r)
		return
	}

	fileData, err := resp.GetContent()

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	parsedContents := make(GetProcfileContentsResp)

	// parse the procfile information
	for _, line := range strings.Split(fileData, "\n") {
		if matches := procfileRegex.FindStringSubmatch(line); matches != nil {
			parsedContents[matches[1]] = matches[2]
		}
	}

	json.NewEncoder(w).Encode(parsedContents)
}

type HandleGetRepoZIPDownloadURLResp struct {
	URLString       string `json:"url"`
	LatestCommitSHA string `json:"latest_commit_sha"`
}

// HandleGetRepoZIPDownloadURL gets the URL for downloading a zip file from a Github
// repository
func (app *App) HandleGetRepoZIPDownloadURL(w http.ResponseWriter, r *http.Request) {

	client, err := app.githubAppClientFromRequest(r)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	owner := chi.URLParam(r, "owner")
	name := chi.URLParam(r, "name")
	branch, err := url.QueryUnescape(chi.URLParam(r, "branch"))

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	branchResp, _, err := client.Repositories.GetBranch(
		context.TODO(),
		owner,
		name,
		branch,
	)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	ghURL, _, err := client.Repositories.GetArchiveLink(
		context.TODO(),
		owner,
		name,
		github.Zipball,
		&github.RepositoryContentGetOptions{
			Ref: *branchResp.Commit.SHA,
		},
	)

	if err != nil {
		app.handleErrorInternal(err, w)
		return
	}

	apiResp := HandleGetRepoZIPDownloadURLResp{
		URLString:       ghURL.String(),
		LatestCommitSHA: *branchResp.Commit.SHA,
	}

	json.NewEncoder(w).Encode(apiResp)
}

// githubAppClientFromRequest gets the github app installation id from the request and authenticates
// using it and a private key file
func (app *App) githubAppClientFromRequest(r *http.Request) (*github.Client, error) {

	installationID, err := strconv.ParseUint(chi.URLParam(r, "installation_id"), 0, 64)

	if err != nil || installationID == 0 {
		return nil, fmt.Errorf("could not read installation id")
	}

	itr, err := ghinstallation.NewKeyFromFile(
		http.DefaultTransport,
		app.GithubAppConf.AppID,
		int64(installationID),
		app.GithubAppConf.SecretPath)

	if err != nil {
		return nil, err
	}

	return github.NewClient(&http.Client{Transport: itr}), nil
}