Setup support for multiple deployment environments (#2839)

.github/workflows/porter_production.yml

@@ -1,25 +1,87 @@
 "on":
   push:
     branches:
-    - master
+      - master
 name: Deploy to Porter
 jobs:
+  build-go:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup Go Cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: porter-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: porter-go-`
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+          cache: false
+      - name: Download Go Modules
+        run: go mod download
+      - name: Build Server Binary
+        run: go build -ldflags="-w -s -X 'main.Version=production'" -o ./bin/app ./cmd/app
+      - name: Build Migration Binary
+        run: go build -ldflags '-w -s' -o ./bin/migrate ./cmd/migrate
+      - name: Store Binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-binaries
+          path: bin/
+  build-npm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install NPM Dependencies
+        run: |
+          cd dashboard
+          npm i --legacy-peer-deps
+      - name: Run NPM Build
+        run: |
+          cd dashboard
+          npm run build
+      - name: Store NPM Static Files
+        uses: actions/upload-artifact@v3
+        with:
+          name: npm-static-files
+          path: dashboard/build/
   porter-deploy:
     runs-on: ubuntu-latest
+    needs: [build-go, build-npm]
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Set Github tag
-      id: vars
-      run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-    - name: Update Porter App
-      timeout-minutes: 20
-      uses: porter-dev/porter-update-action@v0.1.0
-      with:
-        app: production
-        cluster: "8"
-        host: https://dashboard.internal-tools.getporter.dev
-        namespace: default
-        project: "1"
-        tag: ${{ steps.vars.outputs.sha_short }}
-        token: ${{ secrets.PORTER_TOKEN_1 }}
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Get Go Binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: go-binaries
+          path: bin/
+      - name: Get NPM static files
+        uses: actions/download-artifact@v3
+        with:
+          name: npm-static-files
+          path: build/
+      - name: Set Github tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Update Porter App
+        timeout-minutes: 20
+        uses: porter-dev/porter-update-action@v0.1.0
+        with:
+          app: production
+          cluster: "8"
+          host: https://dashboard.internal-tools.getporter.dev
+          namespace: default
+          project: "1"
+          tag: ${{ steps.vars.outputs.sha_short }}
+          token: ${{ secrets.PORTER_TOKEN_1 }}

api/server/handlers/registry/list_images.go

@@ -29,7 +29,8 @@ func NewRegistryListImagesHandler(
 }
 
 func (c *RegistryListImagesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	reg, _ := r.Context().Value(types.RegistryScope).(*models.Registry)
+	ctx := r.Context()
+	reg, _ := ctx.Value(types.RegistryScope).(*models.Registry)
 
 	repoName, _ := requestutils.GetURLParamString(r, types.URLParamWildcard)
 
@@ -37,12 +38,12 @@ func (c *RegistryListImagesHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 	_reg := registry.Registry(*reg)
 	regAPI := &_reg
 
-	imgs, err := regAPI.ListImages(repoName, c.Repo(), c.Config().DOConf)
-
-	if err != nil && strings.Contains(err.Error(), "RepositoryNotFoundException") {
-		c.HandleAPIError(w, r, apierrors.NewErrNotFound(fmt.Errorf("no such repository: %s", repoName)))
-		return
-	} else if err != nil {
+	imgs, err := regAPI.ListImages(ctx, repoName, c.Repo(), c.Config())
+	if err != nil {
+		if strings.Contains(err.Error(), "RepositoryNotFoundException") {
+			c.HandleAPIError(w, r, apierrors.NewErrNotFound(fmt.Errorf("no such repository: %s", repoName)))
+			return
+		}
 		c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
 		return
 	}

api/server/handlers/v1/registry/list_images.go

@@ -29,7 +29,8 @@ func NewRegistryListImagesHandler(
 }
 
 func (c *RegistryListImagesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	reg, _ := r.Context().Value(types.RegistryScope).(*models.Registry)
+	ctx := r.Context()
+	reg, _ := ctx.Value(types.RegistryScope).(*models.Registry)
 
 	repoName, reqErr := requestutils.GetURLParamString(r, types.URLParamWildcard)
 
@@ -79,7 +80,7 @@ func (c *RegistryListImagesHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 
 		res.Images = append(res.Images, imgs...)
 	} else {
-		imgs, err := regAPI.ListImages(repoName, c.Repo(), c.Config().DOConf)
+		imgs, err := regAPI.ListImages(ctx, repoName, c.Repo(), c.Config())
 		if err != nil {
 			c.HandleAPIError(w, r, apierrors.NewErrInternal(err))
 			return

api/server/shared/config/env/envconfs.go

@@ -46,7 +46,10 @@ type ServerConf struct {
 	GithubAppWebhookSecret string `env:"GITHUB_APP_WEBHOOK_SECRET"`
 	GithubAppID            string `env:"GITHUB_APP_ID"`
 	GithubAppSecretPath    string `env:"GITHUB_APP_SECRET_PATH"`
-	GithubAppSecret        []byte
+	// GithubAppSecretBase64 is a base64 encoded version of the GithubAppSecret. This can be used instead of GithubAppSecretPath to pass in a key, allowing for support in systems where mounting the secret is not possible.
+	// If GithubAppSecretBase64 is set, it will check for a file at GithubAppSecretPath. If a file is found, the file will NOT be overwritten. If no file it found, then GithubAppSecretBase64 will be decoded and written to GithubAppSecretPath.
+	GithubAppSecretBase64 string `env:"GITHUB_APP_SECRET_BASE64"`
+	GithubAppSecret       []byte
 
 	GoogleClientID         string `env:"GOOGLE_CLIENT_ID"`
 	GoogleClientSecret     string `env:"GOOGLE_CLIENT_SECRET"`

api/server/shared/config/loader/loader.go

@@ -1,10 +1,13 @@
 package loader
 
 import (
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strconv"
 
 	gorillaws "github.com/gorilla/websocket"
@@ -153,6 +156,7 @@ func (e *EnvConfigLoader) LoadConfig() (res *config.Config, err error) {
 		})
 	}
 
+	// TODO: remove this as part of POR-1055
 	if sc.GithubClientID != "" && sc.GithubClientSecret != "" {
 		res.GithubConf = oauth.NewGithubClient(&oauth.Config{
 			ClientID:     sc.GithubClientID,
@@ -162,6 +166,30 @@ func (e *EnvConfigLoader) LoadConfig() (res *config.Config, err error) {
 		})
 	}
 
+	if sc.GithubAppSecretBase64 != "" {
+		if sc.GithubAppSecretPath == "" {
+			sc.GithubAppSecretPath = "github-app-secret-key"
+		}
+		_, err := os.Stat(sc.GithubAppSecretPath)
+		if err != nil {
+			if !errors.Is(err, os.ErrNotExist) {
+				return nil, fmt.Errorf("GITHUB_APP_SECRET_BASE64 provided, but error checking if GITHUB_APP_SECRET_PATH exists: %w", err)
+			}
+			secret, err := base64.StdEncoding.DecodeString(sc.GithubAppSecretBase64)
+			if err != nil {
+				return nil, fmt.Errorf("GITHUB_APP_SECRET_BASE64 provided, but error decoding: %w", err)
+			}
+			_, err = createDirectoryRecursively(sc.GithubAppSecretPath)
+			if err != nil {
+				return nil, fmt.Errorf("GITHUB_APP_SECRET_BASE64 provided, but error creating directory for GITHUB_APP_SECRET_PATH: %w", err)
+			}
+			err = os.WriteFile(sc.GithubAppSecretPath, secret, os.ModePerm)
+			if err != nil {
+				return nil, fmt.Errorf("GITHUB_APP_SECRET_BASE64 provided, but error writing to GITHUB_APP_SECRET_PATH: %w", err)
+			}
+		}
+	}
+
 	if sc.GithubAppClientID != "" &&
 		sc.GithubAppClientSecret != "" &&
 		sc.GithubAppName != "" &&
@@ -286,3 +314,11 @@ func getProvisionerServiceClient(sc *env.ServerConf) (*client.Client, error) {
 
 	return nil, fmt.Errorf("required env vars not set for provisioner")
 }
+
+// createDirectoryRecursively creates a directory and all its parents if they don't exist
+func createDirectoryRecursively(p string) (*os.File, error) {
+	if err := os.MkdirAll(filepath.Dir(p), 0o770); err != nil {
+		return nil, err
+	}
+	return os.Create(p)
+}

dashboard/src/components/CloudFormationForm.tsx

@@ -47,7 +47,7 @@ const CloudFormationForm: React.FC<Props> = ({
 
   const checkIfRoleExists = async () => {
     let externalId = getExternalId();
-    let targetARN = `arn:aws:iam::${AWSAccountID}:role/porter-role`
+    let targetARN = `arn:aws:iam::${AWSAccountID}:role/porter-manager`
 
     setRoleStatus("loading");
     setErrorMessage(undefined)
@@ -141,7 +141,7 @@ const CloudFormationForm: React.FC<Props> = ({
           }}
           status={
             errorMessage ? (
-              <Error 
+              <Error
                 message={errorMessage}
                 ctaText="Troubleshooting steps"
                 errorModalContents={

dashboard/src/components/ProvisionerSettings.tsx

@@ -75,7 +75,7 @@ const ProvisionerSettings: React.FC<Props> = props => {
   const [isExpanded, setIsExpanded] = useState(false);
   const [minInstances, setMinInstances] = useState(1);
   const [maxInstances, setMaxInstances] = useState(10);
-  const [cidrRange, setCidrRange] = useState("172.0.0.0/16");
+  const [cidrRange, setCidrRange] = useState("10.78.0.0/16");
   const [clusterVersion, setClusterVersion] = useState("v1.24.0");
   const [isReadOnly, setIsReadOnly] = useState(false);
   const [errorMessage, setErrorMessage] = useState<string>(undefined);
@@ -106,7 +106,7 @@ const ProvisionerSettings: React.FC<Props> = props => {
           value: new EKS({
             clusterName,
             clusterVersion: clusterVersion || "v1.24.0",
-            cidrRange: cidrRange || "172.0.0.0/16",
+            cidrRange: cidrRange || "10.78.0.0/16",
             region: awsRegion,
             nodeGroups: [
               new EKSNodeGroup({
@@ -313,7 +313,7 @@ const ProvisionerSettings: React.FC<Props> = props => {
                 value={cidrRange}
                 setValue={(x: string) => setCidrRange(x)}
                 label="VPC CIDR range"
-                placeholder="ex: 172.0.0.0/16"
+                placeholder="ex: 10.78.0.0/16"
               />
             </>
           )

dashboard/src/components/ProvisionerSettingsOld.tsx

@@ -74,15 +74,15 @@ const ProvisionerSettingsOld: React.FC<Props> = props => {
   const [isExpanded, setIsExpanded] = useState(false);
   const [minInstances, setMinInstances] = useState(1);
   const [maxInstances, setMaxInstances] = useState(10);
-  const [cidrRange, setCidrRange] = useState("172.0.0.0/16");
+  const [cidrRange, setCidrRange] = useState("10.78.0.0/16");
   const [clusterVersion, setClusterVersion] = useState("v1.24.0");
   const [isReadOnly, setIsReadOnly] = useState(false);
 
   const markProvisioningStarted = async () => {
     try {
       const res = await api.updateOnboardingStep(
-        "<token>", 
-        { step: "provisioning-started" }, 
+        "<token>",
+        { step: "provisioning-started" },
         {}
       );
     } catch (err) {
@@ -92,7 +92,7 @@ const ProvisionerSettingsOld: React.FC<Props> = props => {
 
   const createCluster = async () => {
     markProvisioningStarted();
-    
+
     var data = new Contract({
       cluster: new Cluster({
         projectId: currentProject.id,
@@ -104,7 +104,7 @@ const ProvisionerSettingsOld: React.FC<Props> = props => {
           value: new EKS({
             clusterName,
             clusterVersion: clusterVersion || "v1.24.0",
-            cidrRange: cidrRange || "172.0.0.0/16",
+            cidrRange: cidrRange || "10.78.0.0/16",
             region: awsRegion,
             nodeGroups: [
               new EKSNodeGroup({
@@ -203,7 +203,7 @@ const ProvisionerSettingsOld: React.FC<Props> = props => {
   }, [props.selectedClusterVersion]);
 
   const renderForm = () => {
-    
+
     // Render simplified form if initial create
     if (!props.clusterId) {
       return (
@@ -294,7 +294,7 @@ const ProvisionerSettingsOld: React.FC<Props> = props => {
                 value={cidrRange}
                 setValue={(x: string) => setCidrRange(x)}
                 label="VPC CIDR range"
-                placeholder="ex: 172.0.0.0/16"
+                placeholder="ex: 10.78.0.0/16"
               />
             </>
           )

internal/registry/registry.go

@@ -883,13 +883,21 @@ func (r *Registry) createGARRepository(
 
 // ListImages lists the images for an image repository
 func (r *Registry) ListImages(
+	ctx context.Context,
 	repoName string,
 	repo repository.Repository,
-	doAuth *oauth2.Config, // only required if using DOCR
+	conf *config.Config,
 ) ([]*ptypes.Image, error) {
 	// switch on the auth mechanism to get a token
 	if r.AWSIntegrationID != 0 {
-		return r.listECRImages(repoName, repo)
+		aws, err := repo.AWSIntegration().ReadAWSIntegration(
+			r.ProjectID,
+			r.AWSIntegrationID,
+		)
+		if err != nil {
+			return nil, err
+		}
+		return r.listECRImages(aws, repoName, repo)
 	}
 
 	if r.AzureIntegrationID != 0 {
@@ -905,13 +913,40 @@ func (r *Registry) ListImages(
 	}
 
 	if r.DOIntegrationID != 0 {
-		return r.listDOCRImages(repoName, repo, doAuth)
+		return r.listDOCRImages(repoName, repo, conf.DOConf)
 	}
 
 	if r.BasicIntegrationID != 0 {
 		return r.listPrivateRegistryImages(repoName, repo)
 	}
 
+	project, err := conf.Repo.Project().ReadProject(r.ProjectID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting project for repository: %w", err)
+	}
+
+	if project.CapiProvisionerEnabled {
+		uri := strings.TrimPrefix(r.URL, "https://")
+		splits := strings.Split(uri, ".")
+		accountID := splits[0]
+		region := splits[3]
+		req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
+			ProjectId:    int64(r.ProjectID),
+			AwsAccountId: accountID,
+		})
+		creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
+		if err != nil {
+			return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
+		}
+		aws := &ints.AWSIntegration{
+			AWSAccessKeyID:     []byte(creds.Msg.AwsAccessId),
+			AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
+			AWSSessionToken:    []byte(creds.Msg.AwsSessionToken),
+			AWSRegion:          region,
+		}
+		return r.listECRImages(aws, repoName, repo)
+	}
+
 	return nil, fmt.Errorf("error listing images")
 }
 
@@ -1025,15 +1060,7 @@ func (r *Registry) GetECRPaginatedImages(
 	return res, resp.NextToken, nil
 }
 
-func (r *Registry) listECRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
-	aws, err := repo.AWSIntegration().ReadAWSIntegration(
-		r.ProjectID,
-		r.AWSIntegrationID,
-	)
-	if err != nil {
-		return nil, err
-	}
-
+func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
 	sess, err := aws.GetSession()
 	if err != nil {
 		return nil, err

internal/repository/gorm/aws_assume_role_chain.go

@@ -3,6 +3,7 @@ package gorm
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/porter-dev/porter/internal/models"
 	"github.com/porter-dev/porter/internal/repository"
@@ -27,7 +28,19 @@ func (cr AWSAssumeRoleChain) List(ctx context.Context, projectID uint) ([]*model
 	if projectID == 0 {
 		return nil, errors.New("must provide a project ID")
 	}
-	tx := cr.db.Where("project_id = ? and target_arn not like '%arn:aws:iam::108458755588%'", projectID).Find(&confs)
+	// porterInternalAccounts are accounts which should be hidden from users, such as bastion or production accounts
+	porterInternalAccounts := []string{
+		"108458755588", // CAPI Bastion
+		"813111008191", // Internal Tooling Cluster
+		"975032674314", // Old production account
+	}
+
+	query := "project_id = ?"
+	for _, account := range porterInternalAccounts {
+		query += fmt.Sprintf(" and target_arn not like '%%arn:aws:iam::%s%%'", account)
+	}
+
+	tx := cr.db.Where(query, projectID).Find(&confs)
 	if tx.Error != nil {
 		return nil, tx.Error
 	}

zarf/docker/Dockerfile.production

@@ -0,0 +1,8 @@
+# All build artifacts are built in github actions, and copied in
+# Ubuntu is used to match Github action, and use github action caching instead of docker caching
+FROM ubuntu as runner
+WORKDIR /porter
+COPY bin/app /porter/
+COPY bin/migrate /porter/
+COPY build/ /porter/static
+RUN chmod +x /porter/app /porter/migrate