Merge pull request #2557 from porter-dev/nafees/gar-fixes

[POR-791] Use correct GAR image URI during launch flow

api/server/router/v1/registry.go

@@ -390,7 +390,7 @@ func getV1RegistryRoutes(
 	//   - name: registry_id
 	//   - name: repository
 	//     in: path
-	//     description: The image repository name
+	//     description: The image repository name. Should be of the form REPOSITORY/IMAGE when using Google Artifact Registry.
 	//     type: string
 	//     required: true
 	//   - name: num

cli/cmd/connect/gar.go

@@ -3,8 +3,8 @@ package connect
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
+	"strings"
 
 	"github.com/fatih/color"
 
@@ -23,8 +23,8 @@ func GAR(
 		return 0, fmt.Errorf("no project set, please run porter config set-project")
 	}
 
-	keyFileLocation, err := utils.PromptPlaintext(fmt.Sprintf(`Please provide the full path to a service account key file.
-Key file location: `))
+	keyFileLocation, err := utils.PromptPlaintext(`Please provide the full path to a service account key file.
+Key file location: `)
 
 	if err != nil {
 		return 0, err
@@ -33,7 +33,7 @@ Key file location: `))
 	// attempt to read the key file location
 	if info, err := os.Stat(keyFileLocation); !os.IsNotExist(err) && !info.IsDir() {
 		// read the file
-		bytes, err := ioutil.ReadFile(keyFileLocation)
+		bytes, err := os.ReadFile(keyFileLocation)
 
 		if err != nil {
 			return 0, err
@@ -54,8 +54,8 @@ Key file location: `))
 
 		color.New(color.FgGreen).Printf("created gcp integration with id %d\n", integration.ID)
 
-		region, err := utils.PromptPlaintext(fmt.Sprintf(`Please enter the artifact registry region. For example, us-central-1.
-Artifact registry region: `))
+		region, err := utils.PromptPlaintext(`Please enter the artifact registry region. For example, us-central1.
+Artifact registry region: `)
 
 		if err != nil {
 			return 0, err
@@ -63,19 +63,35 @@ Artifact registry region: `))
 
 		// create the registry
 		// query for registry name
-		regName, err := utils.PromptPlaintext(fmt.Sprintf(`Give this registry a name: `))
+		regName, err := utils.PromptPlaintext("Give this registry a name: ")
 
 		if err != nil {
 			return 0, err
 		}
 
+		// GCP project IDs can have the ':' character like example.com:my-project
+		// if this is the case then we need to case on this
+		//
+		// see: https://cloud.google.com/artifact-registry/docs/docker/names#domain
+		var registryURL string
+
+		if domain, projectID, found := strings.Cut(integration.GCPProjectID, ":"); found {
+			if domain == "" || projectID == "" {
+				return 0, fmt.Errorf("invalid project ID: %s", integration.GCPProjectID)
+			}
+
+			registryURL = fmt.Sprintf("%s-docker.pkg.dev/%s/%s", region, domain, projectID)
+		} else {
+			registryURL = fmt.Sprintf("%s-docker.pkg.dev/%s", region, integration.GCPProjectID)
+		}
+
 		reg, err := client.CreateRegistry(
 			context.Background(),
 			projectID,
 			&types.CreateRegistryRequest{
 				Name:             regName,
 				GCPIntegrationID: integration.ID,
-				URL:              region + "-docker.pkg.dev/" + integration.GCPProjectID,
+				URL:              registryURL,
 			},
 		)
 

dashboard/src/components/image-selector/TagList.tsx

@@ -46,7 +46,13 @@ export default class TagList extends Component<PropsType, StateType> {
     const { currentProject } = this.context;
 
     let splits = this.props.selectedImageUrl.split("/");
-    let repoName = splits[splits.length - 1];
+    let repoName: string;
+
+    if (this.props.selectedImageUrl.includes("pkg.dev")) {
+      repoName = splits[splits.length - 2] + "/" + splits[splits.length - 1];
+    } else {
+      repoName = splits[splits.length - 1];
+    }
 
     let matches = this.props.selectedImageUrl.match(ecrRepoRegex);
 

dashboard/src/main/home/integrations/create-integration/GARForm.tsx

@@ -59,12 +59,37 @@ const GARForm = (props: { closeForm: () => void }) => {
     }
 
     try {
+      let registryURL: string;
+
+      // GCP project IDs can have the ':' character like example.com:my-project
+      // if this is the case then we need to case on this
+      //
+      // see: https://cloud.google.com/artifact-registry/docs/docker/names#domain
+      if (integration.gcp_project_id.includes(":")) {
+        const domainProjectID = integration.gcp_project_id.split(":");
+
+        if (
+          domainProjectID.length !== 2 ||
+          domainProjectID[0].length === 0 ||
+          domainProjectID[1].length === 0
+        ) {
+          setButtonStatus(
+            "Invalid GCP project ID. Please check your credentials."
+          );
+          return;
+        }
+
+        registryURL = `${region}-docker.pkg.dev/${domainProjectID[0]}/${domainProjectID[1]}`;
+      } else {
+        registryURL = `${region}-docker.pkg.dev/${integration.gcp_project_id}`;
+      }
+
       await api.connectGCRRegistry(
         "token",
         {
           gcp_integration_id: integration.id,
           name: credentialsName,
-          url: `${region}-docker.pkg.dev/${integration.gcp_project_id}`,
+          url: registryURL,
         },
         { id: currentProject.id }
       );

dashboard/src/main/home/launch/launch-flow/LaunchFlow.tsx

@@ -71,11 +71,16 @@ const LaunchFlow: React.FC<PropsType> = (props) => {
   };
 
   const getFullActionConfig = (): FullActionConfigType => {
-    let imageRepoUri = `${selectedRegistry?.url}/${templateName}-${selectedNamespace}`;
+    let imageRepoURI = `${selectedRegistry?.url}/${templateName}-${selectedNamespace}`;
 
     // DockerHub registry integration is per repo
     if (selectedRegistry?.service === "dockerhub") {
-      imageRepoUri = selectedRegistry?.url;
+      imageRepoURI = selectedRegistry?.url;
+    }
+
+    // Customize image repo URI for GAR
+    if (imageRepoURI.includes("pkg.dev")) {
+      imageRepoURI = `${imageRepoURI}/${templateName}-${selectedNamespace}`;
     }
 
     if (actionConfig.kind === "github") {
@@ -86,7 +91,7 @@ const LaunchFlow: React.FC<PropsType> = (props) => {
         registry_id: selectedRegistry?.id,
         dockerfile_path: dockerfilePath,
         folder_path: folderPath,
-        image_repo_uri: imageRepoUri,
+        image_repo_uri: imageRepoURI,
         git_repo_id: actionConfig.git_repo_id,
         should_create_workflow: shouldCreateWorkflow,
       };
@@ -98,7 +103,7 @@ const LaunchFlow: React.FC<PropsType> = (props) => {
         registry_id: selectedRegistry?.id,
         dockerfile_path: dockerfilePath,
         folder_path: folderPath,
-        image_repo_uri: imageRepoUri,
+        image_repo_uri: imageRepoURI,
         gitlab_integration_id: actionConfig.gitlab_integration_id,
         should_create_workflow: shouldCreateWorkflow,
       };

dashboard/src/main/home/onboarding/steps/ConnectRegistry/forms/_GCPRegistryForm.tsx

@@ -351,7 +351,7 @@ export const GARegistryConfig: React.FC<{
 
     setButtonStatus("loading");
 
-    let gcpProjectId = NaN;
+    let gcpProjectId: string;
 
     try {
       const gcp_integration = await api
@@ -375,7 +375,30 @@ export const GARegistryConfig: React.FC<{
       return;
     }
 
-    const registryUrl = `${region}-docker.pkg.dev/${gcpProjectId}`;
+    let registryURL: string;
+
+    // GCP project IDs can have the ':' character like example.com:my-project
+    // if this is the case then we need to case on this
+    //
+    // see: https://cloud.google.com/artifact-registry/docs/docker/names#domain
+    if (gcpProjectId.includes(":")) {
+      const domainProjectID = gcpProjectId.split(":");
+
+      if (
+        domainProjectID.length !== 2 ||
+        domainProjectID[0].length === 0 ||
+        domainProjectID[1].length === 0
+      ) {
+        setButtonStatus(
+          "Invalid GCP project ID. Please check your credentials."
+        );
+        return;
+      }
+
+      registryURL = `${region}-docker.pkg.dev/${domainProjectID[0]}/${domainProjectID[1]}`;
+    } else {
+      registryURL = `${region}-docker.pkg.dev/${gcpProjectId}`;
+    }
 
     try {
       const data = await api
@@ -385,7 +408,7 @@ export const GARegistryConfig: React.FC<{
             name: registryName,
             gcp_integration_id:
               snap.StateHandler.connected_registry.credentials.id,
-            url: registryUrl,
+            url: registryURL,
           },
           {
             id: project.id,
@@ -395,7 +418,7 @@ export const GARegistryConfig: React.FC<{
       nextFormStep({
         settings: {
           registry_connection_id: data.id,
-          gcr_url: registryUrl,
+          gcr_url: registryURL,
           registry_name: registryName,
         },
       });

internal/helm/postrenderer.go

@@ -815,7 +815,18 @@ func getRegNameFromImageRef(image string) (string, error) {
 	if strings.Contains(domain, "docker.io") {
 		regName = "index.docker.io/" + path
 	} else if strings.Contains(domain, "pkg.dev") {
-		regName = domain + "/" + strings.Split(path, "/")[0]
+		pathSlice := strings.Split(path, "/")
+
+		// a GAR image path can either be PROJECT-ID/REPOSITORY/IMAGE or DOMAIN/PROJECT-ID/REPOSITORY/IMAGE
+		//
+		// see: https://cloud.google.com/artifact-registry/docs/docker/names#domain
+		if len(pathSlice) == 3 {
+			regName = fmt.Sprintf("%s/%s", domain, pathSlice[0])
+		} else if len(pathSlice) == 4 {
+			regName = fmt.Sprintf("%s/%s/%s", domain, pathSlice[0], pathSlice[1])
+		} else {
+			return "", fmt.Errorf("invalid GAR image: %s", image)
+		}
 	} else {
 		regName = domain
 

internal/registry/registry.go

@@ -31,6 +31,7 @@ import (
 	"github.com/digitalocean/godo"
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/cli/cli/config/types"
+	"github.com/docker/distribution/reference"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
 
@@ -77,9 +78,9 @@ func (r *Registry) ListRepositories(
 	if r.GCPIntegrationID != 0 {
 		if strings.Contains(r.URL, "pkg.dev") {
 			return r.listGARRepositories(repo)
-		} else {
-			return r.listGCRRepositories(repo)
 		}
+
+		return r.listGCRRepositories(repo)
 	}
 
 	if r.DOIntegrationID != 0 {
@@ -241,6 +242,9 @@ func (source *garTokenSource) Token() (*oauth2.Token, error) {
 	return source.reg.GetGARToken(source.repo)
 }
 
+// GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
+// where a repository is a single image. This function returns the list of fully qualified names
+// of GAR images including their repository names.
 func (r *Registry) listGARRepositories(
 	repo repository.Repository,
 ) ([]*ptypes.RegistryRepository, error) {
@@ -262,7 +266,7 @@ func (r *Registry) listGARRepositories(
 		return nil, err
 	}
 
-	var res []*ptypes.RegistryRepository
+	var repoNames []string
 	nextToken := ""
 
 	parsedURL, err := url.Parse("https://" + r.URL)
@@ -289,14 +293,12 @@ func (r *Registry) listGARRepositories(
 				return nil, err
 			}
 
-			repoSlice := strings.Split(resp.GetName(), "/")
-			repoName := repoSlice[len(repoSlice)-1]
+			if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
+				repoSlice := strings.Split(resp.GetName(), "/")
+				repoName := repoSlice[len(repoSlice)-1]
 
-			res = append(res, &ptypes.RegistryRepository{
-				Name:      resp.GetName(),
-				CreatedAt: resp.GetCreateTime().AsTime(),
-				URI:       parsedURL.Host + "/" + gcpInt.GCPProjectID + "/" + repoName,
-			})
+				repoNames = append(repoNames, repoName)
+			}
 		}
 
 		if it.PageInfo().Token == "" {
@@ -306,6 +308,74 @@ func (r *Registry) listGARRepositories(
 		nextToken = it.PageInfo().Token
 	}
 
+	svc, err := v1artifactregistry.NewService(context.Background(), option.WithTokenSource(&garTokenSource{
+		reg:  r,
+		repo: repo,
+	}), option.WithScopes("roles/artifactregistry.reader"))
+
+	if err != nil {
+		return nil, err
+	}
+
+	nextToken = ""
+
+	dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
+
+	var (
+		wg     sync.WaitGroup
+		resMap sync.Map
+	)
+
+	for _, repoName := range repoNames {
+		wg.Add(1)
+
+		go func(repoName string) {
+			defer wg.Done()
+
+			for {
+				resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
+					gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
+
+				if err != nil {
+					// FIXME: we should report this error using a channel
+					return
+				}
+
+				for _, image := range resp.DockerImages {
+					named, err := reference.ParseNamed(image.Uri)
+
+					if err != nil {
+						// let us skip this image becaue it has a malformed URI coming from the GCP API
+						continue
+					}
+
+					uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
+
+					resMap.Store(named.Name(), &ptypes.RegistryRepository{
+						Name:      repoName,
+						URI:       named.Name(),
+						CreatedAt: uploadTime,
+					})
+				}
+
+				if resp.NextPageToken == "" {
+					break
+				}
+
+				nextToken = resp.NextPageToken
+			}
+		}(repoName)
+	}
+
+	wg.Wait()
+
+	var res []*ptypes.RegistryRepository
+
+	resMap.Range(func(_, value any) bool {
+		res = append(res, value.(*ptypes.RegistryRepository))
+		return true
+	})
+
 	return res, nil
 }
 
@@ -1172,6 +1242,12 @@ func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([
 }
 
 func (r *Registry) listGARImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
+	repoImageSlice := strings.Split(repoName, "/")
+
+	if len(repoImageSlice) != 2 {
+		return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
+	}
+
 	gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
 		r.ProjectID,
 		r.GCPIntegrationID,
@@ -1190,7 +1266,6 @@ func (r *Registry) listGARImages(repoName string, repo repository.Repository) ([
 		return nil, err
 	}
 
-	nextToken := ""
 	var res []*ptypes.Image
 
 	parsedURL, err := url.Parse("https://" + r.URL)
@@ -1200,27 +1275,39 @@ func (r *Registry) listGARImages(repoName string, repo repository.Repository) ([
 	}
 
 	location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
-
 	dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
+	nextToken := ""
 
 	for {
 		resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
-			gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
+			gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
 
 		if err != nil {
 			return nil, err
 		}
 
 		for _, image := range resp.DockerImages {
-			uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
-
-			for _, tag := range image.Tags {
-				res = append(res, &ptypes.Image{
-					RepositoryName: repoName,
-					Tag:            tag,
-					PushedAt:       &uploadTime,
-					Digest:         strings.Split(image.Name, "@")[1],
-				})
+			named, err := reference.ParseNamed(image.Uri)
+
+			if err != nil {
+				continue
+			}
+
+			paths := strings.Split(reference.Path(named), "/")
+
+			imageName := paths[len(paths)-1]
+
+			if imageName == repoImageSlice[1] {
+				uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
+
+				for _, tag := range image.Tags {
+					res = append(res, &ptypes.Image{
+						RepositoryName: repoName,
+						Tag:            tag,
+						PushedAt:       &uploadTime,
+						Digest:         strings.Split(image.Uri, "@")[1],
+					})
+				}
 			}
 		}