registry.go 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976
  1. package registry
  2. import (
  3. "context"
  4. "encoding/base64"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "net/http"
  9. "net/url"
  10. "sort"
  11. "strings"
  12. "sync"
  13. "time"
  14. "github.com/porter-dev/porter/internal/telemetry"
  15. artifactregistry "cloud.google.com/go/artifactregistry/apiv1beta2"
  16. "connectrpc.com/connect"
  17. "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  18. "github.com/aws/aws-sdk-go/aws/awserr"
  19. "github.com/aws/aws-sdk-go/service/ecr"
  20. porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
  21. "github.com/porter-dev/porter/api/server/shared/config"
  22. "github.com/porter-dev/porter/internal/models"
  23. "github.com/porter-dev/porter/internal/oauth"
  24. "github.com/porter-dev/porter/internal/repository"
  25. "golang.org/x/oauth2"
  26. v1artifactregistry "google.golang.org/api/artifactregistry/v1"
  27. "google.golang.org/api/iterator"
  28. "google.golang.org/api/option"
  29. artifactregistrypb "google.golang.org/genproto/googleapis/devtools/artifactregistry/v1beta2"
  30. ints "github.com/porter-dev/porter/internal/models/integrations"
  31. ptypes "github.com/porter-dev/porter/api/types"
  32. "github.com/digitalocean/godo"
  33. "github.com/docker/cli/cli/config/configfile"
  34. "github.com/docker/cli/cli/config/types"
  35. "github.com/docker/distribution/reference"
  36. "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
  37. "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
  38. )
  39. // Registry wraps the gorm Registry model
  40. type Registry models.Registry
  41. func GetECRRegistryURL(awsIntRepo repository.AWSIntegrationRepository, projectID, awsIntID uint) (string, error) {
  42. awsInt, err := awsIntRepo.ReadAWSIntegration(projectID, awsIntID)
  43. if err != nil {
  44. return "", err
  45. }
  46. sess, err := awsInt.GetSession()
  47. if err != nil {
  48. return "", err
  49. }
  50. ecrSvc := ecr.New(sess)
  51. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  52. if err != nil {
  53. return "", err
  54. }
  55. return *output.AuthorizationData[0].ProxyEndpoint, nil
  56. }
  57. // ListRepositories lists the repositories for a registry
  58. func (r *Registry) ListRepositories(
  59. ctx context.Context,
  60. repo repository.Repository,
  61. conf *config.Config,
  62. ) ([]*ptypes.RegistryRepository, error) {
  63. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  64. defer span.End()
  65. telemetry.WithAttributes(span,
  66. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  67. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  68. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  69. )
  70. // switch on the auth mechanism to get a token
  71. if r.AWSIntegrationID != 0 {
  72. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "aws"})
  73. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  74. r.ProjectID,
  75. r.AWSIntegrationID,
  76. )
  77. if err != nil {
  78. return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
  79. }
  80. repos, err := r.listECRRepositories(aws)
  81. if err != nil {
  82. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  83. }
  84. return repos, nil
  85. }
  86. if r.GCPIntegrationID != 0 {
  87. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
  88. if strings.Contains(r.URL, "pkg.dev") {
  89. return r.listGARRepositories(ctx, repo)
  90. }
  91. repos, err := r.listGCRRepositories(repo)
  92. if err != nil {
  93. return nil, telemetry.Error(ctx, span, err, "error listing gcr repositories")
  94. }
  95. return repos, nil
  96. }
  97. if r.DOIntegrationID != 0 {
  98. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "do"})
  99. repos, err := r.listDOCRRepositories(repo, conf.DOConf)
  100. if err != nil {
  101. return nil, telemetry.Error(ctx, span, err, "error listing docr repositories")
  102. }
  103. return repos, nil
  104. }
  105. if r.AzureIntegrationID != 0 {
  106. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "azure"})
  107. repos, err := r.listACRRepositories(ctx, repo)
  108. if err != nil {
  109. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  110. }
  111. return repos, nil
  112. }
  113. if r.BasicIntegrationID != 0 {
  114. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "basic"})
  115. repos, err := r.listPrivateRegistryRepositories(repo)
  116. if err != nil {
  117. return nil, telemetry.Error(ctx, span, err, "error listing private repositories")
  118. }
  119. return repos, nil
  120. }
  121. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  122. if err != nil {
  123. return nil, telemetry.Error(ctx, span, err, "error getting project for repository")
  124. }
  125. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  126. // TODO: Remove this conditional when AWS list repos is supported in CCP
  127. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "registry-uri", Value: r.URL})
  128. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  129. req := connect.NewRequest(&porterv1.ListRepositoriesForRegistryRequest{
  130. ProjectId: int64(r.ProjectID),
  131. RegistryUri: r.URL,
  132. })
  133. resp, err := conf.ClusterControlPlaneClient.ListRepositoriesForRegistry(ctx, req)
  134. if err != nil {
  135. return nil, telemetry.Error(ctx, span, err, "error listing docker repositories")
  136. }
  137. res := make([]*ptypes.RegistryRepository, 0)
  138. parsedURL, err := url.Parse("https://" + r.URL)
  139. if err != nil {
  140. return nil, telemetry.Error(ctx, span, err, "error parsing url")
  141. }
  142. for _, repo := range resp.Msg.Repositories {
  143. res = append(res, &ptypes.RegistryRepository{
  144. Name: repo.Name,
  145. URI: parsedURL.Host + "/" + repo.Name,
  146. })
  147. }
  148. return res, nil
  149. } else {
  150. uri := strings.TrimPrefix(r.URL, "https://")
  151. splits := strings.Split(uri, ".")
  152. if len(splits) < 4 {
  153. return nil, telemetry.Error(ctx, span, nil, "uri does not have enough splits")
  154. }
  155. accountID := splits[0]
  156. region := splits[3]
  157. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  158. ProjectId: int64(r.ProjectID),
  159. AwsAccountId: accountID,
  160. })
  161. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  162. if err != nil {
  163. return nil, telemetry.Error(ctx, span, err, "error getting capi credentials for registry")
  164. }
  165. aws := &ints.AWSIntegration{
  166. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  167. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  168. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  169. AWSRegion: region,
  170. }
  171. repos, err := r.listECRRepositories(aws)
  172. if err != nil {
  173. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  174. }
  175. return repos, nil
  176. }
  177. }
  178. return nil, telemetry.Error(ctx, span, nil, "error listing repositories")
  179. }
  180. type gcrJWT struct {
  181. AccessToken string `json:"token"`
  182. ExpiresInSec int `json:"expires_in"`
  183. }
  184. type gcrErr struct {
  185. Code string `json:"code"`
  186. Message string `json:"message"`
  187. }
  188. type gcrRepositoryResp struct {
  189. Repositories []string `json:"repositories"`
  190. Errors []gcrErr `json:"errors"`
  191. }
  192. func (r *Registry) GetGCRToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  193. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  194. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  195. r.ProjectID,
  196. r.GCPIntegrationID,
  197. )
  198. if err != nil {
  199. return nil, err
  200. }
  201. // get oauth2 access token
  202. return gcp.GetBearerToken(
  203. ctx,
  204. getTokenCache,
  205. r.setTokenCacheFunc(ctx, repo),
  206. "https://www.googleapis.com/auth/devstorage.read_write",
  207. )
  208. }
  209. func (r *Registry) listGCRRepositories(
  210. repo repository.Repository,
  211. ) ([]*ptypes.RegistryRepository, error) {
  212. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  213. r.ProjectID,
  214. r.GCPIntegrationID,
  215. )
  216. if err != nil {
  217. return nil, err
  218. }
  219. // Just use service account key to authenticate, since scopes may not be in place
  220. // for oauth. This also prevents us from making more requests.
  221. client := &http.Client{}
  222. regURL := r.URL
  223. if !strings.HasPrefix(regURL, "http") {
  224. regURL = fmt.Sprintf("https://%s", regURL)
  225. }
  226. regURLParsed, err := url.Parse(regURL)
  227. regHostname := "gcr.io"
  228. if err == nil {
  229. regHostname = regURLParsed.Host
  230. }
  231. req, err := http.NewRequest(
  232. "GET",
  233. fmt.Sprintf("https://%s/v2/_catalog", regHostname),
  234. nil,
  235. )
  236. if err != nil {
  237. return nil, err
  238. }
  239. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  240. resp, err := client.Do(req)
  241. if err != nil {
  242. return nil, err
  243. }
  244. gcrResp := gcrRepositoryResp{}
  245. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  246. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  247. }
  248. if len(gcrResp.Errors) > 0 {
  249. errMsg := ""
  250. for _, gcrErr := range gcrResp.Errors {
  251. errMsg += fmt.Sprintf(": Code %s, message %s", gcrErr.Code, gcrErr.Message)
  252. }
  253. return nil, fmt.Errorf(errMsg)
  254. }
  255. res := make([]*ptypes.RegistryRepository, 0)
  256. parsedURL, err := url.Parse("https://" + r.URL)
  257. if err != nil {
  258. return nil, err
  259. }
  260. for _, repo := range gcrResp.Repositories {
  261. res = append(res, &ptypes.RegistryRepository{
  262. Name: repo,
  263. URI: parsedURL.Host + "/" + repo,
  264. })
  265. }
  266. return res, nil
  267. }
  268. func (r *Registry) GetGARToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  269. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  270. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  271. r.ProjectID,
  272. r.GCPIntegrationID,
  273. )
  274. if err != nil {
  275. return nil, err
  276. }
  277. // get oauth2 access token
  278. return gcp.GetBearerToken(
  279. ctx,
  280. getTokenCache,
  281. r.setTokenCacheFunc(ctx, repo),
  282. "https://www.googleapis.com/auth/cloud-platform",
  283. )
  284. }
  285. type garTokenSource struct {
  286. // ctx is only passed in here as the oauth2.Token() doesnt support contexts
  287. ctx context.Context
  288. reg *Registry
  289. repo repository.Repository
  290. }
  291. func (source *garTokenSource) Token() (*oauth2.Token, error) {
  292. return source.reg.GetGARToken(source.ctx, source.repo)
  293. }
  294. // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
  295. // where a repository is a single image. This function returns the list of fully qualified names
  296. // of GAR images including their repository names.
  297. func (r *Registry) listGARRepositories(
  298. ctx context.Context,
  299. repo repository.Repository,
  300. ) ([]*ptypes.RegistryRepository, error) {
  301. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  302. r.ProjectID,
  303. r.GCPIntegrationID,
  304. )
  305. if err != nil {
  306. return nil, err
  307. }
  308. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  309. reg: r,
  310. repo: repo,
  311. ctx: ctx,
  312. }), option.WithScopes("roles/artifactregistry.reader"))
  313. if err != nil {
  314. return nil, err
  315. }
  316. var repoNames []string
  317. nextToken := ""
  318. parsedURL, err := url.Parse("https://" + r.URL)
  319. if err != nil {
  320. return nil, err
  321. }
  322. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  323. for {
  324. it := client.ListRepositories(context.Background(), &artifactregistrypb.ListRepositoriesRequest{
  325. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  326. PageSize: 1000,
  327. PageToken: nextToken,
  328. })
  329. for {
  330. resp, err := it.Next()
  331. if err == iterator.Done {
  332. break
  333. } else if err != nil {
  334. return nil, err
  335. }
  336. if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
  337. repoSlice := strings.Split(resp.GetName(), "/")
  338. repoName := repoSlice[len(repoSlice)-1]
  339. repoNames = append(repoNames, repoName)
  340. }
  341. }
  342. if it.PageInfo().Token == "" {
  343. break
  344. }
  345. nextToken = it.PageInfo().Token
  346. }
  347. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  348. reg: r,
  349. repo: repo,
  350. ctx: ctx,
  351. }), option.WithScopes("roles/artifactregistry.reader"))
  352. if err != nil {
  353. return nil, err
  354. }
  355. nextToken = ""
  356. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  357. var (
  358. wg sync.WaitGroup
  359. resMap sync.Map
  360. )
  361. for _, repoName := range repoNames {
  362. wg.Add(1)
  363. go func(repoName string) {
  364. defer wg.Done()
  365. for {
  366. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  367. gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
  368. if err != nil {
  369. // FIXME: we should report this error using a channel
  370. return
  371. }
  372. for _, image := range resp.DockerImages {
  373. named, err := reference.ParseNamed(image.Uri)
  374. if err != nil {
  375. // let us skip this image becaue it has a malformed URI coming from the GCP API
  376. continue
  377. }
  378. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  379. resMap.Store(named.Name(), &ptypes.RegistryRepository{
  380. Name: repoName,
  381. URI: named.Name(),
  382. CreatedAt: uploadTime,
  383. })
  384. }
  385. if resp.NextPageToken == "" {
  386. break
  387. }
  388. nextToken = resp.NextPageToken
  389. }
  390. }(repoName)
  391. }
  392. wg.Wait()
  393. var res []*ptypes.RegistryRepository
  394. resMap.Range(func(_, value any) bool {
  395. res = append(res, value.(*ptypes.RegistryRepository))
  396. return true
  397. })
  398. return res, nil
  399. }
  400. func (r *Registry) listECRRepositories(aws *ints.AWSIntegration) ([]*ptypes.RegistryRepository, error) {
  401. sess, err := aws.GetSession()
  402. if err != nil {
  403. return nil, err
  404. }
  405. svc := ecr.New(sess)
  406. res := make([]*ptypes.RegistryRepository, 0)
  407. input := &ecr.DescribeRepositoriesInput{}
  408. for {
  409. resp, err := svc.DescribeRepositories(input)
  410. if err != nil {
  411. return nil, err
  412. }
  413. for _, repo := range resp.Repositories {
  414. res = append(res, &ptypes.RegistryRepository{
  415. Name: *repo.RepositoryName,
  416. CreatedAt: *repo.CreatedAt,
  417. URI: *repo.RepositoryUri,
  418. })
  419. }
  420. if resp.NextToken == nil {
  421. break
  422. }
  423. input.NextToken = resp.NextToken
  424. }
  425. return res, nil
  426. }
  427. type acrRepositoryResp struct {
  428. Repositories []string `json:"repositories"`
  429. }
  430. func (r *Registry) listACRRepositories(ctx context.Context, repo repository.Repository) ([]*ptypes.RegistryRepository, error) {
  431. ctx, span := telemetry.NewSpan(ctx, "list-acr-repositories")
  432. defer span.End()
  433. telemetry.WithAttributes(span,
  434. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  435. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  436. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  437. )
  438. az, err := repo.AzureIntegration().ReadAzureIntegration(
  439. r.ProjectID,
  440. r.AzureIntegrationID,
  441. )
  442. if err != nil {
  443. return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
  444. }
  445. client := &http.Client{}
  446. acrURL := r.URL
  447. if !strings.Contains(acrURL, "http") {
  448. acrURL = fmt.Sprintf("https://%s", acrURL)
  449. }
  450. req, err := http.NewRequest(
  451. "GET",
  452. fmt.Sprintf("%s/v2/_catalog", acrURL),
  453. nil,
  454. )
  455. if err != nil {
  456. return nil, telemetry.Error(ctx, span, err, "error getting http request")
  457. }
  458. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  459. resp, err := client.Do(req)
  460. if err != nil {
  461. return nil, telemetry.Error(ctx, span, err, "error making http call")
  462. }
  463. acrResp := acrRepositoryResp{}
  464. if err := json.NewDecoder(resp.Body).Decode(&acrResp); err != nil {
  465. return nil, telemetry.Error(ctx, span, err, "could not read Azure registry repository response")
  466. }
  467. res := make([]*ptypes.RegistryRepository, 0)
  468. for _, repo := range acrResp.Repositories {
  469. res = append(res, &ptypes.RegistryRepository{
  470. Name: repo,
  471. URI: strings.TrimPrefix(r.URL, "https://") + "/" + repo,
  472. })
  473. }
  474. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "repo-count", Value: len(acrResp.Repositories)})
  475. return res, nil
  476. }
  477. // Returns the username/password pair for the registry
  478. func (r *Registry) GetACRCredentials(repo repository.Repository) (string, string, error) {
  479. az, err := repo.AzureIntegration().ReadAzureIntegration(
  480. r.ProjectID,
  481. r.AzureIntegrationID,
  482. )
  483. if err != nil {
  484. return "", "", err
  485. }
  486. // if the passwords and name aren't set, generate them
  487. if az.ACRTokenName == "" || len(az.ACRPassword1) == 0 {
  488. az.ACRTokenName = "porter-acr-token"
  489. // create an acr repo token
  490. cred, err := azidentity.NewClientSecretCredential(az.AzureTenantID, az.AzureClientID, string(az.ServicePrincipalSecret), nil)
  491. if err != nil {
  492. return "", "", err
  493. }
  494. scopeMapsClient, err := armcontainerregistry.NewScopeMapsClient(az.AzureSubscriptionID, cred, nil)
  495. if err != nil {
  496. return "", "", err
  497. }
  498. smRes, err := scopeMapsClient.Get(
  499. context.Background(),
  500. az.ACRResourceGroupName,
  501. az.ACRName,
  502. "_repositories_admin",
  503. nil,
  504. )
  505. if err != nil {
  506. return "", "", err
  507. }
  508. tokensClient, err := armcontainerregistry.NewTokensClient(az.AzureSubscriptionID, cred, nil)
  509. if err != nil {
  510. return "", "", err
  511. }
  512. pollerResp, err := tokensClient.BeginCreate(
  513. context.Background(),
  514. az.ACRResourceGroupName,
  515. az.ACRName,
  516. "porter-acr-token",
  517. armcontainerregistry.Token{
  518. Properties: &armcontainerregistry.TokenProperties{
  519. ScopeMapID: smRes.ID,
  520. Status: to.Ptr(armcontainerregistry.TokenStatusEnabled),
  521. },
  522. },
  523. nil,
  524. )
  525. if err != nil {
  526. return "", "", err
  527. }
  528. tokResp, err := pollerResp.PollUntilDone(context.Background(), 2*time.Second)
  529. if err != nil {
  530. return "", "", err
  531. }
  532. registriesClient, err := armcontainerregistry.NewRegistriesClient(az.AzureSubscriptionID, cred, nil)
  533. if err != nil {
  534. return "", "", err
  535. }
  536. poller, err := registriesClient.BeginGenerateCredentials(
  537. context.Background(),
  538. az.ACRResourceGroupName,
  539. az.ACRName,
  540. armcontainerregistry.GenerateCredentialsParameters{
  541. TokenID: tokResp.ID,
  542. },
  543. &armcontainerregistry.RegistriesClientBeginGenerateCredentialsOptions{ResumeToken: ""})
  544. if err != nil {
  545. return "", "", err
  546. }
  547. genCredentialsResp, err := poller.PollUntilDone(context.Background(), 2*time.Second)
  548. if err != nil {
  549. return "", "", err
  550. }
  551. for i, tokPassword := range genCredentialsResp.Passwords {
  552. if i == 0 {
  553. az.ACRPassword1 = []byte(*tokPassword.Value)
  554. } else if i == 1 {
  555. az.ACRPassword2 = []byte(*tokPassword.Value)
  556. }
  557. }
  558. // update the az integration
  559. az, err = repo.AzureIntegration().OverwriteAzureIntegration(
  560. az,
  561. )
  562. if err != nil {
  563. return "", "", err
  564. }
  565. }
  566. return az.ACRTokenName, string(az.ACRPassword1), nil
  567. }
  568. func (r *Registry) listDOCRRepositories(
  569. repo repository.Repository,
  570. doAuth *oauth2.Config,
  571. ) ([]*ptypes.RegistryRepository, error) {
  572. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  573. r.ProjectID,
  574. r.DOIntegrationID,
  575. )
  576. if err != nil {
  577. return nil, err
  578. }
  579. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  580. if err != nil {
  581. return nil, err
  582. }
  583. client := godo.NewFromToken(tok)
  584. urlArr := strings.Split(r.URL, "/")
  585. if len(urlArr) != 2 {
  586. return nil, fmt.Errorf("invalid digital ocean registry url")
  587. }
  588. name := urlArr[1]
  589. repos, _, err := client.Registry.ListRepositories(context.TODO(), name, &godo.ListOptions{})
  590. if err != nil {
  591. return nil, err
  592. }
  593. res := make([]*ptypes.RegistryRepository, 0)
  594. for _, repo := range repos {
  595. res = append(res, &ptypes.RegistryRepository{
  596. Name: repo.Name,
  597. URI: r.URL + "/" + repo.Name,
  598. })
  599. }
  600. return res, nil
  601. }
  602. func (r *Registry) listPrivateRegistryRepositories(
  603. repo repository.Repository,
  604. ) ([]*ptypes.RegistryRepository, error) {
  605. // handle dockerhub different, as it doesn't implement the docker registry http api
  606. if strings.Contains(r.URL, "docker.io") {
  607. // in this case, we just return the single dockerhub repository that's linked
  608. res := make([]*ptypes.RegistryRepository, 0)
  609. res = append(res, &ptypes.RegistryRepository{
  610. Name: strings.Split(r.URL, "docker.io/")[1],
  611. URI: r.URL,
  612. })
  613. return res, nil
  614. }
  615. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  616. r.ProjectID,
  617. r.BasicIntegrationID,
  618. )
  619. if err != nil {
  620. return nil, err
  621. }
  622. // Just use service account key to authenticate, since scopes may not be in place
  623. // for oauth. This also prevents us from making more requests.
  624. client := &http.Client{}
  625. // get the host and scheme to make the request
  626. parsedURL, err := url.Parse(r.URL)
  627. req, err := http.NewRequest(
  628. "GET",
  629. fmt.Sprintf("%s://%s/v2/_catalog", parsedURL.Scheme, parsedURL.Host),
  630. nil,
  631. )
  632. if err != nil {
  633. return nil, err
  634. }
  635. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  636. resp, err := client.Do(req)
  637. if err != nil {
  638. return nil, err
  639. }
  640. // if the status code is 404, fallback to the Docker Hub implementation
  641. if resp.StatusCode == 404 {
  642. req, err := http.NewRequest(
  643. "GET",
  644. fmt.Sprintf("%s/", r.URL),
  645. nil,
  646. )
  647. if err != nil {
  648. return nil, err
  649. }
  650. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  651. resp, err = client.Do(req)
  652. if err != nil {
  653. return nil, err
  654. }
  655. }
  656. gcrResp := gcrRepositoryResp{}
  657. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  658. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  659. }
  660. res := make([]*ptypes.RegistryRepository, 0)
  661. if err != nil {
  662. return nil, err
  663. }
  664. for _, repo := range gcrResp.Repositories {
  665. res = append(res, &ptypes.RegistryRepository{
  666. Name: repo,
  667. URI: parsedURL.Host + "/" + repo,
  668. })
  669. }
  670. return res, nil
  671. }
  672. func (r *Registry) getTokenCacheFunc(
  673. ctx context.Context,
  674. repo repository.Repository,
  675. ) ints.GetTokenCacheFunc {
  676. return func(ctx context.Context) (tok *ints.TokenCache, err error) {
  677. reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
  678. if err != nil {
  679. return nil, err
  680. }
  681. return &reg.TokenCache.TokenCache, nil
  682. }
  683. }
  684. func (r *Registry) setTokenCacheFunc(
  685. ctx context.Context,
  686. repo repository.Repository,
  687. ) ints.SetTokenCacheFunc {
  688. return func(ctx context.Context, token string, expiry time.Time) error {
  689. _, err := repo.Registry().UpdateRegistryTokenCache(
  690. &ints.RegTokenCache{
  691. TokenCache: ints.TokenCache{
  692. Token: []byte(token),
  693. Expiry: expiry,
  694. },
  695. RegistryID: r.ID,
  696. },
  697. )
  698. return err
  699. }
  700. }
  701. // CreateRepository creates a repository for a registry, if needed
  702. // (currently only required for ECR)
  703. func (r *Registry) CreateRepository(
  704. ctx context.Context,
  705. conf *config.Config,
  706. name string,
  707. ) error {
  708. ctx, span := telemetry.NewSpan(ctx, "create-repository")
  709. defer span.End()
  710. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "registry-uri", Value: r.URL})
  711. // if aws, create repository
  712. if r.AWSIntegrationID != 0 {
  713. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "aws-integration-id", Value: r.AWSIntegrationID})
  714. aws, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
  715. r.ProjectID,
  716. r.AWSIntegrationID,
  717. )
  718. if err != nil {
  719. return telemetry.Error(ctx, span, err, "error reading aws integration")
  720. }
  721. err = r.createECRRepository(aws, name)
  722. if err != nil {
  723. return telemetry.Error(ctx, span, err, "error creating ecr repository")
  724. }
  725. return nil
  726. } else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
  727. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "gcp-integration-id", Value: r.GCPIntegrationID})
  728. err := r.createGARRepository(ctx, conf.Repo, name)
  729. if err != nil {
  730. return telemetry.Error(ctx, span, err, "error creating gar repository")
  731. }
  732. return nil
  733. }
  734. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  735. if err != nil {
  736. return telemetry.Error(ctx, span, err, "error getting project for repository")
  737. }
  738. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  739. // no need to create repository if pushing to ACR or GAR
  740. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  741. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "skipping-create-repo", Value: true})
  742. return nil
  743. }
  744. uri := strings.TrimPrefix(r.URL, "https://")
  745. splits := strings.Split(uri, ".")
  746. accountID := splits[0]
  747. region := splits[3]
  748. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  749. ProjectId: int64(r.ProjectID),
  750. AwsAccountId: accountID,
  751. })
  752. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "uri", Value: uri})
  753. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  754. if err != nil {
  755. return telemetry.Error(ctx, span, err, "error getting capi credentials for repository")
  756. }
  757. aws := &ints.AWSIntegration{
  758. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  759. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  760. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  761. AWSRegion: region,
  762. }
  763. err = r.createECRRepository(aws, name)
  764. if err != nil {
  765. return telemetry.Error(ctx, span, err, "error creating ecr repository")
  766. }
  767. }
  768. // otherwise, no-op
  769. return nil
  770. }
  771. func (r *Registry) createECRRepository(
  772. aws *ints.AWSIntegration,
  773. name string,
  774. ) error {
  775. sess, err := aws.GetSession()
  776. if err != nil {
  777. return err
  778. }
  779. svc := ecr.New(sess)
  780. // determine if repository already exists
  781. _, err = svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
  782. RepositoryNames: []*string{&name},
  783. })
  784. // if the repository was not found, create it
  785. if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ecr.ErrCodeRepositoryNotFoundException {
  786. _, err = svc.CreateRepository(&ecr.CreateRepositoryInput{
  787. RepositoryName: &name,
  788. })
  789. return err
  790. } else if err != nil {
  791. return err
  792. }
  793. return nil
  794. }
  795. func (r *Registry) createGARRepository(
  796. ctx context.Context,
  797. repo repository.Repository,
  798. name string,
  799. ) error {
  800. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  801. r.ProjectID,
  802. r.GCPIntegrationID,
  803. )
  804. if err != nil {
  805. return err
  806. }
  807. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  808. reg: r,
  809. repo: repo,
  810. ctx: ctx,
  811. }), option.WithScopes("roles/artifactregistry.admin"))
  812. if err != nil {
  813. return err
  814. }
  815. defer client.Close()
  816. parsedURL, err := url.Parse("https://" + r.URL)
  817. if err != nil {
  818. return err
  819. }
  820. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  821. _, err = client.GetRepository(context.Background(), &artifactregistrypb.GetRepositoryRequest{
  822. Name: fmt.Sprintf("projects/%s/locations/%s/repositories/%s", gcpInt.GCPProjectID, location, name),
  823. })
  824. if err != nil && strings.Contains(err.Error(), "not found") {
  825. // create a new repository
  826. _, err := client.CreateRepository(context.Background(), &artifactregistrypb.CreateRepositoryRequest{
  827. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  828. RepositoryId: name,
  829. Repository: &artifactregistrypb.Repository{
  830. Format: artifactregistrypb.Repository_DOCKER,
  831. },
  832. })
  833. if err != nil {
  834. return err
  835. }
  836. } else if err != nil {
  837. return err
  838. }
  839. return nil
  840. }
  841. // ListImages lists the images for an image repository
  842. func (r *Registry) ListImages(
  843. ctx context.Context,
  844. repoName string,
  845. repo repository.Repository,
  846. conf *config.Config,
  847. ) ([]*ptypes.Image, error) {
  848. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  849. defer span.End()
  850. telemetry.WithAttributes(span,
  851. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  852. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  853. telemetry.AttributeKV{Key: "registry-url", Value: r.URL},
  854. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  855. telemetry.AttributeKV{Key: "repo-name", Value: repoName},
  856. )
  857. // switch on the auth mechanism to get a token
  858. if r.AWSIntegrationID != 0 {
  859. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  860. r.ProjectID,
  861. r.AWSIntegrationID,
  862. )
  863. if err != nil {
  864. return nil, err
  865. }
  866. return r.listECRImages(aws, repoName, repo)
  867. }
  868. if r.AzureIntegrationID != 0 {
  869. return r.listACRImages(repoName, repo)
  870. }
  871. if r.GCPIntegrationID != 0 {
  872. if strings.Contains(r.URL, "pkg.dev") {
  873. return r.listGARImages(ctx, repoName, repo)
  874. }
  875. return r.listGCRImages(repoName, repo)
  876. }
  877. if r.DOIntegrationID != 0 {
  878. return r.listDOCRImages(repoName, repo, conf.DOConf)
  879. }
  880. if r.BasicIntegrationID != 0 {
  881. return r.listPrivateRegistryImages(repoName, repo)
  882. }
  883. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  884. if err != nil {
  885. return nil, fmt.Errorf("error getting project for repository: %w", err)
  886. }
  887. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  888. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  889. req := connect.NewRequest(&porterv1.ListImagesForRepositoryRequest{
  890. ProjectId: int64(r.ProjectID),
  891. RegistryUri: r.URL,
  892. RepoName: repoName,
  893. })
  894. resp, err := conf.ClusterControlPlaneClient.ListImagesForRepository(ctx, req)
  895. if err != nil {
  896. return nil, telemetry.Error(ctx, span, err, "error calling ccp list images")
  897. }
  898. res := make([]*ptypes.Image, 0)
  899. for _, image := range resp.Msg.Images {
  900. if image.UpdatedAt == nil {
  901. continue
  902. }
  903. lastUpdateTime := image.UpdatedAt.AsTime()
  904. res = append(res, &ptypes.Image{
  905. Digest: image.Digest,
  906. Tag: image.Tag,
  907. Manifest: "",
  908. RepositoryName: image.RepositoryName,
  909. PushedAt: &lastUpdateTime,
  910. })
  911. }
  912. return res, nil
  913. }
  914. uri := strings.TrimPrefix(r.URL, "https://")
  915. splits := strings.Split(uri, ".")
  916. accountID := splits[0]
  917. region := splits[3]
  918. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  919. ProjectId: int64(r.ProjectID),
  920. AwsAccountId: accountID,
  921. })
  922. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  923. if err != nil {
  924. return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
  925. }
  926. aws := &ints.AWSIntegration{
  927. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  928. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  929. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  930. AWSRegion: region,
  931. }
  932. return r.listECRImages(aws, repoName, repo)
  933. }
  934. return nil, fmt.Errorf("error listing images")
  935. }
  936. func (r *Registry) GetECRPaginatedImages(
  937. repoName string,
  938. maxResults int64,
  939. nextToken *string,
  940. aws *ints.AWSIntegration,
  941. ) ([]*ptypes.Image, *string, error) {
  942. if aws == nil {
  943. return nil, nil, errors.New("aws integration is nil")
  944. }
  945. sess, err := aws.GetSession()
  946. if err != nil {
  947. return nil, nil, err
  948. }
  949. svc := ecr.New(sess)
  950. resp, err := svc.ListImages(&ecr.ListImagesInput{
  951. RepositoryName: &repoName,
  952. MaxResults: &maxResults,
  953. NextToken: nextToken,
  954. })
  955. if err != nil {
  956. return nil, nil, err
  957. }
  958. if len(resp.ImageIds) == 0 {
  959. return []*ptypes.Image{}, nil, nil
  960. }
  961. imageIDLen := len(resp.ImageIds)
  962. imageDetails := make([]*ecr.ImageDetail, 0)
  963. imageIDMap := make(map[string]bool)
  964. for _, id := range resp.ImageIds {
  965. if id != nil && id.ImageTag != nil {
  966. imageIDMap[*id.ImageTag] = true
  967. }
  968. }
  969. var wg sync.WaitGroup
  970. var mu sync.Mutex
  971. // AWS API expects the length of imageIDs to be at max 100 at a time
  972. for start := 0; start < imageIDLen; start += 100 {
  973. end := start + 100
  974. if end > imageIDLen {
  975. end = imageIDLen
  976. }
  977. wg.Add(1)
  978. go func(start, end int) {
  979. defer wg.Done()
  980. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  981. RepositoryName: &repoName,
  982. ImageIds: resp.ImageIds[start:end],
  983. })
  984. if err != nil {
  985. return
  986. }
  987. mu.Lock()
  988. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  989. mu.Unlock()
  990. }(start, end)
  991. }
  992. wg.Wait()
  993. res := make([]*ptypes.Image, 0)
  994. imageInfoMap := make(map[string]*ptypes.Image)
  995. for _, img := range imageDetails {
  996. for _, tag := range img.ImageTags {
  997. newImage := &ptypes.Image{
  998. Digest: *img.ImageDigest,
  999. Tag: *tag,
  1000. RepositoryName: repoName,
  1001. PushedAt: img.ImagePushedAt,
  1002. }
  1003. if _, ok := imageIDMap[*tag]; ok {
  1004. if _, ok := imageInfoMap[*tag]; !ok {
  1005. imageInfoMap[*tag] = newImage
  1006. }
  1007. }
  1008. if len(imageInfoMap) == int(maxResults) {
  1009. break
  1010. }
  1011. }
  1012. if len(imageInfoMap) == int(maxResults) {
  1013. break
  1014. }
  1015. }
  1016. for _, v := range imageInfoMap {
  1017. res = append(res, v)
  1018. }
  1019. sort.Slice(res, func(i, j int) bool {
  1020. if res[i].PushedAt == nil || res[j].PushedAt == nil {
  1021. return false
  1022. }
  1023. return res[i].PushedAt.After(*res[j].PushedAt)
  1024. })
  1025. return res, resp.NextToken, nil
  1026. }
  1027. func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1028. sess, err := aws.GetSession()
  1029. if err != nil {
  1030. return nil, err
  1031. }
  1032. svc := ecr.New(sess)
  1033. maxResults := int64(1000)
  1034. var imageIDs []*ecr.ImageIdentifier
  1035. resp, err := svc.ListImages(&ecr.ListImagesInput{
  1036. RepositoryName: &repoName,
  1037. MaxResults: &maxResults,
  1038. })
  1039. if err != nil {
  1040. return nil, err
  1041. }
  1042. if len(resp.ImageIds) == 0 {
  1043. return []*ptypes.Image{}, nil
  1044. }
  1045. imageIDs = append(imageIDs, resp.ImageIds...)
  1046. nextToken := resp.NextToken
  1047. for nextToken != nil {
  1048. resp, err := svc.ListImages(&ecr.ListImagesInput{
  1049. RepositoryName: &repoName,
  1050. MaxResults: &maxResults,
  1051. NextToken: nextToken,
  1052. })
  1053. if err != nil {
  1054. return nil, err
  1055. }
  1056. imageIDs = append(imageIDs, resp.ImageIds...)
  1057. nextToken = resp.NextToken
  1058. }
  1059. imageIDLen := len(imageIDs)
  1060. imageDetails := make([]*ecr.ImageDetail, 0)
  1061. var wg sync.WaitGroup
  1062. var mu sync.Mutex
  1063. // AWS API expects the length of imageIDs to be at max 100 at a time
  1064. for start := 0; start < imageIDLen; start += 100 {
  1065. end := start + 100
  1066. if end > imageIDLen {
  1067. end = imageIDLen
  1068. }
  1069. wg.Add(1)
  1070. go func(start, end int) {
  1071. defer wg.Done()
  1072. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  1073. RepositoryName: &repoName,
  1074. ImageIds: imageIDs[start:end],
  1075. })
  1076. if err != nil {
  1077. return
  1078. }
  1079. mu.Lock()
  1080. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  1081. mu.Unlock()
  1082. }(start, end)
  1083. }
  1084. wg.Wait()
  1085. res := make([]*ptypes.Image, 0)
  1086. imageInfoMap := make(map[string]*ptypes.Image)
  1087. for _, img := range imageDetails {
  1088. for _, tag := range img.ImageTags {
  1089. newImage := &ptypes.Image{
  1090. Digest: *img.ImageDigest,
  1091. Tag: *tag,
  1092. RepositoryName: repoName,
  1093. PushedAt: img.ImagePushedAt,
  1094. }
  1095. if _, ok := imageInfoMap[*tag]; !ok {
  1096. imageInfoMap[*tag] = newImage
  1097. }
  1098. }
  1099. }
  1100. for _, v := range imageInfoMap {
  1101. res = append(res, v)
  1102. }
  1103. return res, nil
  1104. }
  1105. func (r *Registry) listACRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1106. az, err := repo.AzureIntegration().ReadAzureIntegration(
  1107. r.ProjectID,
  1108. r.AzureIntegrationID,
  1109. )
  1110. if err != nil {
  1111. return nil, err
  1112. }
  1113. // use JWT token to request catalog
  1114. client := &http.Client{}
  1115. req, err := http.NewRequest(
  1116. "GET",
  1117. fmt.Sprintf("%s/v2/%s/tags/list", r.URL, repoName),
  1118. nil,
  1119. )
  1120. if err != nil {
  1121. return nil, err
  1122. }
  1123. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  1124. resp, err := client.Do(req)
  1125. if err != nil {
  1126. return nil, err
  1127. }
  1128. gcrResp := gcrImageResp{}
  1129. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1130. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1131. }
  1132. res := make([]*ptypes.Image, 0)
  1133. for _, tag := range gcrResp.Tags {
  1134. res = append(res, &ptypes.Image{
  1135. RepositoryName: strings.TrimPrefix(repoName, "https://"),
  1136. Tag: tag,
  1137. })
  1138. }
  1139. return res, nil
  1140. }
  1141. type gcrImageResp struct {
  1142. Tags []string `json:"tags"`
  1143. }
  1144. func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1145. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1146. r.ProjectID,
  1147. r.GCPIntegrationID,
  1148. )
  1149. if err != nil {
  1150. return nil, err
  1151. }
  1152. // use JWT token to request catalog
  1153. client := &http.Client{}
  1154. parsedURL, err := url.Parse("https://" + r.URL)
  1155. if err != nil {
  1156. return nil, err
  1157. }
  1158. trimmedPath := strings.Trim(parsedURL.Path, "/")
  1159. req, err := http.NewRequest(
  1160. "GET",
  1161. fmt.Sprintf("https://%s/v2/%s/%s/tags/list", parsedURL.Host, trimmedPath, repoName),
  1162. nil,
  1163. )
  1164. if err != nil {
  1165. return nil, err
  1166. }
  1167. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  1168. resp, err := client.Do(req)
  1169. if err != nil {
  1170. return nil, err
  1171. }
  1172. gcrResp := gcrImageResp{}
  1173. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1174. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1175. }
  1176. res := make([]*ptypes.Image, 0)
  1177. for _, tag := range gcrResp.Tags {
  1178. res = append(res, &ptypes.Image{
  1179. RepositoryName: repoName,
  1180. Tag: tag,
  1181. })
  1182. }
  1183. return res, nil
  1184. }
  1185. func (r *Registry) listGARImages(ctx context.Context, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1186. repoImageSlice := strings.Split(repoName, "/")
  1187. if len(repoImageSlice) != 2 {
  1188. return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
  1189. }
  1190. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  1191. r.ProjectID,
  1192. r.GCPIntegrationID,
  1193. )
  1194. if err != nil {
  1195. return nil, err
  1196. }
  1197. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  1198. reg: r,
  1199. repo: repo,
  1200. ctx: ctx,
  1201. }), option.WithScopes("roles/artifactregistry.reader"))
  1202. if err != nil {
  1203. return nil, err
  1204. }
  1205. var res []*ptypes.Image
  1206. parsedURL, err := url.Parse("https://" + r.URL)
  1207. if err != nil {
  1208. return nil, err
  1209. }
  1210. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  1211. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  1212. nextToken := ""
  1213. for {
  1214. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  1215. gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
  1216. if err != nil {
  1217. return nil, err
  1218. }
  1219. for _, image := range resp.DockerImages {
  1220. named, err := reference.ParseNamed(image.Uri)
  1221. if err != nil {
  1222. continue
  1223. }
  1224. paths := strings.Split(reference.Path(named), "/")
  1225. imageName := paths[len(paths)-1]
  1226. if imageName == repoImageSlice[1] {
  1227. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  1228. for _, tag := range image.Tags {
  1229. res = append(res, &ptypes.Image{
  1230. RepositoryName: repoName,
  1231. Tag: tag,
  1232. PushedAt: &uploadTime,
  1233. Digest: strings.Split(image.Uri, "@")[1],
  1234. })
  1235. }
  1236. }
  1237. }
  1238. if resp.NextPageToken == "" {
  1239. break
  1240. }
  1241. nextToken = resp.NextPageToken
  1242. }
  1243. return res, nil
  1244. }
  1245. func (r *Registry) listDOCRImages(
  1246. repoName string,
  1247. repo repository.Repository,
  1248. doAuth *oauth2.Config,
  1249. ) ([]*ptypes.Image, error) {
  1250. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1251. r.ProjectID,
  1252. r.DOIntegrationID,
  1253. )
  1254. if err != nil {
  1255. return nil, err
  1256. }
  1257. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1258. if err != nil {
  1259. return nil, err
  1260. }
  1261. client := godo.NewFromToken(tok)
  1262. urlArr := strings.Split(r.URL, "/")
  1263. if len(urlArr) != 2 {
  1264. return nil, fmt.Errorf("invalid digital ocean registry url")
  1265. }
  1266. name := urlArr[1]
  1267. var tags []*godo.RepositoryTag
  1268. opt := &godo.ListOptions{
  1269. PerPage: 200,
  1270. }
  1271. for {
  1272. nextTags, resp, err := client.Registry.ListRepositoryTags(context.TODO(), name, repoName, opt)
  1273. if err != nil {
  1274. return nil, err
  1275. }
  1276. tags = append(tags, nextTags...)
  1277. if resp.Links == nil || resp.Links.IsLastPage() {
  1278. break
  1279. }
  1280. page, err := resp.Links.CurrentPage()
  1281. if err != nil {
  1282. return nil, err
  1283. }
  1284. opt.Page = page + 1
  1285. }
  1286. res := make([]*ptypes.Image, 0)
  1287. for _, tag := range tags {
  1288. res = append(res, &ptypes.Image{
  1289. RepositoryName: repoName,
  1290. Tag: tag.Tag,
  1291. })
  1292. }
  1293. return res, nil
  1294. }
  1295. func (r *Registry) listPrivateRegistryImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1296. // handle dockerhub different, as it doesn't implement the docker registry http api
  1297. if strings.Contains(r.URL, "docker.io") {
  1298. return r.listDockerHubImages(repoName, repo)
  1299. }
  1300. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1301. r.ProjectID,
  1302. r.BasicIntegrationID,
  1303. )
  1304. if err != nil {
  1305. return nil, err
  1306. }
  1307. // Just use service account key to authenticate, since scopes may not be in place
  1308. // for oauth. This also prevents us from making more requests.
  1309. client := &http.Client{}
  1310. // get the host and scheme to make the request
  1311. parsedURL, err := url.Parse(r.URL)
  1312. req, err := http.NewRequest(
  1313. "GET",
  1314. fmt.Sprintf("%s://%s/v2/%s/tags/list", parsedURL.Scheme, parsedURL.Host, repoName),
  1315. nil,
  1316. )
  1317. if err != nil {
  1318. return nil, err
  1319. }
  1320. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  1321. resp, err := client.Do(req)
  1322. if err != nil {
  1323. return nil, err
  1324. }
  1325. gcrResp := gcrImageResp{}
  1326. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1327. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1328. }
  1329. res := make([]*ptypes.Image, 0)
  1330. for _, tag := range gcrResp.Tags {
  1331. res = append(res, &ptypes.Image{
  1332. RepositoryName: repoName,
  1333. Tag: tag,
  1334. })
  1335. }
  1336. return res, nil
  1337. }
  1338. type dockerHubImageResult struct {
  1339. Name string `json:"name"`
  1340. }
  1341. type dockerHubImageResp struct {
  1342. Results []dockerHubImageResult `json:"results"`
  1343. }
  1344. type dockerHubLoginReq struct {
  1345. Username string `json:"username"`
  1346. Password string `json:"password"`
  1347. }
  1348. type dockerHubLoginResp struct {
  1349. Token string `json:"token"`
  1350. }
  1351. func (r *Registry) listDockerHubImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1352. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1353. r.ProjectID,
  1354. r.BasicIntegrationID,
  1355. )
  1356. if err != nil {
  1357. return nil, err
  1358. }
  1359. client := &http.Client{}
  1360. // first, make a request for the access token
  1361. data, err := json.Marshal(&dockerHubLoginReq{
  1362. Username: string(basic.Username),
  1363. Password: string(basic.Password),
  1364. })
  1365. if err != nil {
  1366. return nil, err
  1367. }
  1368. req, err := http.NewRequest(
  1369. "POST",
  1370. "https://hub.docker.com/v2/users/login",
  1371. strings.NewReader(string(data)),
  1372. )
  1373. if err != nil {
  1374. return nil, err
  1375. }
  1376. req.Header.Add("Content-Type", "application/json")
  1377. resp, err := client.Do(req)
  1378. if err != nil {
  1379. return nil, err
  1380. }
  1381. tokenObj := dockerHubLoginResp{}
  1382. if err := json.NewDecoder(resp.Body).Decode(&tokenObj); err != nil {
  1383. return nil, fmt.Errorf("Could not decode Dockerhub token from response: %v", err)
  1384. }
  1385. req, err = http.NewRequest(
  1386. "GET",
  1387. fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags", strings.Split(r.URL, "docker.io/")[1]),
  1388. nil,
  1389. )
  1390. if err != nil {
  1391. return nil, err
  1392. }
  1393. req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenObj.Token))
  1394. resp, err = client.Do(req)
  1395. if err != nil {
  1396. return nil, err
  1397. }
  1398. imageResp := dockerHubImageResp{}
  1399. if err := json.NewDecoder(resp.Body).Decode(&imageResp); err != nil {
  1400. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1401. }
  1402. res := make([]*ptypes.Image, 0)
  1403. for _, result := range imageResp.Results {
  1404. res = append(res, &ptypes.Image{
  1405. RepositoryName: repoName,
  1406. Tag: result.Name,
  1407. })
  1408. }
  1409. return res, nil
  1410. }
  1411. // GetDockerConfigJSON returns a dockerconfigjson file contents with "auths"
  1412. // populated.
  1413. func (r *Registry) GetDockerConfigJSON(
  1414. repo repository.Repository,
  1415. doAuth *oauth2.Config, // only required if using DOCR
  1416. ) ([]byte, error) {
  1417. var conf *configfile.ConfigFile
  1418. var err error
  1419. // switch on the auth mechanism to get a token
  1420. if r.AWSIntegrationID != 0 {
  1421. conf, err = r.getECRDockerConfigFile(repo)
  1422. }
  1423. if r.GCPIntegrationID != 0 {
  1424. conf, err = r.getGCRDockerConfigFile(repo)
  1425. }
  1426. if r.DOIntegrationID != 0 {
  1427. conf, err = r.getDOCRDockerConfigFile(repo, doAuth)
  1428. }
  1429. if r.BasicIntegrationID != 0 {
  1430. conf, err = r.getPrivateRegistryDockerConfigFile(repo)
  1431. }
  1432. if r.AzureIntegrationID != 0 {
  1433. conf, err = r.getACRDockerConfigFile(repo)
  1434. }
  1435. if err != nil {
  1436. return nil, err
  1437. }
  1438. return json.Marshal(conf)
  1439. }
  1440. func (r *Registry) getECRDockerConfigFile(
  1441. repo repository.Repository,
  1442. ) (*configfile.ConfigFile, error) {
  1443. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  1444. r.ProjectID,
  1445. r.AWSIntegrationID,
  1446. )
  1447. if err != nil {
  1448. return nil, err
  1449. }
  1450. sess, err := aws.GetSession()
  1451. if err != nil {
  1452. return nil, err
  1453. }
  1454. ecrSvc := ecr.New(sess)
  1455. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  1456. if err != nil {
  1457. return nil, err
  1458. }
  1459. token := *output.AuthorizationData[0].AuthorizationToken
  1460. decodedToken, err := base64.StdEncoding.DecodeString(token)
  1461. if err != nil {
  1462. return nil, err
  1463. }
  1464. parts := strings.SplitN(string(decodedToken), ":", 2)
  1465. if len(parts) < 2 {
  1466. return nil, err
  1467. }
  1468. key := r.URL
  1469. if !strings.Contains(key, "http") {
  1470. key = "https://" + key
  1471. }
  1472. return &configfile.ConfigFile{
  1473. AuthConfigs: map[string]types.AuthConfig{
  1474. key: {
  1475. Username: parts[0],
  1476. Password: parts[1],
  1477. Auth: token,
  1478. },
  1479. },
  1480. }, nil
  1481. }
  1482. func (r *Registry) getGCRDockerConfigFile(
  1483. repo repository.Repository,
  1484. ) (*configfile.ConfigFile, error) {
  1485. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1486. r.ProjectID,
  1487. r.GCPIntegrationID,
  1488. )
  1489. if err != nil {
  1490. return nil, err
  1491. }
  1492. key := r.URL
  1493. if !strings.Contains(key, "http") {
  1494. key = "https://" + key
  1495. }
  1496. parsedURL, _ := url.Parse(key)
  1497. return &configfile.ConfigFile{
  1498. AuthConfigs: map[string]types.AuthConfig{
  1499. parsedURL.Host: {
  1500. Username: "_json_key",
  1501. Password: string(gcp.GCPKeyData),
  1502. Auth: generateAuthToken("_json_key", string(gcp.GCPKeyData)),
  1503. },
  1504. },
  1505. }, nil
  1506. }
  1507. func (r *Registry) getDOCRDockerConfigFile(
  1508. repo repository.Repository,
  1509. doAuth *oauth2.Config,
  1510. ) (*configfile.ConfigFile, error) {
  1511. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1512. r.ProjectID,
  1513. r.DOIntegrationID,
  1514. )
  1515. if err != nil {
  1516. return nil, err
  1517. }
  1518. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1519. if err != nil {
  1520. return nil, err
  1521. }
  1522. key := r.URL
  1523. if !strings.Contains(key, "http") {
  1524. key = "https://" + key
  1525. }
  1526. parsedURL, _ := url.Parse(key)
  1527. return &configfile.ConfigFile{
  1528. AuthConfigs: map[string]types.AuthConfig{
  1529. parsedURL.Host: {
  1530. Username: tok,
  1531. Password: tok,
  1532. Auth: generateAuthToken(tok, tok),
  1533. },
  1534. },
  1535. }, nil
  1536. }
  1537. func (r *Registry) getPrivateRegistryDockerConfigFile(
  1538. repo repository.Repository,
  1539. ) (*configfile.ConfigFile, error) {
  1540. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1541. r.ProjectID,
  1542. r.BasicIntegrationID,
  1543. )
  1544. if err != nil {
  1545. return nil, err
  1546. }
  1547. key := r.URL
  1548. if !strings.Contains(key, "http") {
  1549. key = "https://" + key
  1550. }
  1551. parsedURL, _ := url.Parse(key)
  1552. authConfigKey := parsedURL.Host
  1553. if strings.Contains(r.URL, "index.docker.io") {
  1554. authConfigKey = "https://index.docker.io/v1/"
  1555. }
  1556. return &configfile.ConfigFile{
  1557. AuthConfigs: map[string]types.AuthConfig{
  1558. authConfigKey: {
  1559. Username: string(basic.Username),
  1560. Password: string(basic.Password),
  1561. Auth: generateAuthToken(string(basic.Username), string(basic.Password)),
  1562. },
  1563. },
  1564. }, nil
  1565. }
  1566. func (r *Registry) getACRDockerConfigFile(
  1567. repo repository.Repository,
  1568. ) (*configfile.ConfigFile, error) {
  1569. username, pw, err := r.GetACRCredentials(repo)
  1570. if err != nil {
  1571. return nil, err
  1572. }
  1573. key := r.URL
  1574. if !strings.Contains(key, "http") {
  1575. key = "https://" + key
  1576. }
  1577. parsedURL, _ := url.Parse(key)
  1578. return &configfile.ConfigFile{
  1579. AuthConfigs: map[string]types.AuthConfig{
  1580. parsedURL.Host: {
  1581. Username: string(username),
  1582. Password: string(pw),
  1583. Auth: generateAuthToken(string(username), string(pw)),
  1584. },
  1585. },
  1586. }, nil
  1587. }
  1588. func generateAuthToken(username, password string) string {
  1589. return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
  1590. }