registry.go 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979
  1. package registry
  2. import (
  3. "context"
  4. "encoding/base64"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "math/rand"
  9. "net/http"
  10. "net/url"
  11. "sort"
  12. "strings"
  13. "sync"
  14. "time"
  15. "github.com/porter-dev/porter/internal/telemetry"
  16. artifactregistry "cloud.google.com/go/artifactregistry/apiv1beta2"
  17. "connectrpc.com/connect"
  18. "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  19. "github.com/aws/aws-sdk-go/aws/awserr"
  20. "github.com/aws/aws-sdk-go/service/ecr"
  21. porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
  22. "github.com/porter-dev/porter/api/server/shared/config"
  23. "github.com/porter-dev/porter/internal/models"
  24. "github.com/porter-dev/porter/internal/oauth"
  25. "github.com/porter-dev/porter/internal/repository"
  26. "golang.org/x/oauth2"
  27. v1artifactregistry "google.golang.org/api/artifactregistry/v1"
  28. "google.golang.org/api/iterator"
  29. "google.golang.org/api/option"
  30. artifactregistrypb "google.golang.org/genproto/googleapis/devtools/artifactregistry/v1beta2"
  31. ints "github.com/porter-dev/porter/internal/models/integrations"
  32. ptypes "github.com/porter-dev/porter/api/types"
  33. "github.com/digitalocean/godo"
  34. "github.com/docker/cli/cli/config/configfile"
  35. "github.com/docker/cli/cli/config/types"
  36. "github.com/docker/distribution/reference"
  37. "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
  38. "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
  39. )
  40. // Registry wraps the gorm Registry model
  41. type Registry models.Registry
  42. func GetECRRegistryURL(awsIntRepo repository.AWSIntegrationRepository, projectID, awsIntID uint) (string, error) {
  43. awsInt, err := awsIntRepo.ReadAWSIntegration(projectID, awsIntID)
  44. if err != nil {
  45. return "", err
  46. }
  47. sess, err := awsInt.GetSession()
  48. if err != nil {
  49. return "", err
  50. }
  51. ecrSvc := ecr.New(sess)
  52. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  53. if err != nil {
  54. return "", err
  55. }
  56. return *output.AuthorizationData[0].ProxyEndpoint, nil
  57. }
  58. // ListRepositories lists the repositories for a registry
  59. func (r *Registry) ListRepositories(
  60. ctx context.Context,
  61. repo repository.Repository,
  62. conf *config.Config,
  63. ) ([]*ptypes.RegistryRepository, error) {
  64. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  65. defer span.End()
  66. telemetry.WithAttributes(span,
  67. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  68. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  69. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  70. )
  71. // switch on the auth mechanism to get a token
  72. if r.AWSIntegrationID != 0 {
  73. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "aws"})
  74. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  75. r.ProjectID,
  76. r.AWSIntegrationID,
  77. )
  78. if err != nil {
  79. return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
  80. }
  81. repos, err := r.listECRRepositories(aws)
  82. if err != nil {
  83. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  84. }
  85. return repos, nil
  86. }
  87. if r.GCPIntegrationID != 0 {
  88. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
  89. if strings.Contains(r.URL, "pkg.dev") {
  90. return r.listGARRepositories(ctx, repo)
  91. }
  92. repos, err := r.listGCRRepositories(repo)
  93. if err != nil {
  94. return nil, telemetry.Error(ctx, span, err, "error listing gcr repositories")
  95. }
  96. return repos, nil
  97. }
  98. if r.DOIntegrationID != 0 {
  99. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "do"})
  100. repos, err := r.listDOCRRepositories(repo, conf.DOConf)
  101. if err != nil {
  102. return nil, telemetry.Error(ctx, span, err, "error listing docr repositories")
  103. }
  104. return repos, nil
  105. }
  106. if r.AzureIntegrationID != 0 {
  107. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "azure"})
  108. repos, err := r.listACRRepositories(ctx, repo)
  109. if err != nil {
  110. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  111. }
  112. return repos, nil
  113. }
  114. if r.BasicIntegrationID != 0 {
  115. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "basic"})
  116. repos, err := r.listPrivateRegistryRepositories(repo)
  117. if err != nil {
  118. return nil, telemetry.Error(ctx, span, err, "error listing private repositories")
  119. }
  120. return repos, nil
  121. }
  122. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  123. if err != nil {
  124. return nil, telemetry.Error(ctx, span, err, "error getting project for repository")
  125. }
  126. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  127. // TODO: Remove this conditional when AWS list repos is supported in CCP
  128. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "registry-uri", Value: r.URL})
  129. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  130. req := connect.NewRequest(&porterv1.ListRepositoriesForRegistryRequest{
  131. ProjectId: int64(r.ProjectID),
  132. RegistryUri: r.URL,
  133. })
  134. resp, err := conf.ClusterControlPlaneClient.ListRepositoriesForRegistry(ctx, req)
  135. if err != nil {
  136. return nil, telemetry.Error(ctx, span, err, "error listing docker repositories")
  137. }
  138. res := make([]*ptypes.RegistryRepository, 0)
  139. parsedURL, err := url.Parse("https://" + r.URL)
  140. if err != nil {
  141. return nil, telemetry.Error(ctx, span, err, "error parsing url")
  142. }
  143. for _, repo := range resp.Msg.Repositories {
  144. res = append(res, &ptypes.RegistryRepository{
  145. Name: repo.Name,
  146. URI: parsedURL.Host + "/" + repo.Name,
  147. })
  148. }
  149. return res, nil
  150. } else {
  151. uri := strings.TrimPrefix(r.URL, "https://")
  152. splits := strings.Split(uri, ".")
  153. if len(splits) < 4 {
  154. return nil, telemetry.Error(ctx, span, nil, "uri does not have enough splits")
  155. }
  156. accountID := splits[0]
  157. region := splits[3]
  158. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  159. ProjectId: int64(r.ProjectID),
  160. AwsAccountId: accountID,
  161. })
  162. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  163. if err != nil {
  164. return nil, telemetry.Error(ctx, span, err, "error getting capi credentials for registry")
  165. }
  166. aws := &ints.AWSIntegration{
  167. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  168. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  169. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  170. AWSRegion: region,
  171. }
  172. repos, err := r.listECRRepositories(aws)
  173. if err != nil {
  174. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  175. }
  176. return repos, nil
  177. }
  178. }
  179. return nil, telemetry.Error(ctx, span, nil, "error listing repositories")
  180. }
  181. type gcrJWT struct {
  182. AccessToken string `json:"token"`
  183. ExpiresInSec int `json:"expires_in"`
  184. }
  185. type gcrErr struct {
  186. Code string `json:"code"`
  187. Message string `json:"message"`
  188. }
  189. type gcrRepositoryResp struct {
  190. Repositories []string `json:"repositories"`
  191. Errors []gcrErr `json:"errors"`
  192. }
  193. func (r *Registry) GetGCRToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  194. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  195. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  196. r.ProjectID,
  197. r.GCPIntegrationID,
  198. )
  199. if err != nil {
  200. return nil, err
  201. }
  202. // get oauth2 access token
  203. return gcp.GetBearerToken(
  204. ctx,
  205. getTokenCache,
  206. r.setTokenCacheFunc(ctx, repo),
  207. "https://www.googleapis.com/auth/devstorage.read_write",
  208. )
  209. }
  210. func (r *Registry) listGCRRepositories(
  211. repo repository.Repository,
  212. ) ([]*ptypes.RegistryRepository, error) {
  213. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  214. r.ProjectID,
  215. r.GCPIntegrationID,
  216. )
  217. if err != nil {
  218. return nil, err
  219. }
  220. // Just use service account key to authenticate, since scopes may not be in place
  221. // for oauth. This also prevents us from making more requests.
  222. client := &http.Client{}
  223. regURL := r.URL
  224. if !strings.HasPrefix(regURL, "http") {
  225. regURL = fmt.Sprintf("https://%s", regURL)
  226. }
  227. regURLParsed, err := url.Parse(regURL)
  228. regHostname := "gcr.io"
  229. if err == nil {
  230. regHostname = regURLParsed.Host
  231. }
  232. req, err := http.NewRequest(
  233. "GET",
  234. fmt.Sprintf("https://%s/v2/_catalog", regHostname),
  235. nil,
  236. )
  237. if err != nil {
  238. return nil, err
  239. }
  240. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  241. resp, err := client.Do(req)
  242. if err != nil {
  243. return nil, err
  244. }
  245. gcrResp := gcrRepositoryResp{}
  246. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  247. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  248. }
  249. if len(gcrResp.Errors) > 0 {
  250. errMsg := ""
  251. for _, gcrErr := range gcrResp.Errors {
  252. errMsg += fmt.Sprintf(": Code %s, message %s", gcrErr.Code, gcrErr.Message)
  253. }
  254. return nil, fmt.Errorf(errMsg)
  255. }
  256. res := make([]*ptypes.RegistryRepository, 0)
  257. parsedURL, err := url.Parse("https://" + r.URL)
  258. if err != nil {
  259. return nil, err
  260. }
  261. for _, repo := range gcrResp.Repositories {
  262. res = append(res, &ptypes.RegistryRepository{
  263. Name: repo,
  264. URI: parsedURL.Host + "/" + repo,
  265. })
  266. }
  267. return res, nil
  268. }
  269. func (r *Registry) GetGARToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  270. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  271. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  272. r.ProjectID,
  273. r.GCPIntegrationID,
  274. )
  275. if err != nil {
  276. return nil, err
  277. }
  278. // get oauth2 access token
  279. return gcp.GetBearerToken(
  280. ctx,
  281. getTokenCache,
  282. r.setTokenCacheFunc(ctx, repo),
  283. "https://www.googleapis.com/auth/cloud-platform",
  284. )
  285. }
  286. type garTokenSource struct {
  287. // ctx is only passed in here as the oauth2.Token() doesnt support contexts
  288. ctx context.Context
  289. reg *Registry
  290. repo repository.Repository
  291. }
  292. func (source *garTokenSource) Token() (*oauth2.Token, error) {
  293. return source.reg.GetGARToken(source.ctx, source.repo)
  294. }
  295. // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
  296. // where a repository is a single image. This function returns the list of fully qualified names
  297. // of GAR images including their repository names.
  298. func (r *Registry) listGARRepositories(
  299. ctx context.Context,
  300. repo repository.Repository,
  301. ) ([]*ptypes.RegistryRepository, error) {
  302. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  303. r.ProjectID,
  304. r.GCPIntegrationID,
  305. )
  306. if err != nil {
  307. return nil, err
  308. }
  309. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  310. reg: r,
  311. repo: repo,
  312. ctx: ctx,
  313. }), option.WithScopes("roles/artifactregistry.reader"))
  314. if err != nil {
  315. return nil, err
  316. }
  317. var repoNames []string
  318. nextToken := ""
  319. parsedURL, err := url.Parse("https://" + r.URL)
  320. if err != nil {
  321. return nil, err
  322. }
  323. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  324. for {
  325. it := client.ListRepositories(context.Background(), &artifactregistrypb.ListRepositoriesRequest{
  326. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  327. PageSize: 1000,
  328. PageToken: nextToken,
  329. })
  330. for {
  331. resp, err := it.Next()
  332. if err == iterator.Done {
  333. break
  334. } else if err != nil {
  335. return nil, err
  336. }
  337. if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
  338. repoSlice := strings.Split(resp.GetName(), "/")
  339. repoName := repoSlice[len(repoSlice)-1]
  340. repoNames = append(repoNames, repoName)
  341. }
  342. }
  343. if it.PageInfo().Token == "" {
  344. break
  345. }
  346. nextToken = it.PageInfo().Token
  347. }
  348. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  349. reg: r,
  350. repo: repo,
  351. ctx: ctx,
  352. }), option.WithScopes("roles/artifactregistry.reader"))
  353. if err != nil {
  354. return nil, err
  355. }
  356. nextToken = ""
  357. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  358. var (
  359. wg sync.WaitGroup
  360. resMap sync.Map
  361. )
  362. for _, repoName := range repoNames {
  363. wg.Add(1)
  364. go func(repoName string) {
  365. defer wg.Done()
  366. for {
  367. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  368. gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
  369. if err != nil {
  370. // FIXME: we should report this error using a channel
  371. return
  372. }
  373. for _, image := range resp.DockerImages {
  374. named, err := reference.ParseNamed(image.Uri)
  375. if err != nil {
  376. // let us skip this image becaue it has a malformed URI coming from the GCP API
  377. continue
  378. }
  379. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  380. resMap.Store(named.Name(), &ptypes.RegistryRepository{
  381. Name: repoName,
  382. URI: named.Name(),
  383. CreatedAt: uploadTime,
  384. })
  385. }
  386. if resp.NextPageToken == "" {
  387. break
  388. }
  389. nextToken = resp.NextPageToken
  390. }
  391. }(repoName)
  392. }
  393. wg.Wait()
  394. var res []*ptypes.RegistryRepository
  395. resMap.Range(func(_, value any) bool {
  396. res = append(res, value.(*ptypes.RegistryRepository))
  397. return true
  398. })
  399. return res, nil
  400. }
  401. func (r *Registry) listECRRepositories(aws *ints.AWSIntegration) ([]*ptypes.RegistryRepository, error) {
  402. sess, err := aws.GetSession()
  403. if err != nil {
  404. return nil, err
  405. }
  406. svc := ecr.New(sess)
  407. res := make([]*ptypes.RegistryRepository, 0)
  408. input := &ecr.DescribeRepositoriesInput{}
  409. for {
  410. resp, err := svc.DescribeRepositories(input)
  411. if err != nil {
  412. return nil, err
  413. }
  414. for _, repo := range resp.Repositories {
  415. res = append(res, &ptypes.RegistryRepository{
  416. Name: *repo.RepositoryName,
  417. CreatedAt: *repo.CreatedAt,
  418. URI: *repo.RepositoryUri,
  419. })
  420. }
  421. if resp.NextToken == nil {
  422. break
  423. }
  424. input.NextToken = resp.NextToken
  425. }
  426. return res, nil
  427. }
  428. type acrRepositoryResp struct {
  429. Repositories []string `json:"repositories"`
  430. }
  431. func (r *Registry) listACRRepositories(ctx context.Context, repo repository.Repository) ([]*ptypes.RegistryRepository, error) {
  432. ctx, span := telemetry.NewSpan(ctx, "list-acr-repositories")
  433. defer span.End()
  434. telemetry.WithAttributes(span,
  435. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  436. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  437. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  438. )
  439. az, err := repo.AzureIntegration().ReadAzureIntegration(
  440. r.ProjectID,
  441. r.AzureIntegrationID,
  442. )
  443. if err != nil {
  444. return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
  445. }
  446. client := &http.Client{}
  447. acrURL := r.URL
  448. if !strings.Contains(acrURL, "http") {
  449. acrURL = fmt.Sprintf("https://%s", acrURL)
  450. }
  451. req, err := http.NewRequest(
  452. "GET",
  453. fmt.Sprintf("%s/v2/_catalog", acrURL),
  454. nil,
  455. )
  456. if err != nil {
  457. return nil, telemetry.Error(ctx, span, err, "error getting http request")
  458. }
  459. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  460. resp, err := client.Do(req)
  461. if err != nil {
  462. return nil, telemetry.Error(ctx, span, err, "error making http call")
  463. }
  464. acrResp := acrRepositoryResp{}
  465. if err := json.NewDecoder(resp.Body).Decode(&acrResp); err != nil {
  466. return nil, telemetry.Error(ctx, span, err, "could not read Azure registry repository response")
  467. }
  468. res := make([]*ptypes.RegistryRepository, 0)
  469. for _, repo := range acrResp.Repositories {
  470. res = append(res, &ptypes.RegistryRepository{
  471. Name: repo,
  472. URI: strings.TrimPrefix(r.URL, "https://") + "/" + repo,
  473. })
  474. }
  475. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "repo-count", Value: len(acrResp.Repositories)})
  476. return res, nil
  477. }
  478. // Returns the username/password pair for the registry
  479. func (r *Registry) GetACRCredentials(repo repository.Repository) (string, string, error) {
  480. az, err := repo.AzureIntegration().ReadAzureIntegration(
  481. r.ProjectID,
  482. r.AzureIntegrationID,
  483. )
  484. if err != nil {
  485. return "", "", err
  486. }
  487. // if the passwords and name aren't set, generate them
  488. // unique token name to prevent token expiry during close subsequent builds.
  489. // Token expires in 14 days, limited at 100 tokens/reg.
  490. if az.ACRTokenName == "" || len(az.ACRPassword1) == 0 {
  491. az.ACRTokenName = fmt.Sprintf("porter-acr-token-%s-%d", az.ACRName, rand.Intn(100)) // nolint:gosec
  492. // create an acr repo token
  493. cred, err := azidentity.NewClientSecretCredential(az.AzureTenantID, az.AzureClientID, string(az.ServicePrincipalSecret), nil)
  494. if err != nil {
  495. return "", "", err
  496. }
  497. scopeMapsClient, err := armcontainerregistry.NewScopeMapsClient(az.AzureSubscriptionID, cred, nil)
  498. if err != nil {
  499. return "", "", err
  500. }
  501. smRes, err := scopeMapsClient.Get(
  502. context.Background(),
  503. az.ACRResourceGroupName,
  504. az.ACRName,
  505. "_repositories_admin",
  506. nil,
  507. )
  508. if err != nil {
  509. return "", "", err
  510. }
  511. tokensClient, err := armcontainerregistry.NewTokensClient(az.AzureSubscriptionID, cred, nil)
  512. if err != nil {
  513. return "", "", err
  514. }
  515. pollerResp, err := tokensClient.BeginCreate(
  516. context.Background(),
  517. az.ACRResourceGroupName,
  518. az.ACRName,
  519. az.ACRTokenName,
  520. armcontainerregistry.Token{
  521. Properties: &armcontainerregistry.TokenProperties{
  522. ScopeMapID: smRes.ID,
  523. Status: to.Ptr(armcontainerregistry.TokenStatusEnabled),
  524. },
  525. },
  526. nil,
  527. )
  528. if err != nil {
  529. return "", "", err
  530. }
  531. tokResp, err := pollerResp.PollUntilDone(context.Background(), 2*time.Second)
  532. if err != nil {
  533. return "", "", err
  534. }
  535. registriesClient, err := armcontainerregistry.NewRegistriesClient(az.AzureSubscriptionID, cred, nil)
  536. if err != nil {
  537. return "", "", err
  538. }
  539. poller, err := registriesClient.BeginGenerateCredentials(
  540. context.Background(),
  541. az.ACRResourceGroupName,
  542. az.ACRName,
  543. armcontainerregistry.GenerateCredentialsParameters{
  544. TokenID: tokResp.ID,
  545. },
  546. &armcontainerregistry.RegistriesClientBeginGenerateCredentialsOptions{ResumeToken: ""})
  547. if err != nil {
  548. return "", "", err
  549. }
  550. genCredentialsResp, err := poller.PollUntilDone(context.Background(), 2*time.Second)
  551. if err != nil {
  552. return "", "", err
  553. }
  554. for i, tokPassword := range genCredentialsResp.Passwords {
  555. if i == 0 {
  556. az.ACRPassword1 = []byte(*tokPassword.Value)
  557. } else if i == 1 {
  558. az.ACRPassword2 = []byte(*tokPassword.Value)
  559. }
  560. }
  561. // update the az integration
  562. az, err = repo.AzureIntegration().OverwriteAzureIntegration(
  563. az,
  564. )
  565. if err != nil {
  566. return "", "", err
  567. }
  568. }
  569. return az.ACRTokenName, string(az.ACRPassword1), nil
  570. }
  571. func (r *Registry) listDOCRRepositories(
  572. repo repository.Repository,
  573. doAuth *oauth2.Config,
  574. ) ([]*ptypes.RegistryRepository, error) {
  575. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  576. r.ProjectID,
  577. r.DOIntegrationID,
  578. )
  579. if err != nil {
  580. return nil, err
  581. }
  582. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  583. if err != nil {
  584. return nil, err
  585. }
  586. client := godo.NewFromToken(tok)
  587. urlArr := strings.Split(r.URL, "/")
  588. if len(urlArr) != 2 {
  589. return nil, fmt.Errorf("invalid digital ocean registry url")
  590. }
  591. name := urlArr[1]
  592. repos, _, err := client.Registry.ListRepositories(context.TODO(), name, &godo.ListOptions{})
  593. if err != nil {
  594. return nil, err
  595. }
  596. res := make([]*ptypes.RegistryRepository, 0)
  597. for _, repo := range repos {
  598. res = append(res, &ptypes.RegistryRepository{
  599. Name: repo.Name,
  600. URI: r.URL + "/" + repo.Name,
  601. })
  602. }
  603. return res, nil
  604. }
  605. func (r *Registry) listPrivateRegistryRepositories(
  606. repo repository.Repository,
  607. ) ([]*ptypes.RegistryRepository, error) {
  608. // handle dockerhub different, as it doesn't implement the docker registry http api
  609. if strings.Contains(r.URL, "docker.io") {
  610. // in this case, we just return the single dockerhub repository that's linked
  611. res := make([]*ptypes.RegistryRepository, 0)
  612. res = append(res, &ptypes.RegistryRepository{
  613. Name: strings.Split(r.URL, "docker.io/")[1],
  614. URI: r.URL,
  615. })
  616. return res, nil
  617. }
  618. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  619. r.ProjectID,
  620. r.BasicIntegrationID,
  621. )
  622. if err != nil {
  623. return nil, err
  624. }
  625. // Just use service account key to authenticate, since scopes may not be in place
  626. // for oauth. This also prevents us from making more requests.
  627. client := &http.Client{}
  628. // get the host and scheme to make the request
  629. parsedURL, err := url.Parse(r.URL)
  630. req, err := http.NewRequest(
  631. "GET",
  632. fmt.Sprintf("%s://%s/v2/_catalog", parsedURL.Scheme, parsedURL.Host),
  633. nil,
  634. )
  635. if err != nil {
  636. return nil, err
  637. }
  638. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  639. resp, err := client.Do(req)
  640. if err != nil {
  641. return nil, err
  642. }
  643. // if the status code is 404, fallback to the Docker Hub implementation
  644. if resp.StatusCode == 404 {
  645. req, err := http.NewRequest(
  646. "GET",
  647. fmt.Sprintf("%s/", r.URL),
  648. nil,
  649. )
  650. if err != nil {
  651. return nil, err
  652. }
  653. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  654. resp, err = client.Do(req)
  655. if err != nil {
  656. return nil, err
  657. }
  658. }
  659. gcrResp := gcrRepositoryResp{}
  660. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  661. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  662. }
  663. res := make([]*ptypes.RegistryRepository, 0)
  664. if err != nil {
  665. return nil, err
  666. }
  667. for _, repo := range gcrResp.Repositories {
  668. res = append(res, &ptypes.RegistryRepository{
  669. Name: repo,
  670. URI: parsedURL.Host + "/" + repo,
  671. })
  672. }
  673. return res, nil
  674. }
  675. func (r *Registry) getTokenCacheFunc(
  676. ctx context.Context,
  677. repo repository.Repository,
  678. ) ints.GetTokenCacheFunc {
  679. return func(ctx context.Context) (tok *ints.TokenCache, err error) {
  680. reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
  681. if err != nil {
  682. return nil, err
  683. }
  684. return &reg.TokenCache.TokenCache, nil
  685. }
  686. }
  687. func (r *Registry) setTokenCacheFunc(
  688. ctx context.Context,
  689. repo repository.Repository,
  690. ) ints.SetTokenCacheFunc {
  691. return func(ctx context.Context, token string, expiry time.Time) error {
  692. _, err := repo.Registry().UpdateRegistryTokenCache(
  693. &ints.RegTokenCache{
  694. TokenCache: ints.TokenCache{
  695. Token: []byte(token),
  696. Expiry: expiry,
  697. },
  698. RegistryID: r.ID,
  699. },
  700. )
  701. return err
  702. }
  703. }
  704. // CreateRepository creates a repository for a registry, if needed
  705. // (currently only required for ECR)
  706. func (r *Registry) CreateRepository(
  707. ctx context.Context,
  708. conf *config.Config,
  709. name string,
  710. ) error {
  711. ctx, span := telemetry.NewSpan(ctx, "create-repository")
  712. defer span.End()
  713. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "registry-uri", Value: r.URL})
  714. // if aws, create repository
  715. if r.AWSIntegrationID != 0 {
  716. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "aws-integration-id", Value: r.AWSIntegrationID})
  717. aws, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
  718. r.ProjectID,
  719. r.AWSIntegrationID,
  720. )
  721. if err != nil {
  722. return telemetry.Error(ctx, span, err, "error reading aws integration")
  723. }
  724. err = r.createECRRepository(aws, name)
  725. if err != nil {
  726. return telemetry.Error(ctx, span, err, "error creating ecr repository")
  727. }
  728. return nil
  729. } else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
  730. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "gcp-integration-id", Value: r.GCPIntegrationID})
  731. err := r.createGARRepository(ctx, conf.Repo, name)
  732. if err != nil {
  733. return telemetry.Error(ctx, span, err, "error creating gar repository")
  734. }
  735. return nil
  736. }
  737. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  738. if err != nil {
  739. return telemetry.Error(ctx, span, err, "error getting project for repository")
  740. }
  741. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  742. // no need to create repository if pushing to ACR or GAR
  743. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  744. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "skipping-create-repo", Value: true})
  745. return nil
  746. }
  747. uri := strings.TrimPrefix(r.URL, "https://")
  748. splits := strings.Split(uri, ".")
  749. accountID := splits[0]
  750. region := splits[3]
  751. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  752. ProjectId: int64(r.ProjectID),
  753. AwsAccountId: accountID,
  754. })
  755. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "uri", Value: uri})
  756. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  757. if err != nil {
  758. return telemetry.Error(ctx, span, err, "error getting capi credentials for repository")
  759. }
  760. aws := &ints.AWSIntegration{
  761. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  762. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  763. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  764. AWSRegion: region,
  765. }
  766. err = r.createECRRepository(aws, name)
  767. if err != nil {
  768. return telemetry.Error(ctx, span, err, "error creating ecr repository")
  769. }
  770. }
  771. // otherwise, no-op
  772. return nil
  773. }
  774. func (r *Registry) createECRRepository(
  775. aws *ints.AWSIntegration,
  776. name string,
  777. ) error {
  778. sess, err := aws.GetSession()
  779. if err != nil {
  780. return err
  781. }
  782. svc := ecr.New(sess)
  783. // determine if repository already exists
  784. _, err = svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
  785. RepositoryNames: []*string{&name},
  786. })
  787. // if the repository was not found, create it
  788. if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ecr.ErrCodeRepositoryNotFoundException {
  789. _, err = svc.CreateRepository(&ecr.CreateRepositoryInput{
  790. RepositoryName: &name,
  791. })
  792. return err
  793. } else if err != nil {
  794. return err
  795. }
  796. return nil
  797. }
  798. func (r *Registry) createGARRepository(
  799. ctx context.Context,
  800. repo repository.Repository,
  801. name string,
  802. ) error {
  803. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  804. r.ProjectID,
  805. r.GCPIntegrationID,
  806. )
  807. if err != nil {
  808. return err
  809. }
  810. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  811. reg: r,
  812. repo: repo,
  813. ctx: ctx,
  814. }), option.WithScopes("roles/artifactregistry.admin"))
  815. if err != nil {
  816. return err
  817. }
  818. defer client.Close()
  819. parsedURL, err := url.Parse("https://" + r.URL)
  820. if err != nil {
  821. return err
  822. }
  823. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  824. _, err = client.GetRepository(context.Background(), &artifactregistrypb.GetRepositoryRequest{
  825. Name: fmt.Sprintf("projects/%s/locations/%s/repositories/%s", gcpInt.GCPProjectID, location, name),
  826. })
  827. if err != nil && strings.Contains(err.Error(), "not found") {
  828. // create a new repository
  829. _, err := client.CreateRepository(context.Background(), &artifactregistrypb.CreateRepositoryRequest{
  830. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  831. RepositoryId: name,
  832. Repository: &artifactregistrypb.Repository{
  833. Format: artifactregistrypb.Repository_DOCKER,
  834. },
  835. })
  836. if err != nil {
  837. return err
  838. }
  839. } else if err != nil {
  840. return err
  841. }
  842. return nil
  843. }
  844. // ListImages lists the images for an image repository
  845. func (r *Registry) ListImages(
  846. ctx context.Context,
  847. repoName string,
  848. repo repository.Repository,
  849. conf *config.Config,
  850. ) ([]*ptypes.Image, error) {
  851. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  852. defer span.End()
  853. telemetry.WithAttributes(span,
  854. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  855. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  856. telemetry.AttributeKV{Key: "registry-url", Value: r.URL},
  857. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  858. telemetry.AttributeKV{Key: "repo-name", Value: repoName},
  859. )
  860. // switch on the auth mechanism to get a token
  861. if r.AWSIntegrationID != 0 {
  862. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  863. r.ProjectID,
  864. r.AWSIntegrationID,
  865. )
  866. if err != nil {
  867. return nil, err
  868. }
  869. return r.listECRImages(aws, repoName, repo)
  870. }
  871. if r.AzureIntegrationID != 0 {
  872. return r.listACRImages(repoName, repo)
  873. }
  874. if r.GCPIntegrationID != 0 {
  875. if strings.Contains(r.URL, "pkg.dev") {
  876. return r.listGARImages(ctx, repoName, repo)
  877. }
  878. return r.listGCRImages(repoName, repo)
  879. }
  880. if r.DOIntegrationID != 0 {
  881. return r.listDOCRImages(repoName, repo, conf.DOConf)
  882. }
  883. if r.BasicIntegrationID != 0 {
  884. return r.listPrivateRegistryImages(repoName, repo)
  885. }
  886. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  887. if err != nil {
  888. return nil, fmt.Errorf("error getting project for repository: %w", err)
  889. }
  890. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  891. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  892. req := connect.NewRequest(&porterv1.ListImagesForRepositoryRequest{
  893. ProjectId: int64(r.ProjectID),
  894. RegistryUri: r.URL,
  895. RepoName: repoName,
  896. })
  897. resp, err := conf.ClusterControlPlaneClient.ListImagesForRepository(ctx, req)
  898. if err != nil {
  899. return nil, telemetry.Error(ctx, span, err, "error calling ccp list images")
  900. }
  901. res := make([]*ptypes.Image, 0)
  902. for _, image := range resp.Msg.Images {
  903. if image.UpdatedAt == nil {
  904. continue
  905. }
  906. lastUpdateTime := image.UpdatedAt.AsTime()
  907. res = append(res, &ptypes.Image{
  908. Digest: image.Digest,
  909. Tag: image.Tag,
  910. Manifest: "",
  911. RepositoryName: image.RepositoryName,
  912. PushedAt: &lastUpdateTime,
  913. })
  914. }
  915. return res, nil
  916. }
  917. uri := strings.TrimPrefix(r.URL, "https://")
  918. splits := strings.Split(uri, ".")
  919. accountID := splits[0]
  920. region := splits[3]
  921. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  922. ProjectId: int64(r.ProjectID),
  923. AwsAccountId: accountID,
  924. })
  925. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  926. if err != nil {
  927. return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
  928. }
  929. aws := &ints.AWSIntegration{
  930. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  931. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  932. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  933. AWSRegion: region,
  934. }
  935. return r.listECRImages(aws, repoName, repo)
  936. }
  937. return nil, fmt.Errorf("error listing images")
  938. }
  939. func (r *Registry) GetECRPaginatedImages(
  940. repoName string,
  941. maxResults int64,
  942. nextToken *string,
  943. aws *ints.AWSIntegration,
  944. ) ([]*ptypes.Image, *string, error) {
  945. if aws == nil {
  946. return nil, nil, errors.New("aws integration is nil")
  947. }
  948. sess, err := aws.GetSession()
  949. if err != nil {
  950. return nil, nil, err
  951. }
  952. svc := ecr.New(sess)
  953. resp, err := svc.ListImages(&ecr.ListImagesInput{
  954. RepositoryName: &repoName,
  955. MaxResults: &maxResults,
  956. NextToken: nextToken,
  957. })
  958. if err != nil {
  959. return nil, nil, err
  960. }
  961. if len(resp.ImageIds) == 0 {
  962. return []*ptypes.Image{}, nil, nil
  963. }
  964. imageIDLen := len(resp.ImageIds)
  965. imageDetails := make([]*ecr.ImageDetail, 0)
  966. imageIDMap := make(map[string]bool)
  967. for _, id := range resp.ImageIds {
  968. if id != nil && id.ImageTag != nil {
  969. imageIDMap[*id.ImageTag] = true
  970. }
  971. }
  972. var wg sync.WaitGroup
  973. var mu sync.Mutex
  974. // AWS API expects the length of imageIDs to be at max 100 at a time
  975. for start := 0; start < imageIDLen; start += 100 {
  976. end := start + 100
  977. if end > imageIDLen {
  978. end = imageIDLen
  979. }
  980. wg.Add(1)
  981. go func(start, end int) {
  982. defer wg.Done()
  983. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  984. RepositoryName: &repoName,
  985. ImageIds: resp.ImageIds[start:end],
  986. })
  987. if err != nil {
  988. return
  989. }
  990. mu.Lock()
  991. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  992. mu.Unlock()
  993. }(start, end)
  994. }
  995. wg.Wait()
  996. res := make([]*ptypes.Image, 0)
  997. imageInfoMap := make(map[string]*ptypes.Image)
  998. for _, img := range imageDetails {
  999. for _, tag := range img.ImageTags {
  1000. newImage := &ptypes.Image{
  1001. Digest: *img.ImageDigest,
  1002. Tag: *tag,
  1003. RepositoryName: repoName,
  1004. PushedAt: img.ImagePushedAt,
  1005. }
  1006. if _, ok := imageIDMap[*tag]; ok {
  1007. if _, ok := imageInfoMap[*tag]; !ok {
  1008. imageInfoMap[*tag] = newImage
  1009. }
  1010. }
  1011. if len(imageInfoMap) == int(maxResults) {
  1012. break
  1013. }
  1014. }
  1015. if len(imageInfoMap) == int(maxResults) {
  1016. break
  1017. }
  1018. }
  1019. for _, v := range imageInfoMap {
  1020. res = append(res, v)
  1021. }
  1022. sort.Slice(res, func(i, j int) bool {
  1023. if res[i].PushedAt == nil || res[j].PushedAt == nil {
  1024. return false
  1025. }
  1026. return res[i].PushedAt.After(*res[j].PushedAt)
  1027. })
  1028. return res, resp.NextToken, nil
  1029. }
  1030. func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1031. sess, err := aws.GetSession()
  1032. if err != nil {
  1033. return nil, err
  1034. }
  1035. svc := ecr.New(sess)
  1036. maxResults := int64(1000)
  1037. var imageIDs []*ecr.ImageIdentifier
  1038. resp, err := svc.ListImages(&ecr.ListImagesInput{
  1039. RepositoryName: &repoName,
  1040. MaxResults: &maxResults,
  1041. })
  1042. if err != nil {
  1043. return nil, err
  1044. }
  1045. if len(resp.ImageIds) == 0 {
  1046. return []*ptypes.Image{}, nil
  1047. }
  1048. imageIDs = append(imageIDs, resp.ImageIds...)
  1049. nextToken := resp.NextToken
  1050. for nextToken != nil {
  1051. resp, err := svc.ListImages(&ecr.ListImagesInput{
  1052. RepositoryName: &repoName,
  1053. MaxResults: &maxResults,
  1054. NextToken: nextToken,
  1055. })
  1056. if err != nil {
  1057. return nil, err
  1058. }
  1059. imageIDs = append(imageIDs, resp.ImageIds...)
  1060. nextToken = resp.NextToken
  1061. }
  1062. imageIDLen := len(imageIDs)
  1063. imageDetails := make([]*ecr.ImageDetail, 0)
  1064. var wg sync.WaitGroup
  1065. var mu sync.Mutex
  1066. // AWS API expects the length of imageIDs to be at max 100 at a time
  1067. for start := 0; start < imageIDLen; start += 100 {
  1068. end := start + 100
  1069. if end > imageIDLen {
  1070. end = imageIDLen
  1071. }
  1072. wg.Add(1)
  1073. go func(start, end int) {
  1074. defer wg.Done()
  1075. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  1076. RepositoryName: &repoName,
  1077. ImageIds: imageIDs[start:end],
  1078. })
  1079. if err != nil {
  1080. return
  1081. }
  1082. mu.Lock()
  1083. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  1084. mu.Unlock()
  1085. }(start, end)
  1086. }
  1087. wg.Wait()
  1088. res := make([]*ptypes.Image, 0)
  1089. imageInfoMap := make(map[string]*ptypes.Image)
  1090. for _, img := range imageDetails {
  1091. for _, tag := range img.ImageTags {
  1092. newImage := &ptypes.Image{
  1093. Digest: *img.ImageDigest,
  1094. Tag: *tag,
  1095. RepositoryName: repoName,
  1096. PushedAt: img.ImagePushedAt,
  1097. }
  1098. if _, ok := imageInfoMap[*tag]; !ok {
  1099. imageInfoMap[*tag] = newImage
  1100. }
  1101. }
  1102. }
  1103. for _, v := range imageInfoMap {
  1104. res = append(res, v)
  1105. }
  1106. return res, nil
  1107. }
  1108. func (r *Registry) listACRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1109. az, err := repo.AzureIntegration().ReadAzureIntegration(
  1110. r.ProjectID,
  1111. r.AzureIntegrationID,
  1112. )
  1113. if err != nil {
  1114. return nil, err
  1115. }
  1116. // use JWT token to request catalog
  1117. client := &http.Client{}
  1118. req, err := http.NewRequest(
  1119. "GET",
  1120. fmt.Sprintf("%s/v2/%s/tags/list", r.URL, repoName),
  1121. nil,
  1122. )
  1123. if err != nil {
  1124. return nil, err
  1125. }
  1126. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  1127. resp, err := client.Do(req)
  1128. if err != nil {
  1129. return nil, err
  1130. }
  1131. gcrResp := gcrImageResp{}
  1132. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1133. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1134. }
  1135. res := make([]*ptypes.Image, 0)
  1136. for _, tag := range gcrResp.Tags {
  1137. res = append(res, &ptypes.Image{
  1138. RepositoryName: strings.TrimPrefix(repoName, "https://"),
  1139. Tag: tag,
  1140. })
  1141. }
  1142. return res, nil
  1143. }
  1144. type gcrImageResp struct {
  1145. Tags []string `json:"tags"`
  1146. }
  1147. func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1148. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1149. r.ProjectID,
  1150. r.GCPIntegrationID,
  1151. )
  1152. if err != nil {
  1153. return nil, err
  1154. }
  1155. // use JWT token to request catalog
  1156. client := &http.Client{}
  1157. parsedURL, err := url.Parse("https://" + r.URL)
  1158. if err != nil {
  1159. return nil, err
  1160. }
  1161. trimmedPath := strings.Trim(parsedURL.Path, "/")
  1162. req, err := http.NewRequest(
  1163. "GET",
  1164. fmt.Sprintf("https://%s/v2/%s/%s/tags/list", parsedURL.Host, trimmedPath, repoName),
  1165. nil,
  1166. )
  1167. if err != nil {
  1168. return nil, err
  1169. }
  1170. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  1171. resp, err := client.Do(req)
  1172. if err != nil {
  1173. return nil, err
  1174. }
  1175. gcrResp := gcrImageResp{}
  1176. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1177. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1178. }
  1179. res := make([]*ptypes.Image, 0)
  1180. for _, tag := range gcrResp.Tags {
  1181. res = append(res, &ptypes.Image{
  1182. RepositoryName: repoName,
  1183. Tag: tag,
  1184. })
  1185. }
  1186. return res, nil
  1187. }
  1188. func (r *Registry) listGARImages(ctx context.Context, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1189. repoImageSlice := strings.Split(repoName, "/")
  1190. if len(repoImageSlice) != 2 {
  1191. return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
  1192. }
  1193. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  1194. r.ProjectID,
  1195. r.GCPIntegrationID,
  1196. )
  1197. if err != nil {
  1198. return nil, err
  1199. }
  1200. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  1201. reg: r,
  1202. repo: repo,
  1203. ctx: ctx,
  1204. }), option.WithScopes("roles/artifactregistry.reader"))
  1205. if err != nil {
  1206. return nil, err
  1207. }
  1208. var res []*ptypes.Image
  1209. parsedURL, err := url.Parse("https://" + r.URL)
  1210. if err != nil {
  1211. return nil, err
  1212. }
  1213. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  1214. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  1215. nextToken := ""
  1216. for {
  1217. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  1218. gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
  1219. if err != nil {
  1220. return nil, err
  1221. }
  1222. for _, image := range resp.DockerImages {
  1223. named, err := reference.ParseNamed(image.Uri)
  1224. if err != nil {
  1225. continue
  1226. }
  1227. paths := strings.Split(reference.Path(named), "/")
  1228. imageName := paths[len(paths)-1]
  1229. if imageName == repoImageSlice[1] {
  1230. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  1231. for _, tag := range image.Tags {
  1232. res = append(res, &ptypes.Image{
  1233. RepositoryName: repoName,
  1234. Tag: tag,
  1235. PushedAt: &uploadTime,
  1236. Digest: strings.Split(image.Uri, "@")[1],
  1237. })
  1238. }
  1239. }
  1240. }
  1241. if resp.NextPageToken == "" {
  1242. break
  1243. }
  1244. nextToken = resp.NextPageToken
  1245. }
  1246. return res, nil
  1247. }
  1248. func (r *Registry) listDOCRImages(
  1249. repoName string,
  1250. repo repository.Repository,
  1251. doAuth *oauth2.Config,
  1252. ) ([]*ptypes.Image, error) {
  1253. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1254. r.ProjectID,
  1255. r.DOIntegrationID,
  1256. )
  1257. if err != nil {
  1258. return nil, err
  1259. }
  1260. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1261. if err != nil {
  1262. return nil, err
  1263. }
  1264. client := godo.NewFromToken(tok)
  1265. urlArr := strings.Split(r.URL, "/")
  1266. if len(urlArr) != 2 {
  1267. return nil, fmt.Errorf("invalid digital ocean registry url")
  1268. }
  1269. name := urlArr[1]
  1270. var tags []*godo.RepositoryTag
  1271. opt := &godo.ListOptions{
  1272. PerPage: 200,
  1273. }
  1274. for {
  1275. nextTags, resp, err := client.Registry.ListRepositoryTags(context.TODO(), name, repoName, opt)
  1276. if err != nil {
  1277. return nil, err
  1278. }
  1279. tags = append(tags, nextTags...)
  1280. if resp.Links == nil || resp.Links.IsLastPage() {
  1281. break
  1282. }
  1283. page, err := resp.Links.CurrentPage()
  1284. if err != nil {
  1285. return nil, err
  1286. }
  1287. opt.Page = page + 1
  1288. }
  1289. res := make([]*ptypes.Image, 0)
  1290. for _, tag := range tags {
  1291. res = append(res, &ptypes.Image{
  1292. RepositoryName: repoName,
  1293. Tag: tag.Tag,
  1294. })
  1295. }
  1296. return res, nil
  1297. }
  1298. func (r *Registry) listPrivateRegistryImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1299. // handle dockerhub different, as it doesn't implement the docker registry http api
  1300. if strings.Contains(r.URL, "docker.io") {
  1301. return r.listDockerHubImages(repoName, repo)
  1302. }
  1303. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1304. r.ProjectID,
  1305. r.BasicIntegrationID,
  1306. )
  1307. if err != nil {
  1308. return nil, err
  1309. }
  1310. // Just use service account key to authenticate, since scopes may not be in place
  1311. // for oauth. This also prevents us from making more requests.
  1312. client := &http.Client{}
  1313. // get the host and scheme to make the request
  1314. parsedURL, err := url.Parse(r.URL)
  1315. req, err := http.NewRequest(
  1316. "GET",
  1317. fmt.Sprintf("%s://%s/v2/%s/tags/list", parsedURL.Scheme, parsedURL.Host, repoName),
  1318. nil,
  1319. )
  1320. if err != nil {
  1321. return nil, err
  1322. }
  1323. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  1324. resp, err := client.Do(req)
  1325. if err != nil {
  1326. return nil, err
  1327. }
  1328. gcrResp := gcrImageResp{}
  1329. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1330. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1331. }
  1332. res := make([]*ptypes.Image, 0)
  1333. for _, tag := range gcrResp.Tags {
  1334. res = append(res, &ptypes.Image{
  1335. RepositoryName: repoName,
  1336. Tag: tag,
  1337. })
  1338. }
  1339. return res, nil
  1340. }
  1341. type dockerHubImageResult struct {
  1342. Name string `json:"name"`
  1343. }
  1344. type dockerHubImageResp struct {
  1345. Results []dockerHubImageResult `json:"results"`
  1346. }
  1347. type dockerHubLoginReq struct {
  1348. Username string `json:"username"`
  1349. Password string `json:"password"`
  1350. }
  1351. type dockerHubLoginResp struct {
  1352. Token string `json:"token"`
  1353. }
  1354. func (r *Registry) listDockerHubImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1355. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1356. r.ProjectID,
  1357. r.BasicIntegrationID,
  1358. )
  1359. if err != nil {
  1360. return nil, err
  1361. }
  1362. client := &http.Client{}
  1363. // first, make a request for the access token
  1364. data, err := json.Marshal(&dockerHubLoginReq{
  1365. Username: string(basic.Username),
  1366. Password: string(basic.Password),
  1367. })
  1368. if err != nil {
  1369. return nil, err
  1370. }
  1371. req, err := http.NewRequest(
  1372. "POST",
  1373. "https://hub.docker.com/v2/users/login",
  1374. strings.NewReader(string(data)),
  1375. )
  1376. if err != nil {
  1377. return nil, err
  1378. }
  1379. req.Header.Add("Content-Type", "application/json")
  1380. resp, err := client.Do(req)
  1381. if err != nil {
  1382. return nil, err
  1383. }
  1384. tokenObj := dockerHubLoginResp{}
  1385. if err := json.NewDecoder(resp.Body).Decode(&tokenObj); err != nil {
  1386. return nil, fmt.Errorf("Could not decode Dockerhub token from response: %v", err)
  1387. }
  1388. req, err = http.NewRequest(
  1389. "GET",
  1390. fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags", strings.Split(r.URL, "docker.io/")[1]),
  1391. nil,
  1392. )
  1393. if err != nil {
  1394. return nil, err
  1395. }
  1396. req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenObj.Token))
  1397. resp, err = client.Do(req)
  1398. if err != nil {
  1399. return nil, err
  1400. }
  1401. imageResp := dockerHubImageResp{}
  1402. if err := json.NewDecoder(resp.Body).Decode(&imageResp); err != nil {
  1403. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1404. }
  1405. res := make([]*ptypes.Image, 0)
  1406. for _, result := range imageResp.Results {
  1407. res = append(res, &ptypes.Image{
  1408. RepositoryName: repoName,
  1409. Tag: result.Name,
  1410. })
  1411. }
  1412. return res, nil
  1413. }
  1414. // GetDockerConfigJSON returns a dockerconfigjson file contents with "auths"
  1415. // populated.
  1416. func (r *Registry) GetDockerConfigJSON(
  1417. repo repository.Repository,
  1418. doAuth *oauth2.Config, // only required if using DOCR
  1419. ) ([]byte, error) {
  1420. var conf *configfile.ConfigFile
  1421. var err error
  1422. // switch on the auth mechanism to get a token
  1423. if r.AWSIntegrationID != 0 {
  1424. conf, err = r.getECRDockerConfigFile(repo)
  1425. }
  1426. if r.GCPIntegrationID != 0 {
  1427. conf, err = r.getGCRDockerConfigFile(repo)
  1428. }
  1429. if r.DOIntegrationID != 0 {
  1430. conf, err = r.getDOCRDockerConfigFile(repo, doAuth)
  1431. }
  1432. if r.BasicIntegrationID != 0 {
  1433. conf, err = r.getPrivateRegistryDockerConfigFile(repo)
  1434. }
  1435. if r.AzureIntegrationID != 0 {
  1436. conf, err = r.getACRDockerConfigFile(repo)
  1437. }
  1438. if err != nil {
  1439. return nil, err
  1440. }
  1441. return json.Marshal(conf)
  1442. }
  1443. func (r *Registry) getECRDockerConfigFile(
  1444. repo repository.Repository,
  1445. ) (*configfile.ConfigFile, error) {
  1446. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  1447. r.ProjectID,
  1448. r.AWSIntegrationID,
  1449. )
  1450. if err != nil {
  1451. return nil, err
  1452. }
  1453. sess, err := aws.GetSession()
  1454. if err != nil {
  1455. return nil, err
  1456. }
  1457. ecrSvc := ecr.New(sess)
  1458. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  1459. if err != nil {
  1460. return nil, err
  1461. }
  1462. token := *output.AuthorizationData[0].AuthorizationToken
  1463. decodedToken, err := base64.StdEncoding.DecodeString(token)
  1464. if err != nil {
  1465. return nil, err
  1466. }
  1467. parts := strings.SplitN(string(decodedToken), ":", 2)
  1468. if len(parts) < 2 {
  1469. return nil, err
  1470. }
  1471. key := r.URL
  1472. if !strings.Contains(key, "http") {
  1473. key = "https://" + key
  1474. }
  1475. return &configfile.ConfigFile{
  1476. AuthConfigs: map[string]types.AuthConfig{
  1477. key: {
  1478. Username: parts[0],
  1479. Password: parts[1],
  1480. Auth: token,
  1481. },
  1482. },
  1483. }, nil
  1484. }
  1485. func (r *Registry) getGCRDockerConfigFile(
  1486. repo repository.Repository,
  1487. ) (*configfile.ConfigFile, error) {
  1488. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1489. r.ProjectID,
  1490. r.GCPIntegrationID,
  1491. )
  1492. if err != nil {
  1493. return nil, err
  1494. }
  1495. key := r.URL
  1496. if !strings.Contains(key, "http") {
  1497. key = "https://" + key
  1498. }
  1499. parsedURL, _ := url.Parse(key)
  1500. return &configfile.ConfigFile{
  1501. AuthConfigs: map[string]types.AuthConfig{
  1502. parsedURL.Host: {
  1503. Username: "_json_key",
  1504. Password: string(gcp.GCPKeyData),
  1505. Auth: generateAuthToken("_json_key", string(gcp.GCPKeyData)),
  1506. },
  1507. },
  1508. }, nil
  1509. }
  1510. func (r *Registry) getDOCRDockerConfigFile(
  1511. repo repository.Repository,
  1512. doAuth *oauth2.Config,
  1513. ) (*configfile.ConfigFile, error) {
  1514. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1515. r.ProjectID,
  1516. r.DOIntegrationID,
  1517. )
  1518. if err != nil {
  1519. return nil, err
  1520. }
  1521. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1522. if err != nil {
  1523. return nil, err
  1524. }
  1525. key := r.URL
  1526. if !strings.Contains(key, "http") {
  1527. key = "https://" + key
  1528. }
  1529. parsedURL, _ := url.Parse(key)
  1530. return &configfile.ConfigFile{
  1531. AuthConfigs: map[string]types.AuthConfig{
  1532. parsedURL.Host: {
  1533. Username: tok,
  1534. Password: tok,
  1535. Auth: generateAuthToken(tok, tok),
  1536. },
  1537. },
  1538. }, nil
  1539. }
  1540. func (r *Registry) getPrivateRegistryDockerConfigFile(
  1541. repo repository.Repository,
  1542. ) (*configfile.ConfigFile, error) {
  1543. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1544. r.ProjectID,
  1545. r.BasicIntegrationID,
  1546. )
  1547. if err != nil {
  1548. return nil, err
  1549. }
  1550. key := r.URL
  1551. if !strings.Contains(key, "http") {
  1552. key = "https://" + key
  1553. }
  1554. parsedURL, _ := url.Parse(key)
  1555. authConfigKey := parsedURL.Host
  1556. if strings.Contains(r.URL, "index.docker.io") {
  1557. authConfigKey = "https://index.docker.io/v1/"
  1558. }
  1559. return &configfile.ConfigFile{
  1560. AuthConfigs: map[string]types.AuthConfig{
  1561. authConfigKey: {
  1562. Username: string(basic.Username),
  1563. Password: string(basic.Password),
  1564. Auth: generateAuthToken(string(basic.Username), string(basic.Password)),
  1565. },
  1566. },
  1567. }, nil
  1568. }
  1569. func (r *Registry) getACRDockerConfigFile(
  1570. repo repository.Repository,
  1571. ) (*configfile.ConfigFile, error) {
  1572. username, pw, err := r.GetACRCredentials(repo)
  1573. if err != nil {
  1574. return nil, err
  1575. }
  1576. key := r.URL
  1577. if !strings.Contains(key, "http") {
  1578. key = "https://" + key
  1579. }
  1580. parsedURL, _ := url.Parse(key)
  1581. return &configfile.ConfigFile{
  1582. AuthConfigs: map[string]types.AuthConfig{
  1583. parsedURL.Host: {
  1584. Username: string(username),
  1585. Password: string(pw),
  1586. Auth: generateAuthToken(string(username), string(pw)),
  1587. },
  1588. },
  1589. }, nil
  1590. }
  1591. func generateAuthToken(username, password string) string {
  1592. return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
  1593. }