registry.go 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970
  1. package registry
  2. import (
  3. "context"
  4. "encoding/base64"
  5. "encoding/json"
  6. "fmt"
  7. "net/http"
  8. "net/url"
  9. "sort"
  10. "strings"
  11. "sync"
  12. "time"
  13. "github.com/porter-dev/porter/internal/telemetry"
  14. artifactregistry "cloud.google.com/go/artifactregistry/apiv1beta2"
  15. "connectrpc.com/connect"
  16. "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  17. "github.com/aws/aws-sdk-go/aws/awserr"
  18. "github.com/aws/aws-sdk-go/service/ecr"
  19. porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
  20. "github.com/porter-dev/porter/api/server/shared/config"
  21. "github.com/porter-dev/porter/internal/models"
  22. "github.com/porter-dev/porter/internal/oauth"
  23. "github.com/porter-dev/porter/internal/repository"
  24. "golang.org/x/oauth2"
  25. v1artifactregistry "google.golang.org/api/artifactregistry/v1"
  26. "google.golang.org/api/iterator"
  27. "google.golang.org/api/option"
  28. artifactregistrypb "google.golang.org/genproto/googleapis/devtools/artifactregistry/v1beta2"
  29. ints "github.com/porter-dev/porter/internal/models/integrations"
  30. ptypes "github.com/porter-dev/porter/api/types"
  31. "github.com/digitalocean/godo"
  32. "github.com/docker/cli/cli/config/configfile"
  33. "github.com/docker/cli/cli/config/types"
  34. "github.com/docker/distribution/reference"
  35. "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
  36. "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
  37. )
  38. // Registry wraps the gorm Registry model
  39. type Registry models.Registry
  40. func GetECRRegistryURL(awsIntRepo repository.AWSIntegrationRepository, projectID, awsIntID uint) (string, error) {
  41. awsInt, err := awsIntRepo.ReadAWSIntegration(projectID, awsIntID)
  42. if err != nil {
  43. return "", err
  44. }
  45. sess, err := awsInt.GetSession()
  46. if err != nil {
  47. return "", err
  48. }
  49. ecrSvc := ecr.New(sess)
  50. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  51. if err != nil {
  52. return "", err
  53. }
  54. return *output.AuthorizationData[0].ProxyEndpoint, nil
  55. }
  56. // ListRepositories lists the repositories for a registry
  57. func (r *Registry) ListRepositories(
  58. ctx context.Context,
  59. repo repository.Repository,
  60. conf *config.Config,
  61. ) ([]*ptypes.RegistryRepository, error) {
  62. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  63. defer span.End()
  64. telemetry.WithAttributes(span,
  65. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  66. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  67. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  68. )
  69. // switch on the auth mechanism to get a token
  70. if r.AWSIntegrationID != 0 {
  71. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "aws"})
  72. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  73. r.ProjectID,
  74. r.AWSIntegrationID,
  75. )
  76. if err != nil {
  77. return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
  78. }
  79. repos, err := r.listECRRepositories(aws)
  80. if err != nil {
  81. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  82. }
  83. return repos, nil
  84. }
  85. if r.GCPIntegrationID != 0 {
  86. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
  87. if strings.Contains(r.URL, "pkg.dev") {
  88. return r.listGARRepositories(ctx, repo)
  89. }
  90. repos, err := r.listGCRRepositories(repo)
  91. if err != nil {
  92. return nil, telemetry.Error(ctx, span, err, "error listing gcr repositories")
  93. }
  94. return repos, nil
  95. }
  96. if r.DOIntegrationID != 0 {
  97. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "do"})
  98. repos, err := r.listDOCRRepositories(repo, conf.DOConf)
  99. if err != nil {
  100. return nil, telemetry.Error(ctx, span, err, "error listing docr repositories")
  101. }
  102. return repos, nil
  103. }
  104. if r.AzureIntegrationID != 0 {
  105. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "azure"})
  106. repos, err := r.listACRRepositories(ctx, repo)
  107. if err != nil {
  108. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  109. }
  110. return repos, nil
  111. }
  112. if r.BasicIntegrationID != 0 {
  113. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "basic"})
  114. repos, err := r.listPrivateRegistryRepositories(repo)
  115. if err != nil {
  116. return nil, telemetry.Error(ctx, span, err, "error listing private repositories")
  117. }
  118. return repos, nil
  119. }
  120. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  121. if err != nil {
  122. return nil, telemetry.Error(ctx, span, err, "error getting project for repository")
  123. }
  124. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  125. // TODO: Remove this conditional when AWS list repos is supported in CCP
  126. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "registry-uri", Value: r.URL})
  127. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  128. req := connect.NewRequest(&porterv1.ListRepositoriesForRegistryRequest{
  129. ProjectId: int64(r.ProjectID),
  130. RegistryUri: r.URL,
  131. })
  132. resp, err := conf.ClusterControlPlaneClient.ListRepositoriesForRegistry(ctx, req)
  133. if err != nil {
  134. return nil, telemetry.Error(ctx, span, err, "error listing docker repositories")
  135. }
  136. res := make([]*ptypes.RegistryRepository, 0)
  137. parsedURL, err := url.Parse("https://" + r.URL)
  138. if err != nil {
  139. return nil, telemetry.Error(ctx, span, err, "error parsing url")
  140. }
  141. for _, repo := range resp.Msg.Repositories {
  142. res = append(res, &ptypes.RegistryRepository{
  143. Name: repo.Name,
  144. URI: parsedURL.Host + "/" + repo.Name,
  145. })
  146. }
  147. return res, nil
  148. } else {
  149. uri := strings.TrimPrefix(r.URL, "https://")
  150. splits := strings.Split(uri, ".")
  151. if len(splits) < 4 {
  152. return nil, telemetry.Error(ctx, span, nil, "uri does not have enough splits")
  153. }
  154. accountID := splits[0]
  155. region := splits[3]
  156. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  157. ProjectId: int64(r.ProjectID),
  158. AwsAccountId: accountID,
  159. })
  160. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  161. if err != nil {
  162. return nil, telemetry.Error(ctx, span, err, "error getting capi credentials for registry")
  163. }
  164. aws := &ints.AWSIntegration{
  165. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  166. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  167. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  168. AWSRegion: region,
  169. }
  170. repos, err := r.listECRRepositories(aws)
  171. if err != nil {
  172. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  173. }
  174. return repos, nil
  175. }
  176. }
  177. return nil, telemetry.Error(ctx, span, nil, "error listing repositories")
  178. }
  179. type gcrJWT struct {
  180. AccessToken string `json:"token"`
  181. ExpiresInSec int `json:"expires_in"`
  182. }
  183. type gcrErr struct {
  184. Code string `json:"code"`
  185. Message string `json:"message"`
  186. }
  187. type gcrRepositoryResp struct {
  188. Repositories []string `json:"repositories"`
  189. Errors []gcrErr `json:"errors"`
  190. }
  191. func (r *Registry) GetGCRToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  192. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  193. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  194. r.ProjectID,
  195. r.GCPIntegrationID,
  196. )
  197. if err != nil {
  198. return nil, err
  199. }
  200. // get oauth2 access token
  201. return gcp.GetBearerToken(
  202. ctx,
  203. getTokenCache,
  204. r.setTokenCacheFunc(ctx, repo),
  205. "https://www.googleapis.com/auth/devstorage.read_write",
  206. )
  207. }
  208. func (r *Registry) listGCRRepositories(
  209. repo repository.Repository,
  210. ) ([]*ptypes.RegistryRepository, error) {
  211. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  212. r.ProjectID,
  213. r.GCPIntegrationID,
  214. )
  215. if err != nil {
  216. return nil, err
  217. }
  218. // Just use service account key to authenticate, since scopes may not be in place
  219. // for oauth. This also prevents us from making more requests.
  220. client := &http.Client{}
  221. regURL := r.URL
  222. if !strings.HasPrefix(regURL, "http") {
  223. regURL = fmt.Sprintf("https://%s", regURL)
  224. }
  225. regURLParsed, err := url.Parse(regURL)
  226. regHostname := "gcr.io"
  227. if err == nil {
  228. regHostname = regURLParsed.Host
  229. }
  230. req, err := http.NewRequest(
  231. "GET",
  232. fmt.Sprintf("https://%s/v2/_catalog", regHostname),
  233. nil,
  234. )
  235. if err != nil {
  236. return nil, err
  237. }
  238. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  239. resp, err := client.Do(req)
  240. if err != nil {
  241. return nil, err
  242. }
  243. gcrResp := gcrRepositoryResp{}
  244. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  245. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  246. }
  247. if len(gcrResp.Errors) > 0 {
  248. errMsg := ""
  249. for _, gcrErr := range gcrResp.Errors {
  250. errMsg += fmt.Sprintf(": Code %s, message %s", gcrErr.Code, gcrErr.Message)
  251. }
  252. return nil, fmt.Errorf(errMsg)
  253. }
  254. res := make([]*ptypes.RegistryRepository, 0)
  255. parsedURL, err := url.Parse("https://" + r.URL)
  256. if err != nil {
  257. return nil, err
  258. }
  259. for _, repo := range gcrResp.Repositories {
  260. res = append(res, &ptypes.RegistryRepository{
  261. Name: repo,
  262. URI: parsedURL.Host + "/" + repo,
  263. })
  264. }
  265. return res, nil
  266. }
  267. func (r *Registry) GetGARToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  268. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  269. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  270. r.ProjectID,
  271. r.GCPIntegrationID,
  272. )
  273. if err != nil {
  274. return nil, err
  275. }
  276. // get oauth2 access token
  277. return gcp.GetBearerToken(
  278. ctx,
  279. getTokenCache,
  280. r.setTokenCacheFunc(ctx, repo),
  281. "https://www.googleapis.com/auth/cloud-platform",
  282. )
  283. }
  284. type garTokenSource struct {
  285. // ctx is only passed in here as the oauth2.Token() doesnt support contexts
  286. ctx context.Context
  287. reg *Registry
  288. repo repository.Repository
  289. }
  290. func (source *garTokenSource) Token() (*oauth2.Token, error) {
  291. return source.reg.GetGARToken(source.ctx, source.repo)
  292. }
  293. // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
  294. // where a repository is a single image. This function returns the list of fully qualified names
  295. // of GAR images including their repository names.
  296. func (r *Registry) listGARRepositories(
  297. ctx context.Context,
  298. repo repository.Repository,
  299. ) ([]*ptypes.RegistryRepository, error) {
  300. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  301. r.ProjectID,
  302. r.GCPIntegrationID,
  303. )
  304. if err != nil {
  305. return nil, err
  306. }
  307. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  308. reg: r,
  309. repo: repo,
  310. ctx: ctx,
  311. }), option.WithScopes("roles/artifactregistry.reader"))
  312. if err != nil {
  313. return nil, err
  314. }
  315. var repoNames []string
  316. nextToken := ""
  317. parsedURL, err := url.Parse("https://" + r.URL)
  318. if err != nil {
  319. return nil, err
  320. }
  321. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  322. for {
  323. it := client.ListRepositories(context.Background(), &artifactregistrypb.ListRepositoriesRequest{
  324. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  325. PageSize: 1000,
  326. PageToken: nextToken,
  327. })
  328. for {
  329. resp, err := it.Next()
  330. if err == iterator.Done {
  331. break
  332. } else if err != nil {
  333. return nil, err
  334. }
  335. if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
  336. repoSlice := strings.Split(resp.GetName(), "/")
  337. repoName := repoSlice[len(repoSlice)-1]
  338. repoNames = append(repoNames, repoName)
  339. }
  340. }
  341. if it.PageInfo().Token == "" {
  342. break
  343. }
  344. nextToken = it.PageInfo().Token
  345. }
  346. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  347. reg: r,
  348. repo: repo,
  349. ctx: ctx,
  350. }), option.WithScopes("roles/artifactregistry.reader"))
  351. if err != nil {
  352. return nil, err
  353. }
  354. nextToken = ""
  355. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  356. var (
  357. wg sync.WaitGroup
  358. resMap sync.Map
  359. )
  360. for _, repoName := range repoNames {
  361. wg.Add(1)
  362. go func(repoName string) {
  363. defer wg.Done()
  364. for {
  365. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  366. gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
  367. if err != nil {
  368. // FIXME: we should report this error using a channel
  369. return
  370. }
  371. for _, image := range resp.DockerImages {
  372. named, err := reference.ParseNamed(image.Uri)
  373. if err != nil {
  374. // let us skip this image becaue it has a malformed URI coming from the GCP API
  375. continue
  376. }
  377. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  378. resMap.Store(named.Name(), &ptypes.RegistryRepository{
  379. Name: repoName,
  380. URI: named.Name(),
  381. CreatedAt: uploadTime,
  382. })
  383. }
  384. if resp.NextPageToken == "" {
  385. break
  386. }
  387. nextToken = resp.NextPageToken
  388. }
  389. }(repoName)
  390. }
  391. wg.Wait()
  392. var res []*ptypes.RegistryRepository
  393. resMap.Range(func(_, value any) bool {
  394. res = append(res, value.(*ptypes.RegistryRepository))
  395. return true
  396. })
  397. return res, nil
  398. }
  399. func (r *Registry) listECRRepositories(aws *ints.AWSIntegration) ([]*ptypes.RegistryRepository, error) {
  400. sess, err := aws.GetSession()
  401. if err != nil {
  402. return nil, err
  403. }
  404. svc := ecr.New(sess)
  405. resp, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{})
  406. if err != nil {
  407. return nil, err
  408. }
  409. res := make([]*ptypes.RegistryRepository, 0)
  410. for _, repo := range resp.Repositories {
  411. res = append(res, &ptypes.RegistryRepository{
  412. Name: *repo.RepositoryName,
  413. CreatedAt: *repo.CreatedAt,
  414. URI: *repo.RepositoryUri,
  415. })
  416. }
  417. return res, nil
  418. }
  419. type acrRepositoryResp struct {
  420. Repositories []string `json:"repositories"`
  421. }
  422. func (r *Registry) listACRRepositories(ctx context.Context, repo repository.Repository) ([]*ptypes.RegistryRepository, error) {
  423. ctx, span := telemetry.NewSpan(ctx, "list-acr-repositories")
  424. defer span.End()
  425. telemetry.WithAttributes(span,
  426. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  427. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  428. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  429. )
  430. az, err := repo.AzureIntegration().ReadAzureIntegration(
  431. r.ProjectID,
  432. r.AzureIntegrationID,
  433. )
  434. if err != nil {
  435. return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
  436. }
  437. client := &http.Client{}
  438. acrURL := r.URL
  439. if !strings.Contains(acrURL, "http") {
  440. acrURL = fmt.Sprintf("https://%s", acrURL)
  441. }
  442. req, err := http.NewRequest(
  443. "GET",
  444. fmt.Sprintf("%s/v2/_catalog", acrURL),
  445. nil,
  446. )
  447. if err != nil {
  448. return nil, telemetry.Error(ctx, span, err, "error getting http request")
  449. }
  450. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  451. resp, err := client.Do(req)
  452. if err != nil {
  453. return nil, telemetry.Error(ctx, span, err, "error making http call")
  454. }
  455. acrResp := acrRepositoryResp{}
  456. if err := json.NewDecoder(resp.Body).Decode(&acrResp); err != nil {
  457. return nil, telemetry.Error(ctx, span, err, "could not read Azure registry repository response")
  458. }
  459. res := make([]*ptypes.RegistryRepository, 0)
  460. for _, repo := range acrResp.Repositories {
  461. res = append(res, &ptypes.RegistryRepository{
  462. Name: repo,
  463. URI: strings.TrimPrefix(r.URL, "https://") + "/" + repo,
  464. })
  465. }
  466. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "repo-count", Value: len(acrResp.Repositories)})
  467. return res, nil
  468. }
  469. // Returns the username/password pair for the registry
  470. func (r *Registry) GetACRCredentials(repo repository.Repository) (string, string, error) {
  471. az, err := repo.AzureIntegration().ReadAzureIntegration(
  472. r.ProjectID,
  473. r.AzureIntegrationID,
  474. )
  475. if err != nil {
  476. return "", "", err
  477. }
  478. // if the passwords and name aren't set, generate them
  479. if az.ACRTokenName == "" || len(az.ACRPassword1) == 0 {
  480. az.ACRTokenName = "porter-acr-token"
  481. // create an acr repo token
  482. cred, err := azidentity.NewClientSecretCredential(az.AzureTenantID, az.AzureClientID, string(az.ServicePrincipalSecret), nil)
  483. if err != nil {
  484. return "", "", err
  485. }
  486. scopeMapsClient, err := armcontainerregistry.NewScopeMapsClient(az.AzureSubscriptionID, cred, nil)
  487. if err != nil {
  488. return "", "", err
  489. }
  490. smRes, err := scopeMapsClient.Get(
  491. context.Background(),
  492. az.ACRResourceGroupName,
  493. az.ACRName,
  494. "_repositories_admin",
  495. nil,
  496. )
  497. if err != nil {
  498. return "", "", err
  499. }
  500. tokensClient, err := armcontainerregistry.NewTokensClient(az.AzureSubscriptionID, cred, nil)
  501. if err != nil {
  502. return "", "", err
  503. }
  504. pollerResp, err := tokensClient.BeginCreate(
  505. context.Background(),
  506. az.ACRResourceGroupName,
  507. az.ACRName,
  508. "porter-acr-token",
  509. armcontainerregistry.Token{
  510. Properties: &armcontainerregistry.TokenProperties{
  511. ScopeMapID: smRes.ID,
  512. Status: to.Ptr(armcontainerregistry.TokenStatusEnabled),
  513. },
  514. },
  515. nil,
  516. )
  517. if err != nil {
  518. return "", "", err
  519. }
  520. tokResp, err := pollerResp.PollUntilDone(context.Background(), 2*time.Second)
  521. if err != nil {
  522. return "", "", err
  523. }
  524. registriesClient, err := armcontainerregistry.NewRegistriesClient(az.AzureSubscriptionID, cred, nil)
  525. if err != nil {
  526. return "", "", err
  527. }
  528. poller, err := registriesClient.BeginGenerateCredentials(
  529. context.Background(),
  530. az.ACRResourceGroupName,
  531. az.ACRName,
  532. armcontainerregistry.GenerateCredentialsParameters{
  533. TokenID: tokResp.ID,
  534. },
  535. &armcontainerregistry.RegistriesClientBeginGenerateCredentialsOptions{ResumeToken: ""})
  536. if err != nil {
  537. return "", "", err
  538. }
  539. genCredentialsResp, err := poller.PollUntilDone(context.Background(), 2*time.Second)
  540. if err != nil {
  541. return "", "", err
  542. }
  543. for i, tokPassword := range genCredentialsResp.Passwords {
  544. if i == 0 {
  545. az.ACRPassword1 = []byte(*tokPassword.Value)
  546. } else if i == 1 {
  547. az.ACRPassword2 = []byte(*tokPassword.Value)
  548. }
  549. }
  550. // update the az integration
  551. az, err = repo.AzureIntegration().OverwriteAzureIntegration(
  552. az,
  553. )
  554. if err != nil {
  555. return "", "", err
  556. }
  557. }
  558. return az.ACRTokenName, string(az.ACRPassword1), nil
  559. }
  560. func (r *Registry) listDOCRRepositories(
  561. repo repository.Repository,
  562. doAuth *oauth2.Config,
  563. ) ([]*ptypes.RegistryRepository, error) {
  564. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  565. r.ProjectID,
  566. r.DOIntegrationID,
  567. )
  568. if err != nil {
  569. return nil, err
  570. }
  571. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  572. if err != nil {
  573. return nil, err
  574. }
  575. client := godo.NewFromToken(tok)
  576. urlArr := strings.Split(r.URL, "/")
  577. if len(urlArr) != 2 {
  578. return nil, fmt.Errorf("invalid digital ocean registry url")
  579. }
  580. name := urlArr[1]
  581. repos, _, err := client.Registry.ListRepositories(context.TODO(), name, &godo.ListOptions{})
  582. if err != nil {
  583. return nil, err
  584. }
  585. res := make([]*ptypes.RegistryRepository, 0)
  586. for _, repo := range repos {
  587. res = append(res, &ptypes.RegistryRepository{
  588. Name: repo.Name,
  589. URI: r.URL + "/" + repo.Name,
  590. })
  591. }
  592. return res, nil
  593. }
  594. func (r *Registry) listPrivateRegistryRepositories(
  595. repo repository.Repository,
  596. ) ([]*ptypes.RegistryRepository, error) {
  597. // handle dockerhub different, as it doesn't implement the docker registry http api
  598. if strings.Contains(r.URL, "docker.io") {
  599. // in this case, we just return the single dockerhub repository that's linked
  600. res := make([]*ptypes.RegistryRepository, 0)
  601. res = append(res, &ptypes.RegistryRepository{
  602. Name: strings.Split(r.URL, "docker.io/")[1],
  603. URI: r.URL,
  604. })
  605. return res, nil
  606. }
  607. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  608. r.ProjectID,
  609. r.BasicIntegrationID,
  610. )
  611. if err != nil {
  612. return nil, err
  613. }
  614. // Just use service account key to authenticate, since scopes may not be in place
  615. // for oauth. This also prevents us from making more requests.
  616. client := &http.Client{}
  617. // get the host and scheme to make the request
  618. parsedURL, err := url.Parse(r.URL)
  619. req, err := http.NewRequest(
  620. "GET",
  621. fmt.Sprintf("%s://%s/v2/_catalog", parsedURL.Scheme, parsedURL.Host),
  622. nil,
  623. )
  624. if err != nil {
  625. return nil, err
  626. }
  627. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  628. resp, err := client.Do(req)
  629. if err != nil {
  630. return nil, err
  631. }
  632. // if the status code is 404, fallback to the Docker Hub implementation
  633. if resp.StatusCode == 404 {
  634. req, err := http.NewRequest(
  635. "GET",
  636. fmt.Sprintf("%s/", r.URL),
  637. nil,
  638. )
  639. if err != nil {
  640. return nil, err
  641. }
  642. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  643. resp, err = client.Do(req)
  644. if err != nil {
  645. return nil, err
  646. }
  647. }
  648. gcrResp := gcrRepositoryResp{}
  649. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  650. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  651. }
  652. res := make([]*ptypes.RegistryRepository, 0)
  653. if err != nil {
  654. return nil, err
  655. }
  656. for _, repo := range gcrResp.Repositories {
  657. res = append(res, &ptypes.RegistryRepository{
  658. Name: repo,
  659. URI: parsedURL.Host + "/" + repo,
  660. })
  661. }
  662. return res, nil
  663. }
  664. func (r *Registry) getTokenCacheFunc(
  665. ctx context.Context,
  666. repo repository.Repository,
  667. ) ints.GetTokenCacheFunc {
  668. return func(ctx context.Context) (tok *ints.TokenCache, err error) {
  669. reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
  670. if err != nil {
  671. return nil, err
  672. }
  673. return &reg.TokenCache.TokenCache, nil
  674. }
  675. }
  676. func (r *Registry) setTokenCacheFunc(
  677. ctx context.Context,
  678. repo repository.Repository,
  679. ) ints.SetTokenCacheFunc {
  680. return func(ctx context.Context, token string, expiry time.Time) error {
  681. _, err := repo.Registry().UpdateRegistryTokenCache(
  682. &ints.RegTokenCache{
  683. TokenCache: ints.TokenCache{
  684. Token: []byte(token),
  685. Expiry: expiry,
  686. },
  687. RegistryID: r.ID,
  688. },
  689. )
  690. return err
  691. }
  692. }
  693. // CreateRepository creates a repository for a registry, if needed
  694. // (currently only required for ECR)
  695. func (r *Registry) CreateRepository(
  696. ctx context.Context,
  697. conf *config.Config,
  698. name string,
  699. ) error {
  700. ctx, span := telemetry.NewSpan(ctx, "create-repository")
  701. defer span.End()
  702. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "registry-uri", Value: r.URL})
  703. // if aws, create repository
  704. if r.AWSIntegrationID != 0 {
  705. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "aws-integration-id", Value: r.AWSIntegrationID})
  706. aws, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
  707. r.ProjectID,
  708. r.AWSIntegrationID,
  709. )
  710. if err != nil {
  711. return telemetry.Error(ctx, span, err, "error reading aws integration")
  712. }
  713. err = r.createECRRepository(aws, name)
  714. if err != nil {
  715. return telemetry.Error(ctx, span, err, "error creating ecr repository")
  716. }
  717. return nil
  718. } else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
  719. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "gcp-integration-id", Value: r.GCPIntegrationID})
  720. err := r.createGARRepository(ctx, conf.Repo, name)
  721. if err != nil {
  722. return telemetry.Error(ctx, span, err, "error creating gar repository")
  723. }
  724. return nil
  725. }
  726. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  727. if err != nil {
  728. return telemetry.Error(ctx, span, err, "error getting project for repository")
  729. }
  730. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  731. // no need to create repository if pushing to ACR or GAR
  732. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  733. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "skipping-create-repo", Value: true})
  734. return nil
  735. }
  736. uri := strings.TrimPrefix(r.URL, "https://")
  737. splits := strings.Split(uri, ".")
  738. accountID := splits[0]
  739. region := splits[3]
  740. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  741. ProjectId: int64(r.ProjectID),
  742. AwsAccountId: accountID,
  743. })
  744. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "uri", Value: uri})
  745. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  746. if err != nil {
  747. return telemetry.Error(ctx, span, err, "error getting capi credentials for repository")
  748. }
  749. aws := &ints.AWSIntegration{
  750. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  751. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  752. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  753. AWSRegion: region,
  754. }
  755. err = r.createECRRepository(aws, name)
  756. if err != nil {
  757. return telemetry.Error(ctx, span, err, "error creating ecr repository")
  758. }
  759. }
  760. // otherwise, no-op
  761. return nil
  762. }
  763. func (r *Registry) createECRRepository(
  764. aws *ints.AWSIntegration,
  765. name string,
  766. ) error {
  767. sess, err := aws.GetSession()
  768. if err != nil {
  769. return err
  770. }
  771. svc := ecr.New(sess)
  772. // determine if repository already exists
  773. _, err = svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
  774. RepositoryNames: []*string{&name},
  775. })
  776. // if the repository was not found, create it
  777. if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ecr.ErrCodeRepositoryNotFoundException {
  778. _, err = svc.CreateRepository(&ecr.CreateRepositoryInput{
  779. RepositoryName: &name,
  780. })
  781. return err
  782. } else if err != nil {
  783. return err
  784. }
  785. return nil
  786. }
  787. func (r *Registry) createGARRepository(
  788. ctx context.Context,
  789. repo repository.Repository,
  790. name string,
  791. ) error {
  792. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  793. r.ProjectID,
  794. r.GCPIntegrationID,
  795. )
  796. if err != nil {
  797. return err
  798. }
  799. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  800. reg: r,
  801. repo: repo,
  802. ctx: ctx,
  803. }), option.WithScopes("roles/artifactregistry.admin"))
  804. if err != nil {
  805. return err
  806. }
  807. defer client.Close()
  808. parsedURL, err := url.Parse("https://" + r.URL)
  809. if err != nil {
  810. return err
  811. }
  812. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  813. _, err = client.GetRepository(context.Background(), &artifactregistrypb.GetRepositoryRequest{
  814. Name: fmt.Sprintf("projects/%s/locations/%s/repositories/%s", gcpInt.GCPProjectID, location, name),
  815. })
  816. if err != nil && strings.Contains(err.Error(), "not found") {
  817. // create a new repository
  818. _, err := client.CreateRepository(context.Background(), &artifactregistrypb.CreateRepositoryRequest{
  819. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  820. RepositoryId: name,
  821. Repository: &artifactregistrypb.Repository{
  822. Format: artifactregistrypb.Repository_DOCKER,
  823. },
  824. })
  825. if err != nil {
  826. return err
  827. }
  828. } else if err != nil {
  829. return err
  830. }
  831. return nil
  832. }
  833. // ListImages lists the images for an image repository
  834. func (r *Registry) ListImages(
  835. ctx context.Context,
  836. repoName string,
  837. repo repository.Repository,
  838. conf *config.Config,
  839. ) ([]*ptypes.Image, error) {
  840. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  841. defer span.End()
  842. telemetry.WithAttributes(span,
  843. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  844. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  845. telemetry.AttributeKV{Key: "registry-url", Value: r.URL},
  846. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  847. telemetry.AttributeKV{Key: "repo-name", Value: repoName},
  848. )
  849. // switch on the auth mechanism to get a token
  850. if r.AWSIntegrationID != 0 {
  851. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  852. r.ProjectID,
  853. r.AWSIntegrationID,
  854. )
  855. if err != nil {
  856. return nil, err
  857. }
  858. return r.listECRImages(aws, repoName, repo)
  859. }
  860. if r.AzureIntegrationID != 0 {
  861. return r.listACRImages(repoName, repo)
  862. }
  863. if r.GCPIntegrationID != 0 {
  864. if strings.Contains(r.URL, "pkg.dev") {
  865. return r.listGARImages(ctx, repoName, repo)
  866. }
  867. return r.listGCRImages(repoName, repo)
  868. }
  869. if r.DOIntegrationID != 0 {
  870. return r.listDOCRImages(repoName, repo, conf.DOConf)
  871. }
  872. if r.BasicIntegrationID != 0 {
  873. return r.listPrivateRegistryImages(repoName, repo)
  874. }
  875. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  876. if err != nil {
  877. return nil, fmt.Errorf("error getting project for repository: %w", err)
  878. }
  879. if project.GetFeatureFlag(models.CapiProvisionerEnabled, conf.LaunchDarklyClient) {
  880. if strings.Contains(r.URL, ".azurecr.") || strings.Contains(r.URL, "-docker.pkg.dev") {
  881. req := connect.NewRequest(&porterv1.ListImagesForRepositoryRequest{
  882. ProjectId: int64(r.ProjectID),
  883. RegistryUri: r.URL,
  884. RepoName: repoName,
  885. })
  886. resp, err := conf.ClusterControlPlaneClient.ListImagesForRepository(ctx, req)
  887. if err != nil {
  888. return nil, telemetry.Error(ctx, span, err, "error calling ccp list images")
  889. }
  890. res := make([]*ptypes.Image, 0)
  891. for _, image := range resp.Msg.Images {
  892. if image.UpdatedAt == nil {
  893. continue
  894. }
  895. lastUpdateTime := image.UpdatedAt.AsTime()
  896. res = append(res, &ptypes.Image{
  897. Digest: image.Digest,
  898. Tag: image.Tag,
  899. Manifest: "",
  900. RepositoryName: image.RepositoryName,
  901. PushedAt: &lastUpdateTime,
  902. })
  903. }
  904. return res, nil
  905. }
  906. uri := strings.TrimPrefix(r.URL, "https://")
  907. splits := strings.Split(uri, ".")
  908. accountID := splits[0]
  909. region := splits[3]
  910. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  911. ProjectId: int64(r.ProjectID),
  912. AwsAccountId: accountID,
  913. })
  914. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  915. if err != nil {
  916. return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
  917. }
  918. aws := &ints.AWSIntegration{
  919. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  920. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  921. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  922. AWSRegion: region,
  923. }
  924. return r.listECRImages(aws, repoName, repo)
  925. }
  926. return nil, fmt.Errorf("error listing images")
  927. }
  928. func (r *Registry) GetECRPaginatedImages(
  929. repoName string,
  930. repo repository.Repository,
  931. maxResults int64,
  932. nextToken *string,
  933. ) ([]*ptypes.Image, *string, error) {
  934. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  935. r.ProjectID,
  936. r.AWSIntegrationID,
  937. )
  938. if err != nil {
  939. return nil, nil, err
  940. }
  941. sess, err := aws.GetSession()
  942. if err != nil {
  943. return nil, nil, err
  944. }
  945. svc := ecr.New(sess)
  946. resp, err := svc.ListImages(&ecr.ListImagesInput{
  947. RepositoryName: &repoName,
  948. MaxResults: &maxResults,
  949. NextToken: nextToken,
  950. })
  951. if err != nil {
  952. return nil, nil, err
  953. }
  954. if len(resp.ImageIds) == 0 {
  955. return []*ptypes.Image{}, nil, nil
  956. }
  957. imageIDLen := len(resp.ImageIds)
  958. imageDetails := make([]*ecr.ImageDetail, 0)
  959. imageIDMap := make(map[string]bool)
  960. for _, id := range resp.ImageIds {
  961. if id != nil && id.ImageTag != nil {
  962. imageIDMap[*id.ImageTag] = true
  963. }
  964. }
  965. var wg sync.WaitGroup
  966. var mu sync.Mutex
  967. // AWS API expects the length of imageIDs to be at max 100 at a time
  968. for start := 0; start < imageIDLen; start += 100 {
  969. end := start + 100
  970. if end > imageIDLen {
  971. end = imageIDLen
  972. }
  973. wg.Add(1)
  974. go func(start, end int) {
  975. defer wg.Done()
  976. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  977. RepositoryName: &repoName,
  978. ImageIds: resp.ImageIds[start:end],
  979. })
  980. if err != nil {
  981. return
  982. }
  983. mu.Lock()
  984. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  985. mu.Unlock()
  986. }(start, end)
  987. }
  988. wg.Wait()
  989. res := make([]*ptypes.Image, 0)
  990. imageInfoMap := make(map[string]*ptypes.Image)
  991. for _, img := range imageDetails {
  992. for _, tag := range img.ImageTags {
  993. newImage := &ptypes.Image{
  994. Digest: *img.ImageDigest,
  995. Tag: *tag,
  996. RepositoryName: repoName,
  997. PushedAt: img.ImagePushedAt,
  998. }
  999. if _, ok := imageIDMap[*tag]; ok {
  1000. if _, ok := imageInfoMap[*tag]; !ok {
  1001. imageInfoMap[*tag] = newImage
  1002. }
  1003. }
  1004. if len(imageInfoMap) == int(maxResults) {
  1005. break
  1006. }
  1007. }
  1008. if len(imageInfoMap) == int(maxResults) {
  1009. break
  1010. }
  1011. }
  1012. for _, v := range imageInfoMap {
  1013. res = append(res, v)
  1014. }
  1015. sort.Slice(res, func(i, j int) bool {
  1016. if res[i].PushedAt == nil || res[j].PushedAt == nil {
  1017. return false
  1018. }
  1019. return res[i].PushedAt.After(*res[j].PushedAt)
  1020. })
  1021. return res, resp.NextToken, nil
  1022. }
  1023. func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1024. sess, err := aws.GetSession()
  1025. if err != nil {
  1026. return nil, err
  1027. }
  1028. svc := ecr.New(sess)
  1029. maxResults := int64(1000)
  1030. var imageIDs []*ecr.ImageIdentifier
  1031. resp, err := svc.ListImages(&ecr.ListImagesInput{
  1032. RepositoryName: &repoName,
  1033. MaxResults: &maxResults,
  1034. })
  1035. if err != nil {
  1036. return nil, err
  1037. }
  1038. if len(resp.ImageIds) == 0 {
  1039. return []*ptypes.Image{}, nil
  1040. }
  1041. imageIDs = append(imageIDs, resp.ImageIds...)
  1042. nextToken := resp.NextToken
  1043. for nextToken != nil {
  1044. resp, err := svc.ListImages(&ecr.ListImagesInput{
  1045. RepositoryName: &repoName,
  1046. MaxResults: &maxResults,
  1047. NextToken: nextToken,
  1048. })
  1049. if err != nil {
  1050. return nil, err
  1051. }
  1052. imageIDs = append(imageIDs, resp.ImageIds...)
  1053. nextToken = resp.NextToken
  1054. }
  1055. imageIDLen := len(imageIDs)
  1056. imageDetails := make([]*ecr.ImageDetail, 0)
  1057. var wg sync.WaitGroup
  1058. var mu sync.Mutex
  1059. // AWS API expects the length of imageIDs to be at max 100 at a time
  1060. for start := 0; start < imageIDLen; start += 100 {
  1061. end := start + 100
  1062. if end > imageIDLen {
  1063. end = imageIDLen
  1064. }
  1065. wg.Add(1)
  1066. go func(start, end int) {
  1067. defer wg.Done()
  1068. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  1069. RepositoryName: &repoName,
  1070. ImageIds: imageIDs[start:end],
  1071. })
  1072. if err != nil {
  1073. return
  1074. }
  1075. mu.Lock()
  1076. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  1077. mu.Unlock()
  1078. }(start, end)
  1079. }
  1080. wg.Wait()
  1081. res := make([]*ptypes.Image, 0)
  1082. imageInfoMap := make(map[string]*ptypes.Image)
  1083. for _, img := range imageDetails {
  1084. for _, tag := range img.ImageTags {
  1085. newImage := &ptypes.Image{
  1086. Digest: *img.ImageDigest,
  1087. Tag: *tag,
  1088. RepositoryName: repoName,
  1089. PushedAt: img.ImagePushedAt,
  1090. }
  1091. if _, ok := imageInfoMap[*tag]; !ok {
  1092. imageInfoMap[*tag] = newImage
  1093. }
  1094. }
  1095. }
  1096. for _, v := range imageInfoMap {
  1097. res = append(res, v)
  1098. }
  1099. return res, nil
  1100. }
  1101. func (r *Registry) listACRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1102. az, err := repo.AzureIntegration().ReadAzureIntegration(
  1103. r.ProjectID,
  1104. r.AzureIntegrationID,
  1105. )
  1106. if err != nil {
  1107. return nil, err
  1108. }
  1109. // use JWT token to request catalog
  1110. client := &http.Client{}
  1111. req, err := http.NewRequest(
  1112. "GET",
  1113. fmt.Sprintf("%s/v2/%s/tags/list", r.URL, repoName),
  1114. nil,
  1115. )
  1116. if err != nil {
  1117. return nil, err
  1118. }
  1119. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  1120. resp, err := client.Do(req)
  1121. if err != nil {
  1122. return nil, err
  1123. }
  1124. gcrResp := gcrImageResp{}
  1125. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1126. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1127. }
  1128. res := make([]*ptypes.Image, 0)
  1129. for _, tag := range gcrResp.Tags {
  1130. res = append(res, &ptypes.Image{
  1131. RepositoryName: strings.TrimPrefix(repoName, "https://"),
  1132. Tag: tag,
  1133. })
  1134. }
  1135. return res, nil
  1136. }
  1137. type gcrImageResp struct {
  1138. Tags []string `json:"tags"`
  1139. }
  1140. func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1141. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1142. r.ProjectID,
  1143. r.GCPIntegrationID,
  1144. )
  1145. if err != nil {
  1146. return nil, err
  1147. }
  1148. // use JWT token to request catalog
  1149. client := &http.Client{}
  1150. parsedURL, err := url.Parse("https://" + r.URL)
  1151. if err != nil {
  1152. return nil, err
  1153. }
  1154. trimmedPath := strings.Trim(parsedURL.Path, "/")
  1155. req, err := http.NewRequest(
  1156. "GET",
  1157. fmt.Sprintf("https://%s/v2/%s/%s/tags/list", parsedURL.Host, trimmedPath, repoName),
  1158. nil,
  1159. )
  1160. if err != nil {
  1161. return nil, err
  1162. }
  1163. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  1164. resp, err := client.Do(req)
  1165. if err != nil {
  1166. return nil, err
  1167. }
  1168. gcrResp := gcrImageResp{}
  1169. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1170. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1171. }
  1172. res := make([]*ptypes.Image, 0)
  1173. for _, tag := range gcrResp.Tags {
  1174. res = append(res, &ptypes.Image{
  1175. RepositoryName: repoName,
  1176. Tag: tag,
  1177. })
  1178. }
  1179. return res, nil
  1180. }
  1181. func (r *Registry) listGARImages(ctx context.Context, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1182. repoImageSlice := strings.Split(repoName, "/")
  1183. if len(repoImageSlice) != 2 {
  1184. return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
  1185. }
  1186. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  1187. r.ProjectID,
  1188. r.GCPIntegrationID,
  1189. )
  1190. if err != nil {
  1191. return nil, err
  1192. }
  1193. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  1194. reg: r,
  1195. repo: repo,
  1196. ctx: ctx,
  1197. }), option.WithScopes("roles/artifactregistry.reader"))
  1198. if err != nil {
  1199. return nil, err
  1200. }
  1201. var res []*ptypes.Image
  1202. parsedURL, err := url.Parse("https://" + r.URL)
  1203. if err != nil {
  1204. return nil, err
  1205. }
  1206. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  1207. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  1208. nextToken := ""
  1209. for {
  1210. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  1211. gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
  1212. if err != nil {
  1213. return nil, err
  1214. }
  1215. for _, image := range resp.DockerImages {
  1216. named, err := reference.ParseNamed(image.Uri)
  1217. if err != nil {
  1218. continue
  1219. }
  1220. paths := strings.Split(reference.Path(named), "/")
  1221. imageName := paths[len(paths)-1]
  1222. if imageName == repoImageSlice[1] {
  1223. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  1224. for _, tag := range image.Tags {
  1225. res = append(res, &ptypes.Image{
  1226. RepositoryName: repoName,
  1227. Tag: tag,
  1228. PushedAt: &uploadTime,
  1229. Digest: strings.Split(image.Uri, "@")[1],
  1230. })
  1231. }
  1232. }
  1233. }
  1234. if resp.NextPageToken == "" {
  1235. break
  1236. }
  1237. nextToken = resp.NextPageToken
  1238. }
  1239. return res, nil
  1240. }
  1241. func (r *Registry) listDOCRImages(
  1242. repoName string,
  1243. repo repository.Repository,
  1244. doAuth *oauth2.Config,
  1245. ) ([]*ptypes.Image, error) {
  1246. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1247. r.ProjectID,
  1248. r.DOIntegrationID,
  1249. )
  1250. if err != nil {
  1251. return nil, err
  1252. }
  1253. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1254. if err != nil {
  1255. return nil, err
  1256. }
  1257. client := godo.NewFromToken(tok)
  1258. urlArr := strings.Split(r.URL, "/")
  1259. if len(urlArr) != 2 {
  1260. return nil, fmt.Errorf("invalid digital ocean registry url")
  1261. }
  1262. name := urlArr[1]
  1263. var tags []*godo.RepositoryTag
  1264. opt := &godo.ListOptions{
  1265. PerPage: 200,
  1266. }
  1267. for {
  1268. nextTags, resp, err := client.Registry.ListRepositoryTags(context.TODO(), name, repoName, opt)
  1269. if err != nil {
  1270. return nil, err
  1271. }
  1272. tags = append(tags, nextTags...)
  1273. if resp.Links == nil || resp.Links.IsLastPage() {
  1274. break
  1275. }
  1276. page, err := resp.Links.CurrentPage()
  1277. if err != nil {
  1278. return nil, err
  1279. }
  1280. opt.Page = page + 1
  1281. }
  1282. res := make([]*ptypes.Image, 0)
  1283. for _, tag := range tags {
  1284. res = append(res, &ptypes.Image{
  1285. RepositoryName: repoName,
  1286. Tag: tag.Tag,
  1287. })
  1288. }
  1289. return res, nil
  1290. }
  1291. func (r *Registry) listPrivateRegistryImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1292. // handle dockerhub different, as it doesn't implement the docker registry http api
  1293. if strings.Contains(r.URL, "docker.io") {
  1294. return r.listDockerHubImages(repoName, repo)
  1295. }
  1296. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1297. r.ProjectID,
  1298. r.BasicIntegrationID,
  1299. )
  1300. if err != nil {
  1301. return nil, err
  1302. }
  1303. // Just use service account key to authenticate, since scopes may not be in place
  1304. // for oauth. This also prevents us from making more requests.
  1305. client := &http.Client{}
  1306. // get the host and scheme to make the request
  1307. parsedURL, err := url.Parse(r.URL)
  1308. req, err := http.NewRequest(
  1309. "GET",
  1310. fmt.Sprintf("%s://%s/v2/%s/tags/list", parsedURL.Scheme, parsedURL.Host, repoName),
  1311. nil,
  1312. )
  1313. if err != nil {
  1314. return nil, err
  1315. }
  1316. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  1317. resp, err := client.Do(req)
  1318. if err != nil {
  1319. return nil, err
  1320. }
  1321. gcrResp := gcrImageResp{}
  1322. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1323. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1324. }
  1325. res := make([]*ptypes.Image, 0)
  1326. for _, tag := range gcrResp.Tags {
  1327. res = append(res, &ptypes.Image{
  1328. RepositoryName: repoName,
  1329. Tag: tag,
  1330. })
  1331. }
  1332. return res, nil
  1333. }
  1334. type dockerHubImageResult struct {
  1335. Name string `json:"name"`
  1336. }
  1337. type dockerHubImageResp struct {
  1338. Results []dockerHubImageResult `json:"results"`
  1339. }
  1340. type dockerHubLoginReq struct {
  1341. Username string `json:"username"`
  1342. Password string `json:"password"`
  1343. }
  1344. type dockerHubLoginResp struct {
  1345. Token string `json:"token"`
  1346. }
  1347. func (r *Registry) listDockerHubImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1348. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1349. r.ProjectID,
  1350. r.BasicIntegrationID,
  1351. )
  1352. if err != nil {
  1353. return nil, err
  1354. }
  1355. client := &http.Client{}
  1356. // first, make a request for the access token
  1357. data, err := json.Marshal(&dockerHubLoginReq{
  1358. Username: string(basic.Username),
  1359. Password: string(basic.Password),
  1360. })
  1361. if err != nil {
  1362. return nil, err
  1363. }
  1364. req, err := http.NewRequest(
  1365. "POST",
  1366. "https://hub.docker.com/v2/users/login",
  1367. strings.NewReader(string(data)),
  1368. )
  1369. if err != nil {
  1370. return nil, err
  1371. }
  1372. req.Header.Add("Content-Type", "application/json")
  1373. resp, err := client.Do(req)
  1374. if err != nil {
  1375. return nil, err
  1376. }
  1377. tokenObj := dockerHubLoginResp{}
  1378. if err := json.NewDecoder(resp.Body).Decode(&tokenObj); err != nil {
  1379. return nil, fmt.Errorf("Could not decode Dockerhub token from response: %v", err)
  1380. }
  1381. req, err = http.NewRequest(
  1382. "GET",
  1383. fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags", strings.Split(r.URL, "docker.io/")[1]),
  1384. nil,
  1385. )
  1386. if err != nil {
  1387. return nil, err
  1388. }
  1389. req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenObj.Token))
  1390. resp, err = client.Do(req)
  1391. if err != nil {
  1392. return nil, err
  1393. }
  1394. imageResp := dockerHubImageResp{}
  1395. if err := json.NewDecoder(resp.Body).Decode(&imageResp); err != nil {
  1396. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1397. }
  1398. res := make([]*ptypes.Image, 0)
  1399. for _, result := range imageResp.Results {
  1400. res = append(res, &ptypes.Image{
  1401. RepositoryName: repoName,
  1402. Tag: result.Name,
  1403. })
  1404. }
  1405. return res, nil
  1406. }
  1407. // GetDockerConfigJSON returns a dockerconfigjson file contents with "auths"
  1408. // populated.
  1409. func (r *Registry) GetDockerConfigJSON(
  1410. repo repository.Repository,
  1411. doAuth *oauth2.Config, // only required if using DOCR
  1412. ) ([]byte, error) {
  1413. var conf *configfile.ConfigFile
  1414. var err error
  1415. // switch on the auth mechanism to get a token
  1416. if r.AWSIntegrationID != 0 {
  1417. conf, err = r.getECRDockerConfigFile(repo)
  1418. }
  1419. if r.GCPIntegrationID != 0 {
  1420. conf, err = r.getGCRDockerConfigFile(repo)
  1421. }
  1422. if r.DOIntegrationID != 0 {
  1423. conf, err = r.getDOCRDockerConfigFile(repo, doAuth)
  1424. }
  1425. if r.BasicIntegrationID != 0 {
  1426. conf, err = r.getPrivateRegistryDockerConfigFile(repo)
  1427. }
  1428. if r.AzureIntegrationID != 0 {
  1429. conf, err = r.getACRDockerConfigFile(repo)
  1430. }
  1431. if err != nil {
  1432. return nil, err
  1433. }
  1434. return json.Marshal(conf)
  1435. }
  1436. func (r *Registry) getECRDockerConfigFile(
  1437. repo repository.Repository,
  1438. ) (*configfile.ConfigFile, error) {
  1439. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  1440. r.ProjectID,
  1441. r.AWSIntegrationID,
  1442. )
  1443. if err != nil {
  1444. return nil, err
  1445. }
  1446. sess, err := aws.GetSession()
  1447. if err != nil {
  1448. return nil, err
  1449. }
  1450. ecrSvc := ecr.New(sess)
  1451. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  1452. if err != nil {
  1453. return nil, err
  1454. }
  1455. token := *output.AuthorizationData[0].AuthorizationToken
  1456. decodedToken, err := base64.StdEncoding.DecodeString(token)
  1457. if err != nil {
  1458. return nil, err
  1459. }
  1460. parts := strings.SplitN(string(decodedToken), ":", 2)
  1461. if len(parts) < 2 {
  1462. return nil, err
  1463. }
  1464. key := r.URL
  1465. if !strings.Contains(key, "http") {
  1466. key = "https://" + key
  1467. }
  1468. return &configfile.ConfigFile{
  1469. AuthConfigs: map[string]types.AuthConfig{
  1470. key: {
  1471. Username: parts[0],
  1472. Password: parts[1],
  1473. Auth: token,
  1474. },
  1475. },
  1476. }, nil
  1477. }
  1478. func (r *Registry) getGCRDockerConfigFile(
  1479. repo repository.Repository,
  1480. ) (*configfile.ConfigFile, error) {
  1481. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1482. r.ProjectID,
  1483. r.GCPIntegrationID,
  1484. )
  1485. if err != nil {
  1486. return nil, err
  1487. }
  1488. key := r.URL
  1489. if !strings.Contains(key, "http") {
  1490. key = "https://" + key
  1491. }
  1492. parsedURL, _ := url.Parse(key)
  1493. return &configfile.ConfigFile{
  1494. AuthConfigs: map[string]types.AuthConfig{
  1495. parsedURL.Host: {
  1496. Username: "_json_key",
  1497. Password: string(gcp.GCPKeyData),
  1498. Auth: generateAuthToken("_json_key", string(gcp.GCPKeyData)),
  1499. },
  1500. },
  1501. }, nil
  1502. }
  1503. func (r *Registry) getDOCRDockerConfigFile(
  1504. repo repository.Repository,
  1505. doAuth *oauth2.Config,
  1506. ) (*configfile.ConfigFile, error) {
  1507. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1508. r.ProjectID,
  1509. r.DOIntegrationID,
  1510. )
  1511. if err != nil {
  1512. return nil, err
  1513. }
  1514. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1515. if err != nil {
  1516. return nil, err
  1517. }
  1518. key := r.URL
  1519. if !strings.Contains(key, "http") {
  1520. key = "https://" + key
  1521. }
  1522. parsedURL, _ := url.Parse(key)
  1523. return &configfile.ConfigFile{
  1524. AuthConfigs: map[string]types.AuthConfig{
  1525. parsedURL.Host: {
  1526. Username: tok,
  1527. Password: tok,
  1528. Auth: generateAuthToken(tok, tok),
  1529. },
  1530. },
  1531. }, nil
  1532. }
  1533. func (r *Registry) getPrivateRegistryDockerConfigFile(
  1534. repo repository.Repository,
  1535. ) (*configfile.ConfigFile, error) {
  1536. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1537. r.ProjectID,
  1538. r.BasicIntegrationID,
  1539. )
  1540. if err != nil {
  1541. return nil, err
  1542. }
  1543. key := r.URL
  1544. if !strings.Contains(key, "http") {
  1545. key = "https://" + key
  1546. }
  1547. parsedURL, _ := url.Parse(key)
  1548. authConfigKey := parsedURL.Host
  1549. if strings.Contains(r.URL, "index.docker.io") {
  1550. authConfigKey = "https://index.docker.io/v1/"
  1551. }
  1552. return &configfile.ConfigFile{
  1553. AuthConfigs: map[string]types.AuthConfig{
  1554. authConfigKey: {
  1555. Username: string(basic.Username),
  1556. Password: string(basic.Password),
  1557. Auth: generateAuthToken(string(basic.Username), string(basic.Password)),
  1558. },
  1559. },
  1560. }, nil
  1561. }
  1562. func (r *Registry) getACRDockerConfigFile(
  1563. repo repository.Repository,
  1564. ) (*configfile.ConfigFile, error) {
  1565. username, pw, err := r.GetACRCredentials(repo)
  1566. if err != nil {
  1567. return nil, err
  1568. }
  1569. key := r.URL
  1570. if !strings.Contains(key, "http") {
  1571. key = "https://" + key
  1572. }
  1573. parsedURL, _ := url.Parse(key)
  1574. return &configfile.ConfigFile{
  1575. AuthConfigs: map[string]types.AuthConfig{
  1576. parsedURL.Host: {
  1577. Username: string(username),
  1578. Password: string(pw),
  1579. Auth: generateAuthToken(string(username), string(pw)),
  1580. },
  1581. },
  1582. }, nil
  1583. }
  1584. func generateAuthToken(username, password string) string {
  1585. return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
  1586. }