registry.go 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896
  1. package registry
  2. import (
  3. "context"
  4. "encoding/base64"
  5. "encoding/json"
  6. "fmt"
  7. "net/http"
  8. "net/url"
  9. "sort"
  10. "strings"
  11. "sync"
  12. "time"
  13. "github.com/porter-dev/porter/internal/telemetry"
  14. artifactregistry "cloud.google.com/go/artifactregistry/apiv1beta2"
  15. "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  16. "github.com/aws/aws-sdk-go/aws/awserr"
  17. "github.com/aws/aws-sdk-go/service/ecr"
  18. "github.com/bufbuild/connect-go"
  19. porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
  20. "github.com/porter-dev/porter/api/server/shared/config"
  21. "github.com/porter-dev/porter/internal/models"
  22. "github.com/porter-dev/porter/internal/oauth"
  23. "github.com/porter-dev/porter/internal/repository"
  24. "golang.org/x/oauth2"
  25. v1artifactregistry "google.golang.org/api/artifactregistry/v1"
  26. "google.golang.org/api/iterator"
  27. "google.golang.org/api/option"
  28. artifactregistrypb "google.golang.org/genproto/googleapis/devtools/artifactregistry/v1beta2"
  29. ints "github.com/porter-dev/porter/internal/models/integrations"
  30. ptypes "github.com/porter-dev/porter/api/types"
  31. "github.com/digitalocean/godo"
  32. "github.com/docker/cli/cli/config/configfile"
  33. "github.com/docker/cli/cli/config/types"
  34. "github.com/docker/distribution/reference"
  35. "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
  36. "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
  37. )
  38. // Registry wraps the gorm Registry model
  39. type Registry models.Registry
  40. func GetECRRegistryURL(awsIntRepo repository.AWSIntegrationRepository, projectID, awsIntID uint) (string, error) {
  41. awsInt, err := awsIntRepo.ReadAWSIntegration(projectID, awsIntID)
  42. if err != nil {
  43. return "", err
  44. }
  45. sess, err := awsInt.GetSession()
  46. if err != nil {
  47. return "", err
  48. }
  49. ecrSvc := ecr.New(sess)
  50. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  51. if err != nil {
  52. return "", err
  53. }
  54. return *output.AuthorizationData[0].ProxyEndpoint, nil
  55. }
  56. // ListRepositories lists the repositories for a registry
  57. func (r *Registry) ListRepositories(
  58. ctx context.Context,
  59. repo repository.Repository,
  60. conf *config.Config,
  61. ) ([]*ptypes.RegistryRepository, error) {
  62. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  63. defer span.End()
  64. telemetry.WithAttributes(span,
  65. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  66. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  67. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  68. )
  69. // switch on the auth mechanism to get a token
  70. if r.AWSIntegrationID != 0 {
  71. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "aws"})
  72. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  73. r.ProjectID,
  74. r.AWSIntegrationID,
  75. )
  76. if err != nil {
  77. return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
  78. }
  79. repos, err := r.listECRRepositories(aws)
  80. if err != nil {
  81. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  82. }
  83. return repos, nil
  84. }
  85. if r.GCPIntegrationID != 0 {
  86. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
  87. if strings.Contains(r.URL, "pkg.dev") {
  88. return r.listGARRepositories(ctx, repo)
  89. }
  90. repos, err := r.listGCRRepositories(repo)
  91. if err != nil {
  92. return nil, telemetry.Error(ctx, span, err, "error listing gcr repositories")
  93. }
  94. return repos, nil
  95. }
  96. if r.DOIntegrationID != 0 {
  97. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "do"})
  98. repos, err := r.listDOCRRepositories(repo, conf.DOConf)
  99. if err != nil {
  100. return nil, telemetry.Error(ctx, span, err, "error listing docr repositories")
  101. }
  102. return repos, nil
  103. }
  104. if r.AzureIntegrationID != 0 {
  105. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "azure"})
  106. repos, err := r.listACRRepositories(ctx, repo)
  107. if err != nil {
  108. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  109. }
  110. return repos, nil
  111. }
  112. if r.BasicIntegrationID != 0 {
  113. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "basic"})
  114. repos, err := r.listPrivateRegistryRepositories(repo)
  115. if err != nil {
  116. return nil, telemetry.Error(ctx, span, err, "error listing private repositories")
  117. }
  118. return repos, nil
  119. }
  120. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  121. if err != nil {
  122. return nil, telemetry.Error(ctx, span, err, "error getting project for repository")
  123. }
  124. if project.CapiProvisionerEnabled {
  125. if strings.Contains(r.URL, ".azurecr.") {
  126. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "capi-azure"})
  127. creds, err := conf.Repo.AzureIntegration().ListAzureIntegrationsByProjectID(r.ProjectID)
  128. if err != nil {
  129. return nil, telemetry.Error(ctx, span, err, "error getting azure credentials for capi cluster")
  130. }
  131. if len(creds) == 0 {
  132. return nil, telemetry.Error(ctx, span, err, "no azure credentials for capi cluster")
  133. }
  134. r.AzureIntegrationID = creds[0].ID
  135. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "azure-integration-id", Value: r.AzureIntegrationID})
  136. repos, err := r.listACRRepositories(ctx, repo)
  137. if err != nil {
  138. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  139. }
  140. return repos, nil
  141. } else {
  142. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "capi-aws"})
  143. uri := strings.TrimPrefix(r.URL, "https://")
  144. splits := strings.Split(uri, ".")
  145. if len(splits) < 4 {
  146. return nil, telemetry.Error(ctx, span, nil, "uri does not have enough splits")
  147. }
  148. accountID := splits[0]
  149. region := splits[3]
  150. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  151. ProjectId: int64(r.ProjectID),
  152. AwsAccountId: accountID,
  153. })
  154. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  155. if err != nil {
  156. return nil, telemetry.Error(ctx, span, err, "error getting capi credentials for registry")
  157. }
  158. aws := &ints.AWSIntegration{
  159. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  160. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  161. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  162. AWSRegion: region,
  163. }
  164. repos, err := r.listECRRepositories(aws)
  165. if err != nil {
  166. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  167. }
  168. return repos, nil
  169. }
  170. }
  171. return nil, telemetry.Error(ctx, span, nil, "error listing repositories")
  172. }
  173. type gcrJWT struct {
  174. AccessToken string `json:"token"`
  175. ExpiresInSec int `json:"expires_in"`
  176. }
  177. type gcrErr struct {
  178. Code string `json:"code"`
  179. Message string `json:"message"`
  180. }
  181. type gcrRepositoryResp struct {
  182. Repositories []string `json:"repositories"`
  183. Errors []gcrErr `json:"errors"`
  184. }
  185. func (r *Registry) GetGCRToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  186. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  187. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  188. r.ProjectID,
  189. r.GCPIntegrationID,
  190. )
  191. if err != nil {
  192. return nil, err
  193. }
  194. // get oauth2 access token
  195. return gcp.GetBearerToken(
  196. ctx,
  197. getTokenCache,
  198. r.setTokenCacheFunc(ctx, repo),
  199. "https://www.googleapis.com/auth/devstorage.read_write",
  200. )
  201. }
  202. func (r *Registry) listGCRRepositories(
  203. repo repository.Repository,
  204. ) ([]*ptypes.RegistryRepository, error) {
  205. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  206. r.ProjectID,
  207. r.GCPIntegrationID,
  208. )
  209. if err != nil {
  210. return nil, err
  211. }
  212. // Just use service account key to authenticate, since scopes may not be in place
  213. // for oauth. This also prevents us from making more requests.
  214. client := &http.Client{}
  215. regURL := r.URL
  216. if !strings.HasPrefix(regURL, "http") {
  217. regURL = fmt.Sprintf("https://%s", regURL)
  218. }
  219. regURLParsed, err := url.Parse(regURL)
  220. regHostname := "gcr.io"
  221. if err == nil {
  222. regHostname = regURLParsed.Host
  223. }
  224. req, err := http.NewRequest(
  225. "GET",
  226. fmt.Sprintf("https://%s/v2/_catalog", regHostname),
  227. nil,
  228. )
  229. if err != nil {
  230. return nil, err
  231. }
  232. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  233. resp, err := client.Do(req)
  234. if err != nil {
  235. return nil, err
  236. }
  237. gcrResp := gcrRepositoryResp{}
  238. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  239. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  240. }
  241. if len(gcrResp.Errors) > 0 {
  242. errMsg := ""
  243. for _, gcrErr := range gcrResp.Errors {
  244. errMsg += fmt.Sprintf(": Code %s, message %s", gcrErr.Code, gcrErr.Message)
  245. }
  246. return nil, fmt.Errorf(errMsg)
  247. }
  248. res := make([]*ptypes.RegistryRepository, 0)
  249. parsedURL, err := url.Parse("https://" + r.URL)
  250. if err != nil {
  251. return nil, err
  252. }
  253. for _, repo := range gcrResp.Repositories {
  254. res = append(res, &ptypes.RegistryRepository{
  255. Name: repo,
  256. URI: parsedURL.Host + "/" + repo,
  257. })
  258. }
  259. return res, nil
  260. }
  261. func (r *Registry) GetGARToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  262. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  263. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  264. r.ProjectID,
  265. r.GCPIntegrationID,
  266. )
  267. if err != nil {
  268. return nil, err
  269. }
  270. // get oauth2 access token
  271. return gcp.GetBearerToken(
  272. ctx,
  273. getTokenCache,
  274. r.setTokenCacheFunc(ctx, repo),
  275. "https://www.googleapis.com/auth/cloud-platform",
  276. )
  277. }
  278. type garTokenSource struct {
  279. // ctx is only passed in here as the oauth2.Token() doesnt support contexts
  280. ctx context.Context
  281. reg *Registry
  282. repo repository.Repository
  283. }
  284. func (source *garTokenSource) Token() (*oauth2.Token, error) {
  285. return source.reg.GetGARToken(source.ctx, source.repo)
  286. }
  287. // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
  288. // where a repository is a single image. This function returns the list of fully qualified names
  289. // of GAR images including their repository names.
  290. func (r *Registry) listGARRepositories(
  291. ctx context.Context,
  292. repo repository.Repository,
  293. ) ([]*ptypes.RegistryRepository, error) {
  294. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  295. r.ProjectID,
  296. r.GCPIntegrationID,
  297. )
  298. if err != nil {
  299. return nil, err
  300. }
  301. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  302. reg: r,
  303. repo: repo,
  304. ctx: ctx,
  305. }), option.WithScopes("roles/artifactregistry.reader"))
  306. if err != nil {
  307. return nil, err
  308. }
  309. var repoNames []string
  310. nextToken := ""
  311. parsedURL, err := url.Parse("https://" + r.URL)
  312. if err != nil {
  313. return nil, err
  314. }
  315. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  316. for {
  317. it := client.ListRepositories(context.Background(), &artifactregistrypb.ListRepositoriesRequest{
  318. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  319. PageSize: 1000,
  320. PageToken: nextToken,
  321. })
  322. for {
  323. resp, err := it.Next()
  324. if err == iterator.Done {
  325. break
  326. } else if err != nil {
  327. return nil, err
  328. }
  329. if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
  330. repoSlice := strings.Split(resp.GetName(), "/")
  331. repoName := repoSlice[len(repoSlice)-1]
  332. repoNames = append(repoNames, repoName)
  333. }
  334. }
  335. if it.PageInfo().Token == "" {
  336. break
  337. }
  338. nextToken = it.PageInfo().Token
  339. }
  340. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  341. reg: r,
  342. repo: repo,
  343. ctx: ctx,
  344. }), option.WithScopes("roles/artifactregistry.reader"))
  345. if err != nil {
  346. return nil, err
  347. }
  348. nextToken = ""
  349. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  350. var (
  351. wg sync.WaitGroup
  352. resMap sync.Map
  353. )
  354. for _, repoName := range repoNames {
  355. wg.Add(1)
  356. go func(repoName string) {
  357. defer wg.Done()
  358. for {
  359. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  360. gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
  361. if err != nil {
  362. // FIXME: we should report this error using a channel
  363. return
  364. }
  365. for _, image := range resp.DockerImages {
  366. named, err := reference.ParseNamed(image.Uri)
  367. if err != nil {
  368. // let us skip this image becaue it has a malformed URI coming from the GCP API
  369. continue
  370. }
  371. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  372. resMap.Store(named.Name(), &ptypes.RegistryRepository{
  373. Name: repoName,
  374. URI: named.Name(),
  375. CreatedAt: uploadTime,
  376. })
  377. }
  378. if resp.NextPageToken == "" {
  379. break
  380. }
  381. nextToken = resp.NextPageToken
  382. }
  383. }(repoName)
  384. }
  385. wg.Wait()
  386. var res []*ptypes.RegistryRepository
  387. resMap.Range(func(_, value any) bool {
  388. res = append(res, value.(*ptypes.RegistryRepository))
  389. return true
  390. })
  391. return res, nil
  392. }
  393. func (r *Registry) listECRRepositories(aws *ints.AWSIntegration) ([]*ptypes.RegistryRepository, error) {
  394. sess, err := aws.GetSession()
  395. if err != nil {
  396. return nil, err
  397. }
  398. svc := ecr.New(sess)
  399. resp, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{})
  400. if err != nil {
  401. return nil, err
  402. }
  403. res := make([]*ptypes.RegistryRepository, 0)
  404. for _, repo := range resp.Repositories {
  405. res = append(res, &ptypes.RegistryRepository{
  406. Name: *repo.RepositoryName,
  407. CreatedAt: *repo.CreatedAt,
  408. URI: *repo.RepositoryUri,
  409. })
  410. }
  411. return res, nil
  412. }
  413. type acrRepositoryResp struct {
  414. Repositories []string `json:"repositories"`
  415. }
  416. func (r *Registry) listACRRepositories(ctx context.Context, repo repository.Repository) ([]*ptypes.RegistryRepository, error) {
  417. ctx, span := telemetry.NewSpan(ctx, "list-acr-repositories")
  418. defer span.End()
  419. telemetry.WithAttributes(span,
  420. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  421. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  422. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  423. )
  424. az, err := repo.AzureIntegration().ReadAzureIntegration(
  425. r.ProjectID,
  426. r.AzureIntegrationID,
  427. )
  428. if err != nil {
  429. return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
  430. }
  431. client := &http.Client{}
  432. acrURL := r.URL
  433. if !strings.Contains(acrURL, "http") {
  434. acrURL = fmt.Sprintf("https://%s", acrURL)
  435. }
  436. req, err := http.NewRequest(
  437. "GET",
  438. fmt.Sprintf("%s/v2/_catalog", acrURL),
  439. nil,
  440. )
  441. if err != nil {
  442. return nil, telemetry.Error(ctx, span, err, "error getting http request")
  443. }
  444. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  445. resp, err := client.Do(req)
  446. if err != nil {
  447. return nil, telemetry.Error(ctx, span, err, "error making http call")
  448. }
  449. acrResp := acrRepositoryResp{}
  450. if err := json.NewDecoder(resp.Body).Decode(&acrResp); err != nil {
  451. return nil, telemetry.Error(ctx, span, err, "could not read Azure registry repository response")
  452. }
  453. res := make([]*ptypes.RegistryRepository, 0)
  454. for _, repo := range acrResp.Repositories {
  455. res = append(res, &ptypes.RegistryRepository{
  456. Name: repo,
  457. URI: strings.TrimPrefix(r.URL, "https://") + "/" + repo,
  458. })
  459. }
  460. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "repo-count", Value: len(acrResp.Repositories)})
  461. return res, nil
  462. }
  463. // Returns the username/password pair for the registry
  464. func (r *Registry) GetACRCredentials(repo repository.Repository) (string, string, error) {
  465. az, err := repo.AzureIntegration().ReadAzureIntegration(
  466. r.ProjectID,
  467. r.AzureIntegrationID,
  468. )
  469. if err != nil {
  470. return "", "", err
  471. }
  472. // if the passwords and name aren't set, generate them
  473. if az.ACRTokenName == "" || len(az.ACRPassword1) == 0 {
  474. az.ACRTokenName = "porter-acr-token"
  475. // create an acr repo token
  476. cred, err := azidentity.NewClientSecretCredential(az.AzureTenantID, az.AzureClientID, string(az.ServicePrincipalSecret), nil)
  477. if err != nil {
  478. return "", "", err
  479. }
  480. scopeMapsClient, err := armcontainerregistry.NewScopeMapsClient(az.AzureSubscriptionID, cred, nil)
  481. if err != nil {
  482. return "", "", err
  483. }
  484. smRes, err := scopeMapsClient.Get(
  485. context.Background(),
  486. az.ACRResourceGroupName,
  487. az.ACRName,
  488. "_repositories_admin",
  489. nil,
  490. )
  491. if err != nil {
  492. return "", "", err
  493. }
  494. tokensClient, err := armcontainerregistry.NewTokensClient(az.AzureSubscriptionID, cred, nil)
  495. if err != nil {
  496. return "", "", err
  497. }
  498. pollerResp, err := tokensClient.BeginCreate(
  499. context.Background(),
  500. az.ACRResourceGroupName,
  501. az.ACRName,
  502. "porter-acr-token",
  503. armcontainerregistry.Token{
  504. Properties: &armcontainerregistry.TokenProperties{
  505. ScopeMapID: smRes.ID,
  506. Status: to.Ptr(armcontainerregistry.TokenStatusEnabled),
  507. },
  508. },
  509. nil,
  510. )
  511. if err != nil {
  512. return "", "", err
  513. }
  514. tokResp, err := pollerResp.PollUntilDone(context.Background(), 2*time.Second)
  515. if err != nil {
  516. return "", "", err
  517. }
  518. registriesClient, err := armcontainerregistry.NewRegistriesClient(az.AzureSubscriptionID, cred, nil)
  519. if err != nil {
  520. return "", "", err
  521. }
  522. poller, err := registriesClient.BeginGenerateCredentials(
  523. context.Background(),
  524. az.ACRResourceGroupName,
  525. az.ACRName,
  526. armcontainerregistry.GenerateCredentialsParameters{
  527. TokenID: tokResp.ID,
  528. },
  529. &armcontainerregistry.RegistriesClientBeginGenerateCredentialsOptions{ResumeToken: ""})
  530. if err != nil {
  531. return "", "", err
  532. }
  533. genCredentialsResp, err := poller.PollUntilDone(context.Background(), 2*time.Second)
  534. if err != nil {
  535. return "", "", err
  536. }
  537. for i, tokPassword := range genCredentialsResp.Passwords {
  538. if i == 0 {
  539. az.ACRPassword1 = []byte(*tokPassword.Value)
  540. } else if i == 1 {
  541. az.ACRPassword2 = []byte(*tokPassword.Value)
  542. }
  543. }
  544. // update the az integration
  545. az, err = repo.AzureIntegration().OverwriteAzureIntegration(
  546. az,
  547. )
  548. if err != nil {
  549. return "", "", err
  550. }
  551. }
  552. return az.ACRTokenName, string(az.ACRPassword1), nil
  553. }
  554. func (r *Registry) listDOCRRepositories(
  555. repo repository.Repository,
  556. doAuth *oauth2.Config,
  557. ) ([]*ptypes.RegistryRepository, error) {
  558. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  559. r.ProjectID,
  560. r.DOIntegrationID,
  561. )
  562. if err != nil {
  563. return nil, err
  564. }
  565. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  566. if err != nil {
  567. return nil, err
  568. }
  569. client := godo.NewFromToken(tok)
  570. urlArr := strings.Split(r.URL, "/")
  571. if len(urlArr) != 2 {
  572. return nil, fmt.Errorf("invalid digital ocean registry url")
  573. }
  574. name := urlArr[1]
  575. repos, _, err := client.Registry.ListRepositories(context.TODO(), name, &godo.ListOptions{})
  576. if err != nil {
  577. return nil, err
  578. }
  579. res := make([]*ptypes.RegistryRepository, 0)
  580. for _, repo := range repos {
  581. res = append(res, &ptypes.RegistryRepository{
  582. Name: repo.Name,
  583. URI: r.URL + "/" + repo.Name,
  584. })
  585. }
  586. return res, nil
  587. }
  588. func (r *Registry) listPrivateRegistryRepositories(
  589. repo repository.Repository,
  590. ) ([]*ptypes.RegistryRepository, error) {
  591. // handle dockerhub different, as it doesn't implement the docker registry http api
  592. if strings.Contains(r.URL, "docker.io") {
  593. // in this case, we just return the single dockerhub repository that's linked
  594. res := make([]*ptypes.RegistryRepository, 0)
  595. res = append(res, &ptypes.RegistryRepository{
  596. Name: strings.Split(r.URL, "docker.io/")[1],
  597. URI: r.URL,
  598. })
  599. return res, nil
  600. }
  601. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  602. r.ProjectID,
  603. r.BasicIntegrationID,
  604. )
  605. if err != nil {
  606. return nil, err
  607. }
  608. // Just use service account key to authenticate, since scopes may not be in place
  609. // for oauth. This also prevents us from making more requests.
  610. client := &http.Client{}
  611. // get the host and scheme to make the request
  612. parsedURL, err := url.Parse(r.URL)
  613. req, err := http.NewRequest(
  614. "GET",
  615. fmt.Sprintf("%s://%s/v2/_catalog", parsedURL.Scheme, parsedURL.Host),
  616. nil,
  617. )
  618. if err != nil {
  619. return nil, err
  620. }
  621. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  622. resp, err := client.Do(req)
  623. if err != nil {
  624. return nil, err
  625. }
  626. // if the status code is 404, fallback to the Docker Hub implementation
  627. if resp.StatusCode == 404 {
  628. req, err := http.NewRequest(
  629. "GET",
  630. fmt.Sprintf("%s/", r.URL),
  631. nil,
  632. )
  633. if err != nil {
  634. return nil, err
  635. }
  636. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  637. resp, err = client.Do(req)
  638. if err != nil {
  639. return nil, err
  640. }
  641. }
  642. gcrResp := gcrRepositoryResp{}
  643. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  644. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  645. }
  646. res := make([]*ptypes.RegistryRepository, 0)
  647. if err != nil {
  648. return nil, err
  649. }
  650. for _, repo := range gcrResp.Repositories {
  651. res = append(res, &ptypes.RegistryRepository{
  652. Name: repo,
  653. URI: parsedURL.Host + "/" + repo,
  654. })
  655. }
  656. return res, nil
  657. }
  658. func (r *Registry) getTokenCacheFunc(
  659. ctx context.Context,
  660. repo repository.Repository,
  661. ) ints.GetTokenCacheFunc {
  662. return func(ctx context.Context) (tok *ints.TokenCache, err error) {
  663. reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
  664. if err != nil {
  665. return nil, err
  666. }
  667. return &reg.TokenCache.TokenCache, nil
  668. }
  669. }
  670. func (r *Registry) setTokenCacheFunc(
  671. ctx context.Context,
  672. repo repository.Repository,
  673. ) ints.SetTokenCacheFunc {
  674. return func(ctx context.Context, token string, expiry time.Time) error {
  675. _, err := repo.Registry().UpdateRegistryTokenCache(
  676. &ints.RegTokenCache{
  677. TokenCache: ints.TokenCache{
  678. Token: []byte(token),
  679. Expiry: expiry,
  680. },
  681. RegistryID: r.ID,
  682. },
  683. )
  684. return err
  685. }
  686. }
  687. // CreateRepository creates a repository for a registry, if needed
  688. // (currently only required for ECR)
  689. func (r *Registry) CreateRepository(
  690. ctx context.Context,
  691. conf *config.Config,
  692. name string,
  693. ) error {
  694. // if aws, create repository
  695. if r.AWSIntegrationID != 0 {
  696. aws, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
  697. r.ProjectID,
  698. r.AWSIntegrationID,
  699. )
  700. if err != nil {
  701. return err
  702. }
  703. return r.createECRRepository(aws, name)
  704. } else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
  705. return r.createGARRepository(ctx, conf.Repo, name)
  706. }
  707. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  708. if err != nil {
  709. return fmt.Errorf("error getting project for repository: %w", err)
  710. }
  711. if project.CapiProvisionerEnabled {
  712. // no need to create repository if pushing to ACR
  713. if strings.Contains(r.URL, ".azurecr.") {
  714. return nil
  715. }
  716. uri := strings.TrimPrefix(r.URL, "https://")
  717. splits := strings.Split(uri, ".")
  718. accountID := splits[0]
  719. region := splits[3]
  720. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  721. ProjectId: int64(r.ProjectID),
  722. AwsAccountId: accountID,
  723. })
  724. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  725. if err != nil {
  726. return fmt.Errorf("error getting capi credentials for repository: %w", err)
  727. }
  728. aws := &ints.AWSIntegration{
  729. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  730. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  731. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  732. AWSRegion: region,
  733. }
  734. return r.createECRRepository(aws, name)
  735. }
  736. // otherwise, no-op
  737. return nil
  738. }
  739. func (r *Registry) createECRRepository(
  740. aws *ints.AWSIntegration,
  741. name string,
  742. ) error {
  743. sess, err := aws.GetSession()
  744. if err != nil {
  745. return err
  746. }
  747. svc := ecr.New(sess)
  748. // determine if repository already exists
  749. _, err = svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
  750. RepositoryNames: []*string{&name},
  751. })
  752. // if the repository was not found, create it
  753. if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ecr.ErrCodeRepositoryNotFoundException {
  754. _, err = svc.CreateRepository(&ecr.CreateRepositoryInput{
  755. RepositoryName: &name,
  756. })
  757. return err
  758. } else if err != nil {
  759. return err
  760. }
  761. return nil
  762. }
  763. func (r *Registry) createGARRepository(
  764. ctx context.Context,
  765. repo repository.Repository,
  766. name string,
  767. ) error {
  768. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  769. r.ProjectID,
  770. r.GCPIntegrationID,
  771. )
  772. if err != nil {
  773. return err
  774. }
  775. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  776. reg: r,
  777. repo: repo,
  778. ctx: ctx,
  779. }), option.WithScopes("roles/artifactregistry.admin"))
  780. if err != nil {
  781. return err
  782. }
  783. defer client.Close()
  784. parsedURL, err := url.Parse("https://" + r.URL)
  785. if err != nil {
  786. return err
  787. }
  788. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  789. _, err = client.GetRepository(context.Background(), &artifactregistrypb.GetRepositoryRequest{
  790. Name: fmt.Sprintf("projects/%s/locations/%s/repositories/%s", gcpInt.GCPProjectID, location, name),
  791. })
  792. if err != nil && strings.Contains(err.Error(), "not found") {
  793. // create a new repository
  794. _, err := client.CreateRepository(context.Background(), &artifactregistrypb.CreateRepositoryRequest{
  795. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  796. RepositoryId: name,
  797. Repository: &artifactregistrypb.Repository{
  798. Format: artifactregistrypb.Repository_DOCKER,
  799. },
  800. })
  801. if err != nil {
  802. return err
  803. }
  804. } else if err != nil {
  805. return err
  806. }
  807. return nil
  808. }
  809. // ListImages lists the images for an image repository
  810. func (r *Registry) ListImages(
  811. ctx context.Context,
  812. repoName string,
  813. repo repository.Repository,
  814. conf *config.Config,
  815. ) ([]*ptypes.Image, error) {
  816. // switch on the auth mechanism to get a token
  817. if r.AWSIntegrationID != 0 {
  818. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  819. r.ProjectID,
  820. r.AWSIntegrationID,
  821. )
  822. if err != nil {
  823. return nil, err
  824. }
  825. return r.listECRImages(aws, repoName, repo)
  826. }
  827. if r.AzureIntegrationID != 0 {
  828. return r.listACRImages(repoName, repo)
  829. }
  830. if r.GCPIntegrationID != 0 {
  831. if strings.Contains(r.URL, "pkg.dev") {
  832. return r.listGARImages(ctx, repoName, repo)
  833. }
  834. return r.listGCRImages(repoName, repo)
  835. }
  836. if r.DOIntegrationID != 0 {
  837. return r.listDOCRImages(repoName, repo, conf.DOConf)
  838. }
  839. if r.BasicIntegrationID != 0 {
  840. return r.listPrivateRegistryImages(repoName, repo)
  841. }
  842. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  843. if err != nil {
  844. return nil, fmt.Errorf("error getting project for repository: %w", err)
  845. }
  846. if project.CapiProvisionerEnabled {
  847. uri := strings.TrimPrefix(r.URL, "https://")
  848. splits := strings.Split(uri, ".")
  849. accountID := splits[0]
  850. region := splits[3]
  851. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  852. ProjectId: int64(r.ProjectID),
  853. AwsAccountId: accountID,
  854. })
  855. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  856. if err != nil {
  857. return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
  858. }
  859. aws := &ints.AWSIntegration{
  860. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  861. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  862. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  863. AWSRegion: region,
  864. }
  865. return r.listECRImages(aws, repoName, repo)
  866. }
  867. return nil, fmt.Errorf("error listing images")
  868. }
  869. func (r *Registry) GetECRPaginatedImages(
  870. repoName string,
  871. repo repository.Repository,
  872. maxResults int64,
  873. nextToken *string,
  874. ) ([]*ptypes.Image, *string, error) {
  875. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  876. r.ProjectID,
  877. r.AWSIntegrationID,
  878. )
  879. if err != nil {
  880. return nil, nil, err
  881. }
  882. sess, err := aws.GetSession()
  883. if err != nil {
  884. return nil, nil, err
  885. }
  886. svc := ecr.New(sess)
  887. resp, err := svc.ListImages(&ecr.ListImagesInput{
  888. RepositoryName: &repoName,
  889. MaxResults: &maxResults,
  890. NextToken: nextToken,
  891. })
  892. if err != nil {
  893. return nil, nil, err
  894. }
  895. if len(resp.ImageIds) == 0 {
  896. return []*ptypes.Image{}, nil, nil
  897. }
  898. imageIDLen := len(resp.ImageIds)
  899. imageDetails := make([]*ecr.ImageDetail, 0)
  900. imageIDMap := make(map[string]bool)
  901. for _, id := range resp.ImageIds {
  902. if id != nil && id.ImageTag != nil {
  903. imageIDMap[*id.ImageTag] = true
  904. }
  905. }
  906. var wg sync.WaitGroup
  907. var mu sync.Mutex
  908. // AWS API expects the length of imageIDs to be at max 100 at a time
  909. for start := 0; start < imageIDLen; start += 100 {
  910. end := start + 100
  911. if end > imageIDLen {
  912. end = imageIDLen
  913. }
  914. wg.Add(1)
  915. go func(start, end int) {
  916. defer wg.Done()
  917. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  918. RepositoryName: &repoName,
  919. ImageIds: resp.ImageIds[start:end],
  920. })
  921. if err != nil {
  922. return
  923. }
  924. mu.Lock()
  925. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  926. mu.Unlock()
  927. }(start, end)
  928. }
  929. wg.Wait()
  930. res := make([]*ptypes.Image, 0)
  931. imageInfoMap := make(map[string]*ptypes.Image)
  932. for _, img := range imageDetails {
  933. for _, tag := range img.ImageTags {
  934. newImage := &ptypes.Image{
  935. Digest: *img.ImageDigest,
  936. Tag: *tag,
  937. RepositoryName: repoName,
  938. PushedAt: img.ImagePushedAt,
  939. }
  940. if _, ok := imageIDMap[*tag]; ok {
  941. if _, ok := imageInfoMap[*tag]; !ok {
  942. imageInfoMap[*tag] = newImage
  943. }
  944. }
  945. if len(imageInfoMap) == int(maxResults) {
  946. break
  947. }
  948. }
  949. if len(imageInfoMap) == int(maxResults) {
  950. break
  951. }
  952. }
  953. for _, v := range imageInfoMap {
  954. res = append(res, v)
  955. }
  956. sort.Slice(res, func(i, j int) bool {
  957. if res[i].PushedAt == nil || res[j].PushedAt == nil {
  958. return false
  959. }
  960. return res[i].PushedAt.After(*res[j].PushedAt)
  961. })
  962. return res, resp.NextToken, nil
  963. }
  964. func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  965. sess, err := aws.GetSession()
  966. if err != nil {
  967. return nil, err
  968. }
  969. svc := ecr.New(sess)
  970. maxResults := int64(1000)
  971. var imageIDs []*ecr.ImageIdentifier
  972. resp, err := svc.ListImages(&ecr.ListImagesInput{
  973. RepositoryName: &repoName,
  974. MaxResults: &maxResults,
  975. })
  976. if err != nil {
  977. return nil, err
  978. }
  979. if len(resp.ImageIds) == 0 {
  980. return []*ptypes.Image{}, nil
  981. }
  982. imageIDs = append(imageIDs, resp.ImageIds...)
  983. nextToken := resp.NextToken
  984. for nextToken != nil {
  985. resp, err := svc.ListImages(&ecr.ListImagesInput{
  986. RepositoryName: &repoName,
  987. MaxResults: &maxResults,
  988. NextToken: nextToken,
  989. })
  990. if err != nil {
  991. return nil, err
  992. }
  993. imageIDs = append(imageIDs, resp.ImageIds...)
  994. nextToken = resp.NextToken
  995. }
  996. imageIDLen := len(imageIDs)
  997. imageDetails := make([]*ecr.ImageDetail, 0)
  998. var wg sync.WaitGroup
  999. var mu sync.Mutex
  1000. // AWS API expects the length of imageIDs to be at max 100 at a time
  1001. for start := 0; start < imageIDLen; start += 100 {
  1002. end := start + 100
  1003. if end > imageIDLen {
  1004. end = imageIDLen
  1005. }
  1006. wg.Add(1)
  1007. go func(start, end int) {
  1008. defer wg.Done()
  1009. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  1010. RepositoryName: &repoName,
  1011. ImageIds: imageIDs[start:end],
  1012. })
  1013. if err != nil {
  1014. return
  1015. }
  1016. mu.Lock()
  1017. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  1018. mu.Unlock()
  1019. }(start, end)
  1020. }
  1021. wg.Wait()
  1022. res := make([]*ptypes.Image, 0)
  1023. imageInfoMap := make(map[string]*ptypes.Image)
  1024. for _, img := range imageDetails {
  1025. for _, tag := range img.ImageTags {
  1026. newImage := &ptypes.Image{
  1027. Digest: *img.ImageDigest,
  1028. Tag: *tag,
  1029. RepositoryName: repoName,
  1030. PushedAt: img.ImagePushedAt,
  1031. }
  1032. if _, ok := imageInfoMap[*tag]; !ok {
  1033. imageInfoMap[*tag] = newImage
  1034. }
  1035. }
  1036. }
  1037. for _, v := range imageInfoMap {
  1038. res = append(res, v)
  1039. }
  1040. return res, nil
  1041. }
  1042. func (r *Registry) listACRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1043. az, err := repo.AzureIntegration().ReadAzureIntegration(
  1044. r.ProjectID,
  1045. r.AzureIntegrationID,
  1046. )
  1047. if err != nil {
  1048. return nil, err
  1049. }
  1050. // use JWT token to request catalog
  1051. client := &http.Client{}
  1052. req, err := http.NewRequest(
  1053. "GET",
  1054. fmt.Sprintf("%s/v2/%s/tags/list", r.URL, repoName),
  1055. nil,
  1056. )
  1057. if err != nil {
  1058. return nil, err
  1059. }
  1060. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  1061. resp, err := client.Do(req)
  1062. if err != nil {
  1063. return nil, err
  1064. }
  1065. gcrResp := gcrImageResp{}
  1066. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1067. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1068. }
  1069. res := make([]*ptypes.Image, 0)
  1070. for _, tag := range gcrResp.Tags {
  1071. res = append(res, &ptypes.Image{
  1072. RepositoryName: strings.TrimPrefix(repoName, "https://"),
  1073. Tag: tag,
  1074. })
  1075. }
  1076. return res, nil
  1077. }
  1078. type gcrImageResp struct {
  1079. Tags []string `json:"tags"`
  1080. }
  1081. func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1082. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1083. r.ProjectID,
  1084. r.GCPIntegrationID,
  1085. )
  1086. if err != nil {
  1087. return nil, err
  1088. }
  1089. // use JWT token to request catalog
  1090. client := &http.Client{}
  1091. parsedURL, err := url.Parse("https://" + r.URL)
  1092. if err != nil {
  1093. return nil, err
  1094. }
  1095. trimmedPath := strings.Trim(parsedURL.Path, "/")
  1096. req, err := http.NewRequest(
  1097. "GET",
  1098. fmt.Sprintf("https://%s/v2/%s/%s/tags/list", parsedURL.Host, trimmedPath, repoName),
  1099. nil,
  1100. )
  1101. if err != nil {
  1102. return nil, err
  1103. }
  1104. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  1105. resp, err := client.Do(req)
  1106. if err != nil {
  1107. return nil, err
  1108. }
  1109. gcrResp := gcrImageResp{}
  1110. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1111. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1112. }
  1113. res := make([]*ptypes.Image, 0)
  1114. for _, tag := range gcrResp.Tags {
  1115. res = append(res, &ptypes.Image{
  1116. RepositoryName: repoName,
  1117. Tag: tag,
  1118. })
  1119. }
  1120. return res, nil
  1121. }
  1122. func (r *Registry) listGARImages(ctx context.Context, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1123. repoImageSlice := strings.Split(repoName, "/")
  1124. if len(repoImageSlice) != 2 {
  1125. return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
  1126. }
  1127. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  1128. r.ProjectID,
  1129. r.GCPIntegrationID,
  1130. )
  1131. if err != nil {
  1132. return nil, err
  1133. }
  1134. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  1135. reg: r,
  1136. repo: repo,
  1137. ctx: ctx,
  1138. }), option.WithScopes("roles/artifactregistry.reader"))
  1139. if err != nil {
  1140. return nil, err
  1141. }
  1142. var res []*ptypes.Image
  1143. parsedURL, err := url.Parse("https://" + r.URL)
  1144. if err != nil {
  1145. return nil, err
  1146. }
  1147. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  1148. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  1149. nextToken := ""
  1150. for {
  1151. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  1152. gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
  1153. if err != nil {
  1154. return nil, err
  1155. }
  1156. for _, image := range resp.DockerImages {
  1157. named, err := reference.ParseNamed(image.Uri)
  1158. if err != nil {
  1159. continue
  1160. }
  1161. paths := strings.Split(reference.Path(named), "/")
  1162. imageName := paths[len(paths)-1]
  1163. if imageName == repoImageSlice[1] {
  1164. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  1165. for _, tag := range image.Tags {
  1166. res = append(res, &ptypes.Image{
  1167. RepositoryName: repoName,
  1168. Tag: tag,
  1169. PushedAt: &uploadTime,
  1170. Digest: strings.Split(image.Uri, "@")[1],
  1171. })
  1172. }
  1173. }
  1174. }
  1175. if resp.NextPageToken == "" {
  1176. break
  1177. }
  1178. nextToken = resp.NextPageToken
  1179. }
  1180. return res, nil
  1181. }
  1182. func (r *Registry) listDOCRImages(
  1183. repoName string,
  1184. repo repository.Repository,
  1185. doAuth *oauth2.Config,
  1186. ) ([]*ptypes.Image, error) {
  1187. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1188. r.ProjectID,
  1189. r.DOIntegrationID,
  1190. )
  1191. if err != nil {
  1192. return nil, err
  1193. }
  1194. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1195. if err != nil {
  1196. return nil, err
  1197. }
  1198. client := godo.NewFromToken(tok)
  1199. urlArr := strings.Split(r.URL, "/")
  1200. if len(urlArr) != 2 {
  1201. return nil, fmt.Errorf("invalid digital ocean registry url")
  1202. }
  1203. name := urlArr[1]
  1204. var tags []*godo.RepositoryTag
  1205. opt := &godo.ListOptions{
  1206. PerPage: 200,
  1207. }
  1208. for {
  1209. nextTags, resp, err := client.Registry.ListRepositoryTags(context.TODO(), name, repoName, opt)
  1210. if err != nil {
  1211. return nil, err
  1212. }
  1213. tags = append(tags, nextTags...)
  1214. if resp.Links == nil || resp.Links.IsLastPage() {
  1215. break
  1216. }
  1217. page, err := resp.Links.CurrentPage()
  1218. if err != nil {
  1219. return nil, err
  1220. }
  1221. opt.Page = page + 1
  1222. }
  1223. res := make([]*ptypes.Image, 0)
  1224. for _, tag := range tags {
  1225. res = append(res, &ptypes.Image{
  1226. RepositoryName: repoName,
  1227. Tag: tag.Tag,
  1228. })
  1229. }
  1230. return res, nil
  1231. }
  1232. func (r *Registry) listPrivateRegistryImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1233. // handle dockerhub different, as it doesn't implement the docker registry http api
  1234. if strings.Contains(r.URL, "docker.io") {
  1235. return r.listDockerHubImages(repoName, repo)
  1236. }
  1237. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1238. r.ProjectID,
  1239. r.BasicIntegrationID,
  1240. )
  1241. if err != nil {
  1242. return nil, err
  1243. }
  1244. // Just use service account key to authenticate, since scopes may not be in place
  1245. // for oauth. This also prevents us from making more requests.
  1246. client := &http.Client{}
  1247. // get the host and scheme to make the request
  1248. parsedURL, err := url.Parse(r.URL)
  1249. req, err := http.NewRequest(
  1250. "GET",
  1251. fmt.Sprintf("%s://%s/v2/%s/tags/list", parsedURL.Scheme, parsedURL.Host, repoName),
  1252. nil,
  1253. )
  1254. if err != nil {
  1255. return nil, err
  1256. }
  1257. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  1258. resp, err := client.Do(req)
  1259. if err != nil {
  1260. return nil, err
  1261. }
  1262. gcrResp := gcrImageResp{}
  1263. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1264. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1265. }
  1266. res := make([]*ptypes.Image, 0)
  1267. for _, tag := range gcrResp.Tags {
  1268. res = append(res, &ptypes.Image{
  1269. RepositoryName: repoName,
  1270. Tag: tag,
  1271. })
  1272. }
  1273. return res, nil
  1274. }
  1275. type dockerHubImageResult struct {
  1276. Name string `json:"name"`
  1277. }
  1278. type dockerHubImageResp struct {
  1279. Results []dockerHubImageResult `json:"results"`
  1280. }
  1281. type dockerHubLoginReq struct {
  1282. Username string `json:"username"`
  1283. Password string `json:"password"`
  1284. }
  1285. type dockerHubLoginResp struct {
  1286. Token string `json:"token"`
  1287. }
  1288. func (r *Registry) listDockerHubImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1289. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1290. r.ProjectID,
  1291. r.BasicIntegrationID,
  1292. )
  1293. if err != nil {
  1294. return nil, err
  1295. }
  1296. client := &http.Client{}
  1297. // first, make a request for the access token
  1298. data, err := json.Marshal(&dockerHubLoginReq{
  1299. Username: string(basic.Username),
  1300. Password: string(basic.Password),
  1301. })
  1302. if err != nil {
  1303. return nil, err
  1304. }
  1305. req, err := http.NewRequest(
  1306. "POST",
  1307. "https://hub.docker.com/v2/users/login",
  1308. strings.NewReader(string(data)),
  1309. )
  1310. if err != nil {
  1311. return nil, err
  1312. }
  1313. req.Header.Add("Content-Type", "application/json")
  1314. resp, err := client.Do(req)
  1315. if err != nil {
  1316. return nil, err
  1317. }
  1318. tokenObj := dockerHubLoginResp{}
  1319. if err := json.NewDecoder(resp.Body).Decode(&tokenObj); err != nil {
  1320. return nil, fmt.Errorf("Could not decode Dockerhub token from response: %v", err)
  1321. }
  1322. req, err = http.NewRequest(
  1323. "GET",
  1324. fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags", strings.Split(r.URL, "docker.io/")[1]),
  1325. nil,
  1326. )
  1327. if err != nil {
  1328. return nil, err
  1329. }
  1330. req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenObj.Token))
  1331. resp, err = client.Do(req)
  1332. if err != nil {
  1333. return nil, err
  1334. }
  1335. imageResp := dockerHubImageResp{}
  1336. if err := json.NewDecoder(resp.Body).Decode(&imageResp); err != nil {
  1337. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1338. }
  1339. res := make([]*ptypes.Image, 0)
  1340. for _, result := range imageResp.Results {
  1341. res = append(res, &ptypes.Image{
  1342. RepositoryName: repoName,
  1343. Tag: result.Name,
  1344. })
  1345. }
  1346. return res, nil
  1347. }
  1348. // GetDockerConfigJSON returns a dockerconfigjson file contents with "auths"
  1349. // populated.
  1350. func (r *Registry) GetDockerConfigJSON(
  1351. repo repository.Repository,
  1352. doAuth *oauth2.Config, // only required if using DOCR
  1353. ) ([]byte, error) {
  1354. var conf *configfile.ConfigFile
  1355. var err error
  1356. // switch on the auth mechanism to get a token
  1357. if r.AWSIntegrationID != 0 {
  1358. conf, err = r.getECRDockerConfigFile(repo)
  1359. }
  1360. if r.GCPIntegrationID != 0 {
  1361. conf, err = r.getGCRDockerConfigFile(repo)
  1362. }
  1363. if r.DOIntegrationID != 0 {
  1364. conf, err = r.getDOCRDockerConfigFile(repo, doAuth)
  1365. }
  1366. if r.BasicIntegrationID != 0 {
  1367. conf, err = r.getPrivateRegistryDockerConfigFile(repo)
  1368. }
  1369. if r.AzureIntegrationID != 0 {
  1370. conf, err = r.getACRDockerConfigFile(repo)
  1371. }
  1372. if err != nil {
  1373. return nil, err
  1374. }
  1375. return json.Marshal(conf)
  1376. }
  1377. func (r *Registry) getECRDockerConfigFile(
  1378. repo repository.Repository,
  1379. ) (*configfile.ConfigFile, error) {
  1380. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  1381. r.ProjectID,
  1382. r.AWSIntegrationID,
  1383. )
  1384. if err != nil {
  1385. return nil, err
  1386. }
  1387. sess, err := aws.GetSession()
  1388. if err != nil {
  1389. return nil, err
  1390. }
  1391. ecrSvc := ecr.New(sess)
  1392. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  1393. if err != nil {
  1394. return nil, err
  1395. }
  1396. token := *output.AuthorizationData[0].AuthorizationToken
  1397. decodedToken, err := base64.StdEncoding.DecodeString(token)
  1398. if err != nil {
  1399. return nil, err
  1400. }
  1401. parts := strings.SplitN(string(decodedToken), ":", 2)
  1402. if len(parts) < 2 {
  1403. return nil, err
  1404. }
  1405. key := r.URL
  1406. if !strings.Contains(key, "http") {
  1407. key = "https://" + key
  1408. }
  1409. return &configfile.ConfigFile{
  1410. AuthConfigs: map[string]types.AuthConfig{
  1411. key: {
  1412. Username: parts[0],
  1413. Password: parts[1],
  1414. Auth: token,
  1415. },
  1416. },
  1417. }, nil
  1418. }
  1419. func (r *Registry) getGCRDockerConfigFile(
  1420. repo repository.Repository,
  1421. ) (*configfile.ConfigFile, error) {
  1422. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1423. r.ProjectID,
  1424. r.GCPIntegrationID,
  1425. )
  1426. if err != nil {
  1427. return nil, err
  1428. }
  1429. key := r.URL
  1430. if !strings.Contains(key, "http") {
  1431. key = "https://" + key
  1432. }
  1433. parsedURL, _ := url.Parse(key)
  1434. return &configfile.ConfigFile{
  1435. AuthConfigs: map[string]types.AuthConfig{
  1436. parsedURL.Host: {
  1437. Username: "_json_key",
  1438. Password: string(gcp.GCPKeyData),
  1439. Auth: generateAuthToken("_json_key", string(gcp.GCPKeyData)),
  1440. },
  1441. },
  1442. }, nil
  1443. }
  1444. func (r *Registry) getDOCRDockerConfigFile(
  1445. repo repository.Repository,
  1446. doAuth *oauth2.Config,
  1447. ) (*configfile.ConfigFile, error) {
  1448. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1449. r.ProjectID,
  1450. r.DOIntegrationID,
  1451. )
  1452. if err != nil {
  1453. return nil, err
  1454. }
  1455. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1456. if err != nil {
  1457. return nil, err
  1458. }
  1459. key := r.URL
  1460. if !strings.Contains(key, "http") {
  1461. key = "https://" + key
  1462. }
  1463. parsedURL, _ := url.Parse(key)
  1464. return &configfile.ConfigFile{
  1465. AuthConfigs: map[string]types.AuthConfig{
  1466. parsedURL.Host: {
  1467. Username: tok,
  1468. Password: tok,
  1469. Auth: generateAuthToken(tok, tok),
  1470. },
  1471. },
  1472. }, nil
  1473. }
  1474. func (r *Registry) getPrivateRegistryDockerConfigFile(
  1475. repo repository.Repository,
  1476. ) (*configfile.ConfigFile, error) {
  1477. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1478. r.ProjectID,
  1479. r.BasicIntegrationID,
  1480. )
  1481. if err != nil {
  1482. return nil, err
  1483. }
  1484. key := r.URL
  1485. if !strings.Contains(key, "http") {
  1486. key = "https://" + key
  1487. }
  1488. parsedURL, _ := url.Parse(key)
  1489. authConfigKey := parsedURL.Host
  1490. if strings.Contains(r.URL, "index.docker.io") {
  1491. authConfigKey = "https://index.docker.io/v1/"
  1492. }
  1493. return &configfile.ConfigFile{
  1494. AuthConfigs: map[string]types.AuthConfig{
  1495. authConfigKey: {
  1496. Username: string(basic.Username),
  1497. Password: string(basic.Password),
  1498. Auth: generateAuthToken(string(basic.Username), string(basic.Password)),
  1499. },
  1500. },
  1501. }, nil
  1502. }
  1503. func (r *Registry) getACRDockerConfigFile(
  1504. repo repository.Repository,
  1505. ) (*configfile.ConfigFile, error) {
  1506. username, pw, err := r.GetACRCredentials(repo)
  1507. if err != nil {
  1508. return nil, err
  1509. }
  1510. key := r.URL
  1511. if !strings.Contains(key, "http") {
  1512. key = "https://" + key
  1513. }
  1514. parsedURL, _ := url.Parse(key)
  1515. return &configfile.ConfigFile{
  1516. AuthConfigs: map[string]types.AuthConfig{
  1517. parsedURL.Host: {
  1518. Username: string(username),
  1519. Password: string(pw),
  1520. Auth: generateAuthToken(string(username), string(pw)),
  1521. },
  1522. },
  1523. }, nil
  1524. }
  1525. func generateAuthToken(username, password string) string {
  1526. return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
  1527. }