registry.go 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974
  1. package registry
  2. import (
  3. "context"
  4. "encoding/base64"
  5. "encoding/json"
  6. "fmt"
  7. "net/http"
  8. "net/url"
  9. "sort"
  10. "strings"
  11. "sync"
  12. "time"
  13. "github.com/porter-dev/api-contracts/generated/go/porter/v1/porterv1connect"
  14. "github.com/porter-dev/porter/internal/telemetry"
  15. artifactregistry "cloud.google.com/go/artifactregistry/apiv1beta2"
  16. "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  17. "github.com/aws/aws-sdk-go/aws/awserr"
  18. "github.com/aws/aws-sdk-go/service/ecr"
  19. "github.com/bufbuild/connect-go"
  20. porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
  21. "github.com/porter-dev/porter/api/server/shared/config"
  22. "github.com/porter-dev/porter/internal/models"
  23. "github.com/porter-dev/porter/internal/oauth"
  24. "github.com/porter-dev/porter/internal/repository"
  25. "golang.org/x/oauth2"
  26. v1artifactregistry "google.golang.org/api/artifactregistry/v1"
  27. "google.golang.org/api/iterator"
  28. "google.golang.org/api/option"
  29. artifactregistrypb "google.golang.org/genproto/googleapis/devtools/artifactregistry/v1beta2"
  30. ints "github.com/porter-dev/porter/internal/models/integrations"
  31. ptypes "github.com/porter-dev/porter/api/types"
  32. "github.com/digitalocean/godo"
  33. "github.com/docker/cli/cli/config/configfile"
  34. "github.com/docker/cli/cli/config/types"
  35. "github.com/docker/distribution/reference"
  36. "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
  37. "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
  38. )
  39. // Registry wraps the gorm Registry model
  40. type Registry models.Registry
  41. func GetECRRegistryURL(awsIntRepo repository.AWSIntegrationRepository, projectID, awsIntID uint) (string, error) {
  42. awsInt, err := awsIntRepo.ReadAWSIntegration(projectID, awsIntID)
  43. if err != nil {
  44. return "", err
  45. }
  46. sess, err := awsInt.GetSession()
  47. if err != nil {
  48. return "", err
  49. }
  50. ecrSvc := ecr.New(sess)
  51. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  52. if err != nil {
  53. return "", err
  54. }
  55. return *output.AuthorizationData[0].ProxyEndpoint, nil
  56. }
  57. // ListRepositories lists the repositories for a registry
  58. func (r *Registry) ListRepositories(
  59. ctx context.Context,
  60. repo repository.Repository,
  61. conf *config.Config,
  62. ) ([]*ptypes.RegistryRepository, error) {
  63. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  64. defer span.End()
  65. telemetry.WithAttributes(span,
  66. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  67. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  68. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  69. )
  70. // switch on the auth mechanism to get a token
  71. if r.AWSIntegrationID != 0 {
  72. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "aws"})
  73. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  74. r.ProjectID,
  75. r.AWSIntegrationID,
  76. )
  77. if err != nil {
  78. return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
  79. }
  80. repos, err := r.listECRRepositories(aws)
  81. if err != nil {
  82. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  83. }
  84. return repos, nil
  85. }
  86. if r.GCPIntegrationID != 0 {
  87. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
  88. if strings.Contains(r.URL, "pkg.dev") {
  89. return r.listGARRepositories(ctx, repo)
  90. }
  91. repos, err := r.listGCRRepositories(repo)
  92. if err != nil {
  93. return nil, telemetry.Error(ctx, span, err, "error listing gcr repositories")
  94. }
  95. return repos, nil
  96. }
  97. if r.DOIntegrationID != 0 {
  98. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "do"})
  99. repos, err := r.listDOCRRepositories(repo, conf.DOConf)
  100. if err != nil {
  101. return nil, telemetry.Error(ctx, span, err, "error listing docr repositories")
  102. }
  103. return repos, nil
  104. }
  105. if r.AzureIntegrationID != 0 {
  106. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "azure"})
  107. repos, err := r.listACRRepositories(ctx, repo)
  108. if err != nil {
  109. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  110. }
  111. return repos, nil
  112. }
  113. if r.BasicIntegrationID != 0 {
  114. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "basic"})
  115. repos, err := r.listPrivateRegistryRepositories(repo)
  116. if err != nil {
  117. return nil, telemetry.Error(ctx, span, err, "error listing private repositories")
  118. }
  119. return repos, nil
  120. }
  121. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  122. if err != nil {
  123. return nil, telemetry.Error(ctx, span, err, "error getting project for repository")
  124. }
  125. if project.CapiProvisionerEnabled {
  126. if strings.Contains(r.URL, ".azurecr.") {
  127. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "capi-azure"})
  128. req := connect.NewRequest(&porterv1.ListRepositoriesForRegistryRequest{
  129. ProjectId: int64(r.ProjectID),
  130. RegistryUri: r.URL,
  131. })
  132. resp, err := conf.ClusterControlPlaneClient.ListRepositoriesForRegistry(ctx, req)
  133. if err != nil {
  134. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  135. }
  136. res := make([]*ptypes.RegistryRepository, 0)
  137. parsedURL, err := url.Parse("https://" + r.URL)
  138. if err != nil {
  139. return nil, err
  140. }
  141. for _, repo := range resp.Msg.Repositories {
  142. res = append(res, &ptypes.RegistryRepository{
  143. Name: repo.Name,
  144. URI: parsedURL.Host + "/" + repo.Name,
  145. })
  146. }
  147. return res, nil
  148. } else {
  149. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "capi-aws"})
  150. uri := strings.TrimPrefix(r.URL, "https://")
  151. splits := strings.Split(uri, ".")
  152. if len(splits) < 4 {
  153. return nil, telemetry.Error(ctx, span, nil, "uri does not have enough splits")
  154. }
  155. accountID := splits[0]
  156. region := splits[3]
  157. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  158. ProjectId: int64(r.ProjectID),
  159. AwsAccountId: accountID,
  160. })
  161. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  162. if err != nil {
  163. return nil, telemetry.Error(ctx, span, err, "error getting capi credentials for registry")
  164. }
  165. aws := &ints.AWSIntegration{
  166. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  167. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  168. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  169. AWSRegion: region,
  170. }
  171. repos, err := r.listECRRepositories(aws)
  172. if err != nil {
  173. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  174. }
  175. return repos, nil
  176. }
  177. }
  178. return nil, telemetry.Error(ctx, span, nil, "error listing repositories")
  179. }
  180. type gcrJWT struct {
  181. AccessToken string `json:"token"`
  182. ExpiresInSec int `json:"expires_in"`
  183. }
  184. type gcrErr struct {
  185. Code string `json:"code"`
  186. Message string `json:"message"`
  187. }
  188. type gcrRepositoryResp struct {
  189. Repositories []string `json:"repositories"`
  190. Errors []gcrErr `json:"errors"`
  191. }
  192. func (r *Registry) GetGCRToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  193. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  194. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  195. r.ProjectID,
  196. r.GCPIntegrationID,
  197. )
  198. if err != nil {
  199. return nil, err
  200. }
  201. // get oauth2 access token
  202. return gcp.GetBearerToken(
  203. ctx,
  204. getTokenCache,
  205. r.setTokenCacheFunc(ctx, repo),
  206. "https://www.googleapis.com/auth/devstorage.read_write",
  207. )
  208. }
  209. func (r *Registry) listGCRRepositories(
  210. repo repository.Repository,
  211. ) ([]*ptypes.RegistryRepository, error) {
  212. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  213. r.ProjectID,
  214. r.GCPIntegrationID,
  215. )
  216. if err != nil {
  217. return nil, err
  218. }
  219. // Just use service account key to authenticate, since scopes may not be in place
  220. // for oauth. This also prevents us from making more requests.
  221. client := &http.Client{}
  222. regURL := r.URL
  223. if !strings.HasPrefix(regURL, "http") {
  224. regURL = fmt.Sprintf("https://%s", regURL)
  225. }
  226. regURLParsed, err := url.Parse(regURL)
  227. regHostname := "gcr.io"
  228. if err == nil {
  229. regHostname = regURLParsed.Host
  230. }
  231. req, err := http.NewRequest(
  232. "GET",
  233. fmt.Sprintf("https://%s/v2/_catalog", regHostname),
  234. nil,
  235. )
  236. if err != nil {
  237. return nil, err
  238. }
  239. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  240. resp, err := client.Do(req)
  241. if err != nil {
  242. return nil, err
  243. }
  244. gcrResp := gcrRepositoryResp{}
  245. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  246. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  247. }
  248. if len(gcrResp.Errors) > 0 {
  249. errMsg := ""
  250. for _, gcrErr := range gcrResp.Errors {
  251. errMsg += fmt.Sprintf(": Code %s, message %s", gcrErr.Code, gcrErr.Message)
  252. }
  253. return nil, fmt.Errorf(errMsg)
  254. }
  255. res := make([]*ptypes.RegistryRepository, 0)
  256. parsedURL, err := url.Parse("https://" + r.URL)
  257. if err != nil {
  258. return nil, err
  259. }
  260. for _, repo := range gcrResp.Repositories {
  261. res = append(res, &ptypes.RegistryRepository{
  262. Name: repo,
  263. URI: parsedURL.Host + "/" + repo,
  264. })
  265. }
  266. return res, nil
  267. }
  268. func (r *Registry) GetGARToken(ctx context.Context, repo repository.Repository) (*oauth2.Token, error) {
  269. getTokenCache := r.getTokenCacheFunc(ctx, repo)
  270. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  271. r.ProjectID,
  272. r.GCPIntegrationID,
  273. )
  274. if err != nil {
  275. return nil, err
  276. }
  277. // get oauth2 access token
  278. return gcp.GetBearerToken(
  279. ctx,
  280. getTokenCache,
  281. r.setTokenCacheFunc(ctx, repo),
  282. "https://www.googleapis.com/auth/cloud-platform",
  283. )
  284. }
  285. type garTokenSource struct {
  286. // ctx is only passed in here as the oauth2.Token() doesnt support contexts
  287. ctx context.Context
  288. reg *Registry
  289. repo repository.Repository
  290. }
  291. func (source *garTokenSource) Token() (*oauth2.Token, error) {
  292. return source.reg.GetGARToken(source.ctx, source.repo)
  293. }
  294. // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
  295. // where a repository is a single image. This function returns the list of fully qualified names
  296. // of GAR images including their repository names.
  297. func (r *Registry) listGARRepositories(
  298. ctx context.Context,
  299. repo repository.Repository,
  300. ) ([]*ptypes.RegistryRepository, error) {
  301. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  302. r.ProjectID,
  303. r.GCPIntegrationID,
  304. )
  305. if err != nil {
  306. return nil, err
  307. }
  308. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  309. reg: r,
  310. repo: repo,
  311. ctx: ctx,
  312. }), option.WithScopes("roles/artifactregistry.reader"))
  313. if err != nil {
  314. return nil, err
  315. }
  316. var repoNames []string
  317. nextToken := ""
  318. parsedURL, err := url.Parse("https://" + r.URL)
  319. if err != nil {
  320. return nil, err
  321. }
  322. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  323. for {
  324. it := client.ListRepositories(context.Background(), &artifactregistrypb.ListRepositoriesRequest{
  325. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  326. PageSize: 1000,
  327. PageToken: nextToken,
  328. })
  329. for {
  330. resp, err := it.Next()
  331. if err == iterator.Done {
  332. break
  333. } else if err != nil {
  334. return nil, err
  335. }
  336. if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
  337. repoSlice := strings.Split(resp.GetName(), "/")
  338. repoName := repoSlice[len(repoSlice)-1]
  339. repoNames = append(repoNames, repoName)
  340. }
  341. }
  342. if it.PageInfo().Token == "" {
  343. break
  344. }
  345. nextToken = it.PageInfo().Token
  346. }
  347. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  348. reg: r,
  349. repo: repo,
  350. ctx: ctx,
  351. }), option.WithScopes("roles/artifactregistry.reader"))
  352. if err != nil {
  353. return nil, err
  354. }
  355. nextToken = ""
  356. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  357. var (
  358. wg sync.WaitGroup
  359. resMap sync.Map
  360. )
  361. for _, repoName := range repoNames {
  362. wg.Add(1)
  363. go func(repoName string) {
  364. defer wg.Done()
  365. for {
  366. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  367. gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
  368. if err != nil {
  369. // FIXME: we should report this error using a channel
  370. return
  371. }
  372. for _, image := range resp.DockerImages {
  373. named, err := reference.ParseNamed(image.Uri)
  374. if err != nil {
  375. // let us skip this image becaue it has a malformed URI coming from the GCP API
  376. continue
  377. }
  378. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  379. resMap.Store(named.Name(), &ptypes.RegistryRepository{
  380. Name: repoName,
  381. URI: named.Name(),
  382. CreatedAt: uploadTime,
  383. })
  384. }
  385. if resp.NextPageToken == "" {
  386. break
  387. }
  388. nextToken = resp.NextPageToken
  389. }
  390. }(repoName)
  391. }
  392. wg.Wait()
  393. var res []*ptypes.RegistryRepository
  394. resMap.Range(func(_, value any) bool {
  395. res = append(res, value.(*ptypes.RegistryRepository))
  396. return true
  397. })
  398. return res, nil
  399. }
  400. func (r *Registry) listECRRepositories(aws *ints.AWSIntegration) ([]*ptypes.RegistryRepository, error) {
  401. sess, err := aws.GetSession()
  402. if err != nil {
  403. return nil, err
  404. }
  405. svc := ecr.New(sess)
  406. resp, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{})
  407. if err != nil {
  408. return nil, err
  409. }
  410. res := make([]*ptypes.RegistryRepository, 0)
  411. for _, repo := range resp.Repositories {
  412. res = append(res, &ptypes.RegistryRepository{
  413. Name: *repo.RepositoryName,
  414. CreatedAt: *repo.CreatedAt,
  415. URI: *repo.RepositoryUri,
  416. })
  417. }
  418. return res, nil
  419. }
  420. type acrRepositoryResp struct {
  421. Repositories []string `json:"repositories"`
  422. }
  423. func (r *Registry) listACRRepositories(ctx context.Context, repo repository.Repository) ([]*ptypes.RegistryRepository, error) {
  424. ctx, span := telemetry.NewSpan(ctx, "list-acr-repositories")
  425. defer span.End()
  426. telemetry.WithAttributes(span,
  427. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  428. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  429. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  430. )
  431. az, err := repo.AzureIntegration().ReadAzureIntegration(
  432. r.ProjectID,
  433. r.AzureIntegrationID,
  434. )
  435. if err != nil {
  436. return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
  437. }
  438. client := &http.Client{}
  439. acrURL := r.URL
  440. if !strings.Contains(acrURL, "http") {
  441. acrURL = fmt.Sprintf("https://%s", acrURL)
  442. }
  443. req, err := http.NewRequest(
  444. "GET",
  445. fmt.Sprintf("%s/v2/_catalog", acrURL),
  446. nil,
  447. )
  448. if err != nil {
  449. return nil, telemetry.Error(ctx, span, err, "error getting http request")
  450. }
  451. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  452. resp, err := client.Do(req)
  453. if err != nil {
  454. return nil, telemetry.Error(ctx, span, err, "error making http call")
  455. }
  456. acrResp := acrRepositoryResp{}
  457. if err := json.NewDecoder(resp.Body).Decode(&acrResp); err != nil {
  458. return nil, telemetry.Error(ctx, span, err, "could not read Azure registry repository response")
  459. }
  460. res := make([]*ptypes.RegistryRepository, 0)
  461. for _, repo := range acrResp.Repositories {
  462. res = append(res, &ptypes.RegistryRepository{
  463. Name: repo,
  464. URI: strings.TrimPrefix(r.URL, "https://") + "/" + repo,
  465. })
  466. }
  467. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "repo-count", Value: len(acrResp.Repositories)})
  468. return res, nil
  469. }
  470. // Returns the username/password pair for the registry
  471. func (r *Registry) GetACRCredentials(repo repository.Repository) (string, string, error) {
  472. az, err := repo.AzureIntegration().ReadAzureIntegration(
  473. r.ProjectID,
  474. r.AzureIntegrationID,
  475. )
  476. if err != nil {
  477. return "", "", err
  478. }
  479. // if the passwords and name aren't set, generate them
  480. if az.ACRTokenName == "" || len(az.ACRPassword1) == 0 {
  481. az.ACRTokenName = "porter-acr-token"
  482. // create an acr repo token
  483. cred, err := azidentity.NewClientSecretCredential(az.AzureTenantID, az.AzureClientID, string(az.ServicePrincipalSecret), nil)
  484. if err != nil {
  485. return "", "", err
  486. }
  487. scopeMapsClient, err := armcontainerregistry.NewScopeMapsClient(az.AzureSubscriptionID, cred, nil)
  488. if err != nil {
  489. return "", "", err
  490. }
  491. smRes, err := scopeMapsClient.Get(
  492. context.Background(),
  493. az.ACRResourceGroupName,
  494. az.ACRName,
  495. "_repositories_admin",
  496. nil,
  497. )
  498. if err != nil {
  499. return "", "", err
  500. }
  501. tokensClient, err := armcontainerregistry.NewTokensClient(az.AzureSubscriptionID, cred, nil)
  502. if err != nil {
  503. return "", "", err
  504. }
  505. pollerResp, err := tokensClient.BeginCreate(
  506. context.Background(),
  507. az.ACRResourceGroupName,
  508. az.ACRName,
  509. "porter-acr-token",
  510. armcontainerregistry.Token{
  511. Properties: &armcontainerregistry.TokenProperties{
  512. ScopeMapID: smRes.ID,
  513. Status: to.Ptr(armcontainerregistry.TokenStatusEnabled),
  514. },
  515. },
  516. nil,
  517. )
  518. if err != nil {
  519. return "", "", err
  520. }
  521. tokResp, err := pollerResp.PollUntilDone(context.Background(), 2*time.Second)
  522. if err != nil {
  523. return "", "", err
  524. }
  525. registriesClient, err := armcontainerregistry.NewRegistriesClient(az.AzureSubscriptionID, cred, nil)
  526. if err != nil {
  527. return "", "", err
  528. }
  529. poller, err := registriesClient.BeginGenerateCredentials(
  530. context.Background(),
  531. az.ACRResourceGroupName,
  532. az.ACRName,
  533. armcontainerregistry.GenerateCredentialsParameters{
  534. TokenID: tokResp.ID,
  535. },
  536. &armcontainerregistry.RegistriesClientBeginGenerateCredentialsOptions{ResumeToken: ""})
  537. if err != nil {
  538. return "", "", err
  539. }
  540. genCredentialsResp, err := poller.PollUntilDone(context.Background(), 2*time.Second)
  541. if err != nil {
  542. return "", "", err
  543. }
  544. for i, tokPassword := range genCredentialsResp.Passwords {
  545. if i == 0 {
  546. az.ACRPassword1 = []byte(*tokPassword.Value)
  547. } else if i == 1 {
  548. az.ACRPassword2 = []byte(*tokPassword.Value)
  549. }
  550. }
  551. // update the az integration
  552. az, err = repo.AzureIntegration().OverwriteAzureIntegration(
  553. az,
  554. )
  555. if err != nil {
  556. return "", "", err
  557. }
  558. }
  559. return az.ACRTokenName, string(az.ACRPassword1), nil
  560. }
  561. func (r *Registry) listDOCRRepositories(
  562. repo repository.Repository,
  563. doAuth *oauth2.Config,
  564. ) ([]*ptypes.RegistryRepository, error) {
  565. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  566. r.ProjectID,
  567. r.DOIntegrationID,
  568. )
  569. if err != nil {
  570. return nil, err
  571. }
  572. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  573. if err != nil {
  574. return nil, err
  575. }
  576. client := godo.NewFromToken(tok)
  577. urlArr := strings.Split(r.URL, "/")
  578. if len(urlArr) != 2 {
  579. return nil, fmt.Errorf("invalid digital ocean registry url")
  580. }
  581. name := urlArr[1]
  582. repos, _, err := client.Registry.ListRepositories(context.TODO(), name, &godo.ListOptions{})
  583. if err != nil {
  584. return nil, err
  585. }
  586. res := make([]*ptypes.RegistryRepository, 0)
  587. for _, repo := range repos {
  588. res = append(res, &ptypes.RegistryRepository{
  589. Name: repo.Name,
  590. URI: r.URL + "/" + repo.Name,
  591. })
  592. }
  593. return res, nil
  594. }
  595. func (r *Registry) listPrivateRegistryRepositories(
  596. repo repository.Repository,
  597. ) ([]*ptypes.RegistryRepository, error) {
  598. // handle dockerhub different, as it doesn't implement the docker registry http api
  599. if strings.Contains(r.URL, "docker.io") {
  600. // in this case, we just return the single dockerhub repository that's linked
  601. res := make([]*ptypes.RegistryRepository, 0)
  602. res = append(res, &ptypes.RegistryRepository{
  603. Name: strings.Split(r.URL, "docker.io/")[1],
  604. URI: r.URL,
  605. })
  606. return res, nil
  607. }
  608. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  609. r.ProjectID,
  610. r.BasicIntegrationID,
  611. )
  612. if err != nil {
  613. return nil, err
  614. }
  615. // Just use service account key to authenticate, since scopes may not be in place
  616. // for oauth. This also prevents us from making more requests.
  617. client := &http.Client{}
  618. // get the host and scheme to make the request
  619. parsedURL, err := url.Parse(r.URL)
  620. req, err := http.NewRequest(
  621. "GET",
  622. fmt.Sprintf("%s://%s/v2/_catalog", parsedURL.Scheme, parsedURL.Host),
  623. nil,
  624. )
  625. if err != nil {
  626. return nil, err
  627. }
  628. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  629. resp, err := client.Do(req)
  630. if err != nil {
  631. return nil, err
  632. }
  633. // if the status code is 404, fallback to the Docker Hub implementation
  634. if resp.StatusCode == 404 {
  635. req, err := http.NewRequest(
  636. "GET",
  637. fmt.Sprintf("%s/", r.URL),
  638. nil,
  639. )
  640. if err != nil {
  641. return nil, err
  642. }
  643. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  644. resp, err = client.Do(req)
  645. if err != nil {
  646. return nil, err
  647. }
  648. }
  649. gcrResp := gcrRepositoryResp{}
  650. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  651. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  652. }
  653. res := make([]*ptypes.RegistryRepository, 0)
  654. if err != nil {
  655. return nil, err
  656. }
  657. for _, repo := range gcrResp.Repositories {
  658. res = append(res, &ptypes.RegistryRepository{
  659. Name: repo,
  660. URI: parsedURL.Host + "/" + repo,
  661. })
  662. }
  663. return res, nil
  664. }
  665. func (r *Registry) getTokenCacheFunc(
  666. ctx context.Context,
  667. repo repository.Repository,
  668. ) ints.GetTokenCacheFunc {
  669. return func(ctx context.Context) (tok *ints.TokenCache, err error) {
  670. reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
  671. if err != nil {
  672. return nil, err
  673. }
  674. return &reg.TokenCache.TokenCache, nil
  675. }
  676. }
  677. func (r *Registry) setTokenCacheFunc(
  678. ctx context.Context,
  679. repo repository.Repository,
  680. ) ints.SetTokenCacheFunc {
  681. return func(ctx context.Context, token string, expiry time.Time) error {
  682. _, err := repo.Registry().UpdateRegistryTokenCache(
  683. &ints.RegTokenCache{
  684. TokenCache: ints.TokenCache{
  685. Token: []byte(token),
  686. Expiry: expiry,
  687. },
  688. RegistryID: r.ID,
  689. },
  690. )
  691. return err
  692. }
  693. }
  694. // CreateRepository creates a repository for a registry, if needed
  695. // (currently only required for ECR)
  696. func (r *Registry) CreateRepository(
  697. ctx context.Context,
  698. conf *config.Config,
  699. name string,
  700. ) error {
  701. // if aws, create repository
  702. if r.AWSIntegrationID != 0 {
  703. aws, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
  704. r.ProjectID,
  705. r.AWSIntegrationID,
  706. )
  707. if err != nil {
  708. return err
  709. }
  710. return r.createECRRepository(aws, name)
  711. } else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
  712. return r.createGARRepository(ctx, conf.Repo, name)
  713. }
  714. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  715. if err != nil {
  716. return fmt.Errorf("error getting project for repository: %w", err)
  717. }
  718. if project.CapiProvisionerEnabled {
  719. // no need to create repository if pushing to ACR
  720. if strings.Contains(r.URL, ".azurecr.") {
  721. return nil
  722. }
  723. uri := strings.TrimPrefix(r.URL, "https://")
  724. splits := strings.Split(uri, ".")
  725. accountID := splits[0]
  726. region := splits[3]
  727. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  728. ProjectId: int64(r.ProjectID),
  729. AwsAccountId: accountID,
  730. })
  731. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  732. if err != nil {
  733. return fmt.Errorf("error getting capi credentials for repository: %w", err)
  734. }
  735. aws := &ints.AWSIntegration{
  736. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  737. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  738. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  739. AWSRegion: region,
  740. }
  741. return r.createECRRepository(aws, name)
  742. }
  743. // otherwise, no-op
  744. return nil
  745. }
  746. func (r *Registry) createECRRepository(
  747. aws *ints.AWSIntegration,
  748. name string,
  749. ) error {
  750. sess, err := aws.GetSession()
  751. if err != nil {
  752. return err
  753. }
  754. svc := ecr.New(sess)
  755. // determine if repository already exists
  756. _, err = svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
  757. RepositoryNames: []*string{&name},
  758. })
  759. // if the repository was not found, create it
  760. if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ecr.ErrCodeRepositoryNotFoundException {
  761. _, err = svc.CreateRepository(&ecr.CreateRepositoryInput{
  762. RepositoryName: &name,
  763. })
  764. return err
  765. } else if err != nil {
  766. return err
  767. }
  768. return nil
  769. }
  770. func (r *Registry) createGARRepository(
  771. ctx context.Context,
  772. repo repository.Repository,
  773. name string,
  774. ) error {
  775. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  776. r.ProjectID,
  777. r.GCPIntegrationID,
  778. )
  779. if err != nil {
  780. return err
  781. }
  782. client, err := artifactregistry.NewClient(ctx, option.WithTokenSource(&garTokenSource{
  783. reg: r,
  784. repo: repo,
  785. ctx: ctx,
  786. }), option.WithScopes("roles/artifactregistry.admin"))
  787. if err != nil {
  788. return err
  789. }
  790. defer client.Close()
  791. parsedURL, err := url.Parse("https://" + r.URL)
  792. if err != nil {
  793. return err
  794. }
  795. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  796. _, err = client.GetRepository(context.Background(), &artifactregistrypb.GetRepositoryRequest{
  797. Name: fmt.Sprintf("projects/%s/locations/%s/repositories/%s", gcpInt.GCPProjectID, location, name),
  798. })
  799. if err != nil && strings.Contains(err.Error(), "not found") {
  800. // create a new repository
  801. _, err := client.CreateRepository(context.Background(), &artifactregistrypb.CreateRepositoryRequest{
  802. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  803. RepositoryId: name,
  804. Repository: &artifactregistrypb.Repository{
  805. Format: artifactregistrypb.Repository_DOCKER,
  806. },
  807. })
  808. if err != nil {
  809. return err
  810. }
  811. } else if err != nil {
  812. return err
  813. }
  814. return nil
  815. }
  816. // ListImages lists the images for an image repository
  817. func (r *Registry) ListImages(
  818. ctx context.Context,
  819. repoName string,
  820. repo repository.Repository,
  821. conf *config.Config,
  822. ) ([]*ptypes.Image, error) {
  823. // switch on the auth mechanism to get a token
  824. if r.AWSIntegrationID != 0 {
  825. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  826. r.ProjectID,
  827. r.AWSIntegrationID,
  828. )
  829. if err != nil {
  830. return nil, err
  831. }
  832. return r.listECRImages(aws, repoName, repo)
  833. }
  834. if r.AzureIntegrationID != 0 {
  835. return r.listACRImages(repoName, repo)
  836. }
  837. if r.GCPIntegrationID != 0 {
  838. if strings.Contains(r.URL, "pkg.dev") {
  839. return r.listGARImages(ctx, repoName, repo)
  840. }
  841. return r.listGCRImages(repoName, repo)
  842. }
  843. if r.DOIntegrationID != 0 {
  844. return r.listDOCRImages(repoName, repo, conf.DOConf)
  845. }
  846. if r.BasicIntegrationID != 0 {
  847. return r.listPrivateRegistryImages(repoName, repo)
  848. }
  849. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  850. if err != nil {
  851. return nil, fmt.Errorf("error getting project for repository: %w", err)
  852. }
  853. if project.CapiProvisionerEnabled {
  854. uri := strings.TrimPrefix(r.URL, "https://")
  855. splits := strings.Split(uri, ".")
  856. accountID := splits[0]
  857. region := splits[3]
  858. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  859. ProjectId: int64(r.ProjectID),
  860. AwsAccountId: accountID,
  861. })
  862. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  863. if err != nil {
  864. return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
  865. }
  866. aws := &ints.AWSIntegration{
  867. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  868. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  869. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  870. AWSRegion: region,
  871. }
  872. return r.listECRImages(aws, repoName, repo)
  873. }
  874. return nil, fmt.Errorf("error listing images")
  875. }
  876. func (r *Registry) GetECRPaginatedImages(
  877. repoName string,
  878. repo repository.Repository,
  879. maxResults int64,
  880. nextToken *string,
  881. ) ([]*ptypes.Image, *string, error) {
  882. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  883. r.ProjectID,
  884. r.AWSIntegrationID,
  885. )
  886. if err != nil {
  887. return nil, nil, err
  888. }
  889. sess, err := aws.GetSession()
  890. if err != nil {
  891. return nil, nil, err
  892. }
  893. svc := ecr.New(sess)
  894. resp, err := svc.ListImages(&ecr.ListImagesInput{
  895. RepositoryName: &repoName,
  896. MaxResults: &maxResults,
  897. NextToken: nextToken,
  898. })
  899. if err != nil {
  900. return nil, nil, err
  901. }
  902. if len(resp.ImageIds) == 0 {
  903. return []*ptypes.Image{}, nil, nil
  904. }
  905. imageIDLen := len(resp.ImageIds)
  906. imageDetails := make([]*ecr.ImageDetail, 0)
  907. imageIDMap := make(map[string]bool)
  908. for _, id := range resp.ImageIds {
  909. if id != nil && id.ImageTag != nil {
  910. imageIDMap[*id.ImageTag] = true
  911. }
  912. }
  913. var wg sync.WaitGroup
  914. var mu sync.Mutex
  915. // AWS API expects the length of imageIDs to be at max 100 at a time
  916. for start := 0; start < imageIDLen; start += 100 {
  917. end := start + 100
  918. if end > imageIDLen {
  919. end = imageIDLen
  920. }
  921. wg.Add(1)
  922. go func(start, end int) {
  923. defer wg.Done()
  924. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  925. RepositoryName: &repoName,
  926. ImageIds: resp.ImageIds[start:end],
  927. })
  928. if err != nil {
  929. return
  930. }
  931. mu.Lock()
  932. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  933. mu.Unlock()
  934. }(start, end)
  935. }
  936. wg.Wait()
  937. res := make([]*ptypes.Image, 0)
  938. imageInfoMap := make(map[string]*ptypes.Image)
  939. for _, img := range imageDetails {
  940. for _, tag := range img.ImageTags {
  941. newImage := &ptypes.Image{
  942. Digest: *img.ImageDigest,
  943. Tag: *tag,
  944. RepositoryName: repoName,
  945. PushedAt: img.ImagePushedAt,
  946. }
  947. if _, ok := imageIDMap[*tag]; ok {
  948. if _, ok := imageInfoMap[*tag]; !ok {
  949. imageInfoMap[*tag] = newImage
  950. }
  951. }
  952. if len(imageInfoMap) == int(maxResults) {
  953. break
  954. }
  955. }
  956. if len(imageInfoMap) == int(maxResults) {
  957. break
  958. }
  959. }
  960. for _, v := range imageInfoMap {
  961. res = append(res, v)
  962. }
  963. sort.Slice(res, func(i, j int) bool {
  964. if res[i].PushedAt == nil || res[j].PushedAt == nil {
  965. return false
  966. }
  967. return res[i].PushedAt.After(*res[j].PushedAt)
  968. })
  969. return res, resp.NextToken, nil
  970. }
  971. func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  972. sess, err := aws.GetSession()
  973. if err != nil {
  974. return nil, err
  975. }
  976. svc := ecr.New(sess)
  977. maxResults := int64(1000)
  978. var imageIDs []*ecr.ImageIdentifier
  979. resp, err := svc.ListImages(&ecr.ListImagesInput{
  980. RepositoryName: &repoName,
  981. MaxResults: &maxResults,
  982. })
  983. if err != nil {
  984. return nil, err
  985. }
  986. if len(resp.ImageIds) == 0 {
  987. return []*ptypes.Image{}, nil
  988. }
  989. imageIDs = append(imageIDs, resp.ImageIds...)
  990. nextToken := resp.NextToken
  991. for nextToken != nil {
  992. resp, err := svc.ListImages(&ecr.ListImagesInput{
  993. RepositoryName: &repoName,
  994. MaxResults: &maxResults,
  995. NextToken: nextToken,
  996. })
  997. if err != nil {
  998. return nil, err
  999. }
  1000. imageIDs = append(imageIDs, resp.ImageIds...)
  1001. nextToken = resp.NextToken
  1002. }
  1003. imageIDLen := len(imageIDs)
  1004. imageDetails := make([]*ecr.ImageDetail, 0)
  1005. var wg sync.WaitGroup
  1006. var mu sync.Mutex
  1007. // AWS API expects the length of imageIDs to be at max 100 at a time
  1008. for start := 0; start < imageIDLen; start += 100 {
  1009. end := start + 100
  1010. if end > imageIDLen {
  1011. end = imageIDLen
  1012. }
  1013. wg.Add(1)
  1014. go func(start, end int) {
  1015. defer wg.Done()
  1016. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  1017. RepositoryName: &repoName,
  1018. ImageIds: imageIDs[start:end],
  1019. })
  1020. if err != nil {
  1021. return
  1022. }
  1023. mu.Lock()
  1024. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  1025. mu.Unlock()
  1026. }(start, end)
  1027. }
  1028. wg.Wait()
  1029. res := make([]*ptypes.Image, 0)
  1030. imageInfoMap := make(map[string]*ptypes.Image)
  1031. for _, img := range imageDetails {
  1032. for _, tag := range img.ImageTags {
  1033. newImage := &ptypes.Image{
  1034. Digest: *img.ImageDigest,
  1035. Tag: *tag,
  1036. RepositoryName: repoName,
  1037. PushedAt: img.ImagePushedAt,
  1038. }
  1039. if _, ok := imageInfoMap[*tag]; !ok {
  1040. imageInfoMap[*tag] = newImage
  1041. }
  1042. }
  1043. }
  1044. for _, v := range imageInfoMap {
  1045. res = append(res, v)
  1046. }
  1047. return res, nil
  1048. }
  1049. func (r *Registry) listACRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1050. az, err := repo.AzureIntegration().ReadAzureIntegration(
  1051. r.ProjectID,
  1052. r.AzureIntegrationID,
  1053. )
  1054. if err != nil {
  1055. return nil, err
  1056. }
  1057. // use JWT token to request catalog
  1058. client := &http.Client{}
  1059. req, err := http.NewRequest(
  1060. "GET",
  1061. fmt.Sprintf("%s/v2/%s/tags/list", r.URL, repoName),
  1062. nil,
  1063. )
  1064. if err != nil {
  1065. return nil, err
  1066. }
  1067. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  1068. resp, err := client.Do(req)
  1069. if err != nil {
  1070. return nil, err
  1071. }
  1072. gcrResp := gcrImageResp{}
  1073. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1074. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1075. }
  1076. res := make([]*ptypes.Image, 0)
  1077. for _, tag := range gcrResp.Tags {
  1078. res = append(res, &ptypes.Image{
  1079. RepositoryName: strings.TrimPrefix(repoName, "https://"),
  1080. Tag: tag,
  1081. })
  1082. }
  1083. return res, nil
  1084. }
  1085. type gcrImageResp struct {
  1086. Tags []string `json:"tags"`
  1087. }
  1088. func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1089. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1090. r.ProjectID,
  1091. r.GCPIntegrationID,
  1092. )
  1093. if err != nil {
  1094. return nil, err
  1095. }
  1096. // use JWT token to request catalog
  1097. client := &http.Client{}
  1098. parsedURL, err := url.Parse("https://" + r.URL)
  1099. if err != nil {
  1100. return nil, err
  1101. }
  1102. trimmedPath := strings.Trim(parsedURL.Path, "/")
  1103. req, err := http.NewRequest(
  1104. "GET",
  1105. fmt.Sprintf("https://%s/v2/%s/%s/tags/list", parsedURL.Host, trimmedPath, repoName),
  1106. nil,
  1107. )
  1108. if err != nil {
  1109. return nil, err
  1110. }
  1111. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  1112. resp, err := client.Do(req)
  1113. if err != nil {
  1114. return nil, err
  1115. }
  1116. gcrResp := gcrImageResp{}
  1117. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1118. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1119. }
  1120. res := make([]*ptypes.Image, 0)
  1121. for _, tag := range gcrResp.Tags {
  1122. res = append(res, &ptypes.Image{
  1123. RepositoryName: repoName,
  1124. Tag: tag,
  1125. })
  1126. }
  1127. return res, nil
  1128. }
  1129. func (r *Registry) listGARImages(ctx context.Context, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1130. repoImageSlice := strings.Split(repoName, "/")
  1131. if len(repoImageSlice) != 2 {
  1132. return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
  1133. }
  1134. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  1135. r.ProjectID,
  1136. r.GCPIntegrationID,
  1137. )
  1138. if err != nil {
  1139. return nil, err
  1140. }
  1141. svc, err := v1artifactregistry.NewService(ctx, option.WithTokenSource(&garTokenSource{
  1142. reg: r,
  1143. repo: repo,
  1144. ctx: ctx,
  1145. }), option.WithScopes("roles/artifactregistry.reader"))
  1146. if err != nil {
  1147. return nil, err
  1148. }
  1149. var res []*ptypes.Image
  1150. parsedURL, err := url.Parse("https://" + r.URL)
  1151. if err != nil {
  1152. return nil, err
  1153. }
  1154. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  1155. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  1156. nextToken := ""
  1157. for {
  1158. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  1159. gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
  1160. if err != nil {
  1161. return nil, err
  1162. }
  1163. for _, image := range resp.DockerImages {
  1164. named, err := reference.ParseNamed(image.Uri)
  1165. if err != nil {
  1166. continue
  1167. }
  1168. paths := strings.Split(reference.Path(named), "/")
  1169. imageName := paths[len(paths)-1]
  1170. if imageName == repoImageSlice[1] {
  1171. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  1172. for _, tag := range image.Tags {
  1173. res = append(res, &ptypes.Image{
  1174. RepositoryName: repoName,
  1175. Tag: tag,
  1176. PushedAt: &uploadTime,
  1177. Digest: strings.Split(image.Uri, "@")[1],
  1178. })
  1179. }
  1180. }
  1181. }
  1182. if resp.NextPageToken == "" {
  1183. break
  1184. }
  1185. nextToken = resp.NextPageToken
  1186. }
  1187. return res, nil
  1188. }
  1189. func (r *Registry) listDOCRImages(
  1190. repoName string,
  1191. repo repository.Repository,
  1192. doAuth *oauth2.Config,
  1193. ) ([]*ptypes.Image, error) {
  1194. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1195. r.ProjectID,
  1196. r.DOIntegrationID,
  1197. )
  1198. if err != nil {
  1199. return nil, err
  1200. }
  1201. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1202. if err != nil {
  1203. return nil, err
  1204. }
  1205. client := godo.NewFromToken(tok)
  1206. urlArr := strings.Split(r.URL, "/")
  1207. if len(urlArr) != 2 {
  1208. return nil, fmt.Errorf("invalid digital ocean registry url")
  1209. }
  1210. name := urlArr[1]
  1211. var tags []*godo.RepositoryTag
  1212. opt := &godo.ListOptions{
  1213. PerPage: 200,
  1214. }
  1215. for {
  1216. nextTags, resp, err := client.Registry.ListRepositoryTags(context.TODO(), name, repoName, opt)
  1217. if err != nil {
  1218. return nil, err
  1219. }
  1220. tags = append(tags, nextTags...)
  1221. if resp.Links == nil || resp.Links.IsLastPage() {
  1222. break
  1223. }
  1224. page, err := resp.Links.CurrentPage()
  1225. if err != nil {
  1226. return nil, err
  1227. }
  1228. opt.Page = page + 1
  1229. }
  1230. res := make([]*ptypes.Image, 0)
  1231. for _, tag := range tags {
  1232. res = append(res, &ptypes.Image{
  1233. RepositoryName: repoName,
  1234. Tag: tag.Tag,
  1235. })
  1236. }
  1237. return res, nil
  1238. }
  1239. func (r *Registry) listPrivateRegistryImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1240. // handle dockerhub different, as it doesn't implement the docker registry http api
  1241. if strings.Contains(r.URL, "docker.io") {
  1242. return r.listDockerHubImages(repoName, repo)
  1243. }
  1244. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1245. r.ProjectID,
  1246. r.BasicIntegrationID,
  1247. )
  1248. if err != nil {
  1249. return nil, err
  1250. }
  1251. // Just use service account key to authenticate, since scopes may not be in place
  1252. // for oauth. This also prevents us from making more requests.
  1253. client := &http.Client{}
  1254. // get the host and scheme to make the request
  1255. parsedURL, err := url.Parse(r.URL)
  1256. req, err := http.NewRequest(
  1257. "GET",
  1258. fmt.Sprintf("%s://%s/v2/%s/tags/list", parsedURL.Scheme, parsedURL.Host, repoName),
  1259. nil,
  1260. )
  1261. if err != nil {
  1262. return nil, err
  1263. }
  1264. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  1265. resp, err := client.Do(req)
  1266. if err != nil {
  1267. return nil, err
  1268. }
  1269. gcrResp := gcrImageResp{}
  1270. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1271. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1272. }
  1273. res := make([]*ptypes.Image, 0)
  1274. for _, tag := range gcrResp.Tags {
  1275. res = append(res, &ptypes.Image{
  1276. RepositoryName: repoName,
  1277. Tag: tag,
  1278. })
  1279. }
  1280. return res, nil
  1281. }
  1282. type dockerHubImageResult struct {
  1283. Name string `json:"name"`
  1284. }
  1285. type dockerHubImageResp struct {
  1286. Results []dockerHubImageResult `json:"results"`
  1287. }
  1288. type dockerHubLoginReq struct {
  1289. Username string `json:"username"`
  1290. Password string `json:"password"`
  1291. }
  1292. type dockerHubLoginResp struct {
  1293. Token string `json:"token"`
  1294. }
  1295. func (r *Registry) listDockerHubImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1296. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1297. r.ProjectID,
  1298. r.BasicIntegrationID,
  1299. )
  1300. if err != nil {
  1301. return nil, err
  1302. }
  1303. client := &http.Client{}
  1304. // first, make a request for the access token
  1305. data, err := json.Marshal(&dockerHubLoginReq{
  1306. Username: string(basic.Username),
  1307. Password: string(basic.Password),
  1308. })
  1309. if err != nil {
  1310. return nil, err
  1311. }
  1312. req, err := http.NewRequest(
  1313. "POST",
  1314. "https://hub.docker.com/v2/users/login",
  1315. strings.NewReader(string(data)),
  1316. )
  1317. if err != nil {
  1318. return nil, err
  1319. }
  1320. req.Header.Add("Content-Type", "application/json")
  1321. resp, err := client.Do(req)
  1322. if err != nil {
  1323. return nil, err
  1324. }
  1325. tokenObj := dockerHubLoginResp{}
  1326. if err := json.NewDecoder(resp.Body).Decode(&tokenObj); err != nil {
  1327. return nil, fmt.Errorf("Could not decode Dockerhub token from response: %v", err)
  1328. }
  1329. req, err = http.NewRequest(
  1330. "GET",
  1331. fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags", strings.Split(r.URL, "docker.io/")[1]),
  1332. nil,
  1333. )
  1334. if err != nil {
  1335. return nil, err
  1336. }
  1337. req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenObj.Token))
  1338. resp, err = client.Do(req)
  1339. if err != nil {
  1340. return nil, err
  1341. }
  1342. imageResp := dockerHubImageResp{}
  1343. if err := json.NewDecoder(resp.Body).Decode(&imageResp); err != nil {
  1344. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1345. }
  1346. res := make([]*ptypes.Image, 0)
  1347. for _, result := range imageResp.Results {
  1348. res = append(res, &ptypes.Image{
  1349. RepositoryName: repoName,
  1350. Tag: result.Name,
  1351. })
  1352. }
  1353. return res, nil
  1354. }
  1355. // GetDockerConfigJSON returns a dockerconfigjson file contents with "auths"
  1356. // populated.
  1357. func (r *Registry) GetDockerConfigJSON(
  1358. repo repository.Repository,
  1359. doAuth *oauth2.Config, // only required if using DOCR
  1360. ccpClient porterv1connect.ClusterControlPlaneServiceClient,
  1361. ) ([]byte, error) {
  1362. ctx, span := telemetry.NewSpan(context.Background(), "docker-config-json")
  1363. defer span.End()
  1364. telemetry.WithAttributes(span,
  1365. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  1366. telemetry.AttributeKV{Key: "registry-url", Value: r.URL},
  1367. )
  1368. var conf *configfile.ConfigFile
  1369. var err error
  1370. // switch on the auth mechanism to get a token
  1371. if r.AWSIntegrationID != 0 {
  1372. conf, err = r.getECRDockerConfigFile(repo)
  1373. if err != nil {
  1374. return nil, telemetry.Error(ctx, span, err, "error getting ECR docker config file")
  1375. }
  1376. return json.Marshal(conf)
  1377. }
  1378. if r.GCPIntegrationID != 0 {
  1379. conf, err = r.getGCRDockerConfigFile(repo)
  1380. if err != nil {
  1381. return nil, telemetry.Error(ctx, span, err, "error getting GCP docker config file")
  1382. }
  1383. return json.Marshal(conf)
  1384. }
  1385. if r.DOIntegrationID != 0 {
  1386. conf, err = r.getDOCRDockerConfigFile(repo, doAuth)
  1387. if err != nil {
  1388. return nil, telemetry.Error(ctx, span, err, "error getting DO docker config file")
  1389. }
  1390. return json.Marshal(conf)
  1391. }
  1392. if r.BasicIntegrationID != 0 {
  1393. conf, err = r.getPrivateRegistryDockerConfigFile(repo)
  1394. if err != nil {
  1395. return nil, telemetry.Error(ctx, span, err, "error getting private docker config file")
  1396. }
  1397. return json.Marshal(conf)
  1398. }
  1399. if r.AzureIntegrationID != 0 {
  1400. conf, err = r.getACRDockerConfigFile(repo)
  1401. if err != nil {
  1402. return nil, telemetry.Error(ctx, span, err, "error getting ACR docker config file")
  1403. }
  1404. return json.Marshal(conf)
  1405. }
  1406. proj, err := repo.Project().ReadProject(r.ProjectID)
  1407. if err != nil {
  1408. return nil, telemetry.Error(ctx, span, err, "error reading project id")
  1409. }
  1410. if proj.CapiProvisionerEnabled && strings.Contains(r.URL, ".azurecr") {
  1411. if ccpClient == nil {
  1412. return nil, telemetry.Error(ctx, span, nil, "ccpClient cannot be nil")
  1413. }
  1414. dockerConfigReq := connect.NewRequest(&porterv1.DockerConfigFileForRegistryRequest{
  1415. ProjectId: int64(proj.ID),
  1416. RegistryUri: r.URL,
  1417. })
  1418. dockerConfigResp, err := ccpClient.DockerConfigFileForRegistry(ctx, dockerConfigReq)
  1419. if err != nil {
  1420. return nil, telemetry.Error(ctx, span, err, "error getting token response from ccp")
  1421. }
  1422. if dockerConfigResp.Msg == nil || dockerConfigResp.Msg.DockerConfigFile == nil {
  1423. return nil, telemetry.Error(ctx, span, err, "no docker config found in response")
  1424. }
  1425. return dockerConfigResp.Msg.DockerConfigFile, nil
  1426. }
  1427. marshalledConf, err := json.Marshal(conf)
  1428. if err != nil {
  1429. return nil, telemetry.Error(ctx, span, err, "error marshalling conf")
  1430. }
  1431. return marshalledConf, nil
  1432. }
  1433. func (r *Registry) getECRDockerConfigFile(
  1434. repo repository.Repository,
  1435. ) (*configfile.ConfigFile, error) {
  1436. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  1437. r.ProjectID,
  1438. r.AWSIntegrationID,
  1439. )
  1440. if err != nil {
  1441. return nil, err
  1442. }
  1443. sess, err := aws.GetSession()
  1444. if err != nil {
  1445. return nil, err
  1446. }
  1447. ecrSvc := ecr.New(sess)
  1448. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  1449. if err != nil {
  1450. return nil, err
  1451. }
  1452. token := *output.AuthorizationData[0].AuthorizationToken
  1453. decodedToken, err := base64.StdEncoding.DecodeString(token)
  1454. if err != nil {
  1455. return nil, err
  1456. }
  1457. parts := strings.SplitN(string(decodedToken), ":", 2)
  1458. if len(parts) < 2 {
  1459. return nil, err
  1460. }
  1461. key := r.URL
  1462. if !strings.Contains(key, "http") {
  1463. key = "https://" + key
  1464. }
  1465. return &configfile.ConfigFile{
  1466. AuthConfigs: map[string]types.AuthConfig{
  1467. key: {
  1468. Username: parts[0],
  1469. Password: parts[1],
  1470. Auth: token,
  1471. },
  1472. },
  1473. }, nil
  1474. }
  1475. func (r *Registry) getGCRDockerConfigFile(
  1476. repo repository.Repository,
  1477. ) (*configfile.ConfigFile, error) {
  1478. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1479. r.ProjectID,
  1480. r.GCPIntegrationID,
  1481. )
  1482. if err != nil {
  1483. return nil, err
  1484. }
  1485. key := r.URL
  1486. if !strings.Contains(key, "http") {
  1487. key = "https://" + key
  1488. }
  1489. parsedURL, _ := url.Parse(key)
  1490. return &configfile.ConfigFile{
  1491. AuthConfigs: map[string]types.AuthConfig{
  1492. parsedURL.Host: {
  1493. Username: "_json_key",
  1494. Password: string(gcp.GCPKeyData),
  1495. Auth: generateAuthToken("_json_key", string(gcp.GCPKeyData)),
  1496. },
  1497. },
  1498. }, nil
  1499. }
  1500. func (r *Registry) getDOCRDockerConfigFile(
  1501. repo repository.Repository,
  1502. doAuth *oauth2.Config,
  1503. ) (*configfile.ConfigFile, error) {
  1504. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1505. r.ProjectID,
  1506. r.DOIntegrationID,
  1507. )
  1508. if err != nil {
  1509. return nil, err
  1510. }
  1511. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1512. if err != nil {
  1513. return nil, err
  1514. }
  1515. key := r.URL
  1516. if !strings.Contains(key, "http") {
  1517. key = "https://" + key
  1518. }
  1519. parsedURL, _ := url.Parse(key)
  1520. return &configfile.ConfigFile{
  1521. AuthConfigs: map[string]types.AuthConfig{
  1522. parsedURL.Host: {
  1523. Username: tok,
  1524. Password: tok,
  1525. Auth: generateAuthToken(tok, tok),
  1526. },
  1527. },
  1528. }, nil
  1529. }
  1530. func (r *Registry) getPrivateRegistryDockerConfigFile(
  1531. repo repository.Repository,
  1532. ) (*configfile.ConfigFile, error) {
  1533. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1534. r.ProjectID,
  1535. r.BasicIntegrationID,
  1536. )
  1537. if err != nil {
  1538. return nil, err
  1539. }
  1540. key := r.URL
  1541. if !strings.Contains(key, "http") {
  1542. key = "https://" + key
  1543. }
  1544. parsedURL, _ := url.Parse(key)
  1545. authConfigKey := parsedURL.Host
  1546. if strings.Contains(r.URL, "index.docker.io") {
  1547. authConfigKey = "https://index.docker.io/v1/"
  1548. }
  1549. return &configfile.ConfigFile{
  1550. AuthConfigs: map[string]types.AuthConfig{
  1551. authConfigKey: {
  1552. Username: string(basic.Username),
  1553. Password: string(basic.Password),
  1554. Auth: generateAuthToken(string(basic.Username), string(basic.Password)),
  1555. },
  1556. },
  1557. }, nil
  1558. }
  1559. func (r *Registry) getACRDockerConfigFile(
  1560. repo repository.Repository,
  1561. ) (*configfile.ConfigFile, error) {
  1562. username, pw, err := r.GetACRCredentials(repo)
  1563. if err != nil {
  1564. return nil, err
  1565. }
  1566. key := r.URL
  1567. if !strings.Contains(key, "http") {
  1568. key = "https://" + key
  1569. }
  1570. parsedURL, _ := url.Parse(key)
  1571. return &configfile.ConfigFile{
  1572. AuthConfigs: map[string]types.AuthConfig{
  1573. parsedURL.Host: {
  1574. Username: string(username),
  1575. Password: string(pw),
  1576. Auth: generateAuthToken(string(username), string(pw)),
  1577. },
  1578. },
  1579. }, nil
  1580. }
  1581. func generateAuthToken(username, password string) string {
  1582. return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
  1583. }