registry.go 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884
  1. package registry
  2. import (
  3. "context"
  4. "encoding/base64"
  5. "encoding/json"
  6. "fmt"
  7. "net/http"
  8. "net/url"
  9. "sort"
  10. "strings"
  11. "sync"
  12. "time"
  13. "github.com/porter-dev/porter/internal/telemetry"
  14. artifactregistry "cloud.google.com/go/artifactregistry/apiv1beta2"
  15. "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
  16. "github.com/aws/aws-sdk-go/aws/awserr"
  17. "github.com/aws/aws-sdk-go/service/ecr"
  18. "github.com/bufbuild/connect-go"
  19. porterv1 "github.com/porter-dev/api-contracts/generated/go/porter/v1"
  20. "github.com/porter-dev/porter/api/server/shared/config"
  21. "github.com/porter-dev/porter/internal/models"
  22. "github.com/porter-dev/porter/internal/oauth"
  23. "github.com/porter-dev/porter/internal/repository"
  24. "golang.org/x/oauth2"
  25. v1artifactregistry "google.golang.org/api/artifactregistry/v1"
  26. "google.golang.org/api/iterator"
  27. "google.golang.org/api/option"
  28. artifactregistrypb "google.golang.org/genproto/googleapis/devtools/artifactregistry/v1beta2"
  29. ints "github.com/porter-dev/porter/internal/models/integrations"
  30. ptypes "github.com/porter-dev/porter/api/types"
  31. "github.com/digitalocean/godo"
  32. "github.com/docker/cli/cli/config/configfile"
  33. "github.com/docker/cli/cli/config/types"
  34. "github.com/docker/distribution/reference"
  35. "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
  36. "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
  37. )
  38. // Registry wraps the gorm Registry model
  39. type Registry models.Registry
  40. func GetECRRegistryURL(awsIntRepo repository.AWSIntegrationRepository, projectID, awsIntID uint) (string, error) {
  41. awsInt, err := awsIntRepo.ReadAWSIntegration(projectID, awsIntID)
  42. if err != nil {
  43. return "", err
  44. }
  45. sess, err := awsInt.GetSession()
  46. if err != nil {
  47. return "", err
  48. }
  49. ecrSvc := ecr.New(sess)
  50. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  51. if err != nil {
  52. return "", err
  53. }
  54. return *output.AuthorizationData[0].ProxyEndpoint, nil
  55. }
  56. // ListRepositories lists the repositories for a registry
  57. func (r *Registry) ListRepositories(
  58. ctx context.Context,
  59. repo repository.Repository,
  60. conf *config.Config,
  61. ) ([]*ptypes.RegistryRepository, error) {
  62. ctx, span := telemetry.NewSpan(ctx, "list-repositories")
  63. defer span.End()
  64. telemetry.WithAttributes(span,
  65. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  66. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  67. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  68. )
  69. // switch on the auth mechanism to get a token
  70. if r.AWSIntegrationID != 0 {
  71. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "aws"})
  72. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  73. r.ProjectID,
  74. r.AWSIntegrationID,
  75. )
  76. if err != nil {
  77. return nil, telemetry.Error(ctx, span, err, "error reading aws integration")
  78. }
  79. repos, err := r.listECRRepositories(aws)
  80. if err != nil {
  81. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  82. }
  83. return repos, nil
  84. }
  85. if r.GCPIntegrationID != 0 {
  86. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "gcp"})
  87. if strings.Contains(r.URL, "pkg.dev") {
  88. return r.listGARRepositories(repo)
  89. }
  90. repos, err := r.listGCRRepositories(repo)
  91. if err != nil {
  92. return nil, telemetry.Error(ctx, span, err, "error listing gcr repositories")
  93. }
  94. return repos, nil
  95. }
  96. if r.DOIntegrationID != 0 {
  97. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "do"})
  98. repos, err := r.listDOCRRepositories(repo, conf.DOConf)
  99. if err != nil {
  100. return nil, telemetry.Error(ctx, span, err, "error listing docr repositories")
  101. }
  102. return repos, nil
  103. }
  104. if r.AzureIntegrationID != 0 {
  105. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "azure"})
  106. repos, err := r.listACRRepositories(ctx, repo)
  107. if err != nil {
  108. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  109. }
  110. return repos, nil
  111. }
  112. if r.BasicIntegrationID != 0 {
  113. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "basic"})
  114. repos, err := r.listPrivateRegistryRepositories(repo)
  115. if err != nil {
  116. return nil, telemetry.Error(ctx, span, err, "error listing private repositories")
  117. }
  118. return repos, nil
  119. }
  120. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  121. if err != nil {
  122. return nil, telemetry.Error(ctx, span, err, "error getting project for repository")
  123. }
  124. if project.CapiProvisionerEnabled {
  125. if strings.Contains(r.URL, ".azurecr.") {
  126. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "capi-azure"})
  127. creds, err := conf.Repo.AzureIntegration().ListAzureIntegrationsByProjectID(r.ProjectID)
  128. if err != nil {
  129. return nil, telemetry.Error(ctx, span, err, "error getting azure credentials for capi cluster")
  130. }
  131. if len(creds) == 0 {
  132. return nil, telemetry.Error(ctx, span, err, "no azure credentials for capi cluster")
  133. }
  134. r.AzureIntegrationID = creds[0].ID
  135. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "azure-integration-id", Value: r.AzureIntegrationID})
  136. repos, err := r.listACRRepositories(ctx, repo)
  137. if err != nil {
  138. return nil, telemetry.Error(ctx, span, err, "error listing acr repositories")
  139. }
  140. return repos, nil
  141. } else {
  142. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "auth-mechanism", Value: "capi-aws"})
  143. uri := strings.TrimPrefix(r.URL, "https://")
  144. splits := strings.Split(uri, ".")
  145. if len(splits) < 4 {
  146. return nil, telemetry.Error(ctx, span, nil, "uri does not have enough splits")
  147. }
  148. accountID := splits[0]
  149. region := splits[3]
  150. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  151. ProjectId: int64(r.ProjectID),
  152. AwsAccountId: accountID,
  153. })
  154. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  155. if err != nil {
  156. return nil, telemetry.Error(ctx, span, err, "error getting capi credentials for registry")
  157. }
  158. aws := &ints.AWSIntegration{
  159. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  160. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  161. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  162. AWSRegion: region,
  163. }
  164. repos, err := r.listECRRepositories(aws)
  165. if err != nil {
  166. return nil, telemetry.Error(ctx, span, err, "error listing ecr repositories")
  167. }
  168. return repos, nil
  169. }
  170. }
  171. return nil, telemetry.Error(ctx, span, nil, "error listing repositories")
  172. }
  173. type gcrJWT struct {
  174. AccessToken string `json:"token"`
  175. ExpiresInSec int `json:"expires_in"`
  176. }
  177. type gcrErr struct {
  178. Code string `json:"code"`
  179. Message string `json:"message"`
  180. }
  181. type gcrRepositoryResp struct {
  182. Repositories []string `json:"repositories"`
  183. Errors []gcrErr `json:"errors"`
  184. }
  185. func (r *Registry) GetGCRToken(repo repository.Repository) (*oauth2.Token, error) {
  186. getTokenCache := r.getTokenCacheFunc(repo)
  187. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  188. r.ProjectID,
  189. r.GCPIntegrationID,
  190. )
  191. if err != nil {
  192. return nil, err
  193. }
  194. // get oauth2 access token
  195. return gcp.GetBearerToken(
  196. getTokenCache,
  197. r.setTokenCacheFunc(repo),
  198. "https://www.googleapis.com/auth/devstorage.read_write",
  199. )
  200. }
  201. func (r *Registry) listGCRRepositories(
  202. repo repository.Repository,
  203. ) ([]*ptypes.RegistryRepository, error) {
  204. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  205. r.ProjectID,
  206. r.GCPIntegrationID,
  207. )
  208. if err != nil {
  209. return nil, err
  210. }
  211. // Just use service account key to authenticate, since scopes may not be in place
  212. // for oauth. This also prevents us from making more requests.
  213. client := &http.Client{}
  214. regURL := r.URL
  215. if !strings.HasPrefix(regURL, "http") {
  216. regURL = fmt.Sprintf("https://%s", regURL)
  217. }
  218. regURLParsed, err := url.Parse(regURL)
  219. regHostname := "gcr.io"
  220. if err == nil {
  221. regHostname = regURLParsed.Host
  222. }
  223. req, err := http.NewRequest(
  224. "GET",
  225. fmt.Sprintf("https://%s/v2/_catalog", regHostname),
  226. nil,
  227. )
  228. if err != nil {
  229. return nil, err
  230. }
  231. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  232. resp, err := client.Do(req)
  233. if err != nil {
  234. return nil, err
  235. }
  236. gcrResp := gcrRepositoryResp{}
  237. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  238. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  239. }
  240. if len(gcrResp.Errors) > 0 {
  241. errMsg := ""
  242. for _, gcrErr := range gcrResp.Errors {
  243. errMsg += fmt.Sprintf(": Code %s, message %s", gcrErr.Code, gcrErr.Message)
  244. }
  245. return nil, fmt.Errorf(errMsg)
  246. }
  247. res := make([]*ptypes.RegistryRepository, 0)
  248. parsedURL, err := url.Parse("https://" + r.URL)
  249. if err != nil {
  250. return nil, err
  251. }
  252. for _, repo := range gcrResp.Repositories {
  253. res = append(res, &ptypes.RegistryRepository{
  254. Name: repo,
  255. URI: parsedURL.Host + "/" + repo,
  256. })
  257. }
  258. return res, nil
  259. }
  260. func (r *Registry) GetGARToken(repo repository.Repository) (*oauth2.Token, error) {
  261. getTokenCache := r.getTokenCacheFunc(repo)
  262. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  263. r.ProjectID,
  264. r.GCPIntegrationID,
  265. )
  266. if err != nil {
  267. return nil, err
  268. }
  269. // get oauth2 access token
  270. return gcp.GetBearerToken(
  271. getTokenCache,
  272. r.setTokenCacheFunc(repo),
  273. "https://www.googleapis.com/auth/cloud-platform",
  274. )
  275. }
  276. type garTokenSource struct {
  277. reg *Registry
  278. repo repository.Repository
  279. }
  280. func (source *garTokenSource) Token() (*oauth2.Token, error) {
  281. return source.reg.GetGARToken(source.repo)
  282. }
  283. // GAR has the concept of a "repository" which is a collection of images, unlike ECR or others
  284. // where a repository is a single image. This function returns the list of fully qualified names
  285. // of GAR images including their repository names.
  286. func (r *Registry) listGARRepositories(
  287. repo repository.Repository,
  288. ) ([]*ptypes.RegistryRepository, error) {
  289. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  290. r.ProjectID,
  291. r.GCPIntegrationID,
  292. )
  293. if err != nil {
  294. return nil, err
  295. }
  296. client, err := artifactregistry.NewClient(context.Background(), option.WithTokenSource(&garTokenSource{
  297. reg: r,
  298. repo: repo,
  299. }), option.WithScopes("roles/artifactregistry.reader"))
  300. if err != nil {
  301. return nil, err
  302. }
  303. var repoNames []string
  304. nextToken := ""
  305. parsedURL, err := url.Parse("https://" + r.URL)
  306. if err != nil {
  307. return nil, err
  308. }
  309. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  310. for {
  311. it := client.ListRepositories(context.Background(), &artifactregistrypb.ListRepositoriesRequest{
  312. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  313. PageSize: 1000,
  314. PageToken: nextToken,
  315. })
  316. for {
  317. resp, err := it.Next()
  318. if err == iterator.Done {
  319. break
  320. } else if err != nil {
  321. return nil, err
  322. }
  323. if resp.GetFormat() == artifactregistrypb.Repository_DOCKER { // we only care about
  324. repoSlice := strings.Split(resp.GetName(), "/")
  325. repoName := repoSlice[len(repoSlice)-1]
  326. repoNames = append(repoNames, repoName)
  327. }
  328. }
  329. if it.PageInfo().Token == "" {
  330. break
  331. }
  332. nextToken = it.PageInfo().Token
  333. }
  334. svc, err := v1artifactregistry.NewService(context.Background(), option.WithTokenSource(&garTokenSource{
  335. reg: r,
  336. repo: repo,
  337. }), option.WithScopes("roles/artifactregistry.reader"))
  338. if err != nil {
  339. return nil, err
  340. }
  341. nextToken = ""
  342. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  343. var (
  344. wg sync.WaitGroup
  345. resMap sync.Map
  346. )
  347. for _, repoName := range repoNames {
  348. wg.Add(1)
  349. go func(repoName string) {
  350. defer wg.Done()
  351. for {
  352. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  353. gcpInt.GCPProjectID, location, repoName)).PageSize(1000).PageToken(nextToken).Do()
  354. if err != nil {
  355. // FIXME: we should report this error using a channel
  356. return
  357. }
  358. for _, image := range resp.DockerImages {
  359. named, err := reference.ParseNamed(image.Uri)
  360. if err != nil {
  361. // let us skip this image becaue it has a malformed URI coming from the GCP API
  362. continue
  363. }
  364. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  365. resMap.Store(named.Name(), &ptypes.RegistryRepository{
  366. Name: repoName,
  367. URI: named.Name(),
  368. CreatedAt: uploadTime,
  369. })
  370. }
  371. if resp.NextPageToken == "" {
  372. break
  373. }
  374. nextToken = resp.NextPageToken
  375. }
  376. }(repoName)
  377. }
  378. wg.Wait()
  379. var res []*ptypes.RegistryRepository
  380. resMap.Range(func(_, value any) bool {
  381. res = append(res, value.(*ptypes.RegistryRepository))
  382. return true
  383. })
  384. return res, nil
  385. }
  386. func (r *Registry) listECRRepositories(aws *ints.AWSIntegration) ([]*ptypes.RegistryRepository, error) {
  387. sess, err := aws.GetSession()
  388. if err != nil {
  389. return nil, err
  390. }
  391. svc := ecr.New(sess)
  392. resp, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{})
  393. if err != nil {
  394. return nil, err
  395. }
  396. res := make([]*ptypes.RegistryRepository, 0)
  397. for _, repo := range resp.Repositories {
  398. res = append(res, &ptypes.RegistryRepository{
  399. Name: *repo.RepositoryName,
  400. CreatedAt: *repo.CreatedAt,
  401. URI: *repo.RepositoryUri,
  402. })
  403. }
  404. return res, nil
  405. }
  406. type acrRepositoryResp struct {
  407. Repositories []string `json:"repositories"`
  408. }
  409. func (r *Registry) listACRRepositories(ctx context.Context, repo repository.Repository) ([]*ptypes.RegistryRepository, error) {
  410. ctx, span := telemetry.NewSpan(ctx, "list-acr-repositories")
  411. defer span.End()
  412. telemetry.WithAttributes(span,
  413. telemetry.AttributeKV{Key: "registry-name", Value: r.Name},
  414. telemetry.AttributeKV{Key: "registry-id", Value: r.ID},
  415. telemetry.AttributeKV{Key: "project-id", Value: r.ProjectID},
  416. )
  417. az, err := repo.AzureIntegration().ReadAzureIntegration(
  418. r.ProjectID,
  419. r.AzureIntegrationID,
  420. )
  421. if err != nil {
  422. return nil, telemetry.Error(ctx, span, err, "error reading azure integration")
  423. }
  424. client := &http.Client{}
  425. acrURL := r.URL
  426. if !strings.Contains(acrURL, "http") {
  427. acrURL = fmt.Sprintf("https://%s", acrURL)
  428. }
  429. req, err := http.NewRequest(
  430. "GET",
  431. fmt.Sprintf("%s/v2/_catalog", acrURL),
  432. nil,
  433. )
  434. if err != nil {
  435. return nil, telemetry.Error(ctx, span, err, "error getting http request")
  436. }
  437. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  438. resp, err := client.Do(req)
  439. if err != nil {
  440. return nil, telemetry.Error(ctx, span, err, "error making http call")
  441. }
  442. acrResp := acrRepositoryResp{}
  443. if err := json.NewDecoder(resp.Body).Decode(&acrResp); err != nil {
  444. return nil, telemetry.Error(ctx, span, err, "could not read Azure registry repository response")
  445. }
  446. res := make([]*ptypes.RegistryRepository, 0)
  447. for _, repo := range acrResp.Repositories {
  448. res = append(res, &ptypes.RegistryRepository{
  449. Name: repo,
  450. URI: strings.TrimPrefix(r.URL, "https://") + "/" + repo,
  451. })
  452. }
  453. telemetry.WithAttributes(span, telemetry.AttributeKV{Key: "repo-count", Value: len(acrResp.Repositories)})
  454. return res, nil
  455. }
  456. // Returns the username/password pair for the registry
  457. func (r *Registry) GetACRCredentials(repo repository.Repository) (string, string, error) {
  458. az, err := repo.AzureIntegration().ReadAzureIntegration(
  459. r.ProjectID,
  460. r.AzureIntegrationID,
  461. )
  462. if err != nil {
  463. return "", "", err
  464. }
  465. // if the passwords and name aren't set, generate them
  466. if az.ACRTokenName == "" || len(az.ACRPassword1) == 0 {
  467. az.ACRTokenName = "porter-acr-token"
  468. // create an acr repo token
  469. cred, err := azidentity.NewClientSecretCredential(az.AzureTenantID, az.AzureClientID, string(az.ServicePrincipalSecret), nil)
  470. if err != nil {
  471. return "", "", err
  472. }
  473. scopeMapsClient, err := armcontainerregistry.NewScopeMapsClient(az.AzureSubscriptionID, cred, nil)
  474. if err != nil {
  475. return "", "", err
  476. }
  477. smRes, err := scopeMapsClient.Get(
  478. context.Background(),
  479. az.ACRResourceGroupName,
  480. az.ACRName,
  481. "_repositories_admin",
  482. nil,
  483. )
  484. if err != nil {
  485. return "", "", err
  486. }
  487. tokensClient, err := armcontainerregistry.NewTokensClient(az.AzureSubscriptionID, cred, nil)
  488. if err != nil {
  489. return "", "", err
  490. }
  491. pollerResp, err := tokensClient.BeginCreate(
  492. context.Background(),
  493. az.ACRResourceGroupName,
  494. az.ACRName,
  495. "porter-acr-token",
  496. armcontainerregistry.Token{
  497. Properties: &armcontainerregistry.TokenProperties{
  498. ScopeMapID: smRes.ID,
  499. Status: to.Ptr(armcontainerregistry.TokenStatusEnabled),
  500. },
  501. },
  502. nil,
  503. )
  504. if err != nil {
  505. return "", "", err
  506. }
  507. tokResp, err := pollerResp.PollUntilDone(context.Background(), 2*time.Second)
  508. if err != nil {
  509. return "", "", err
  510. }
  511. registriesClient, err := armcontainerregistry.NewRegistriesClient(az.AzureSubscriptionID, cred, nil)
  512. if err != nil {
  513. return "", "", err
  514. }
  515. poller, err := registriesClient.BeginGenerateCredentials(
  516. context.Background(),
  517. az.ACRResourceGroupName,
  518. az.ACRName,
  519. armcontainerregistry.GenerateCredentialsParameters{
  520. TokenID: tokResp.ID,
  521. },
  522. &armcontainerregistry.RegistriesClientBeginGenerateCredentialsOptions{ResumeToken: ""})
  523. if err != nil {
  524. return "", "", err
  525. }
  526. genCredentialsResp, err := poller.PollUntilDone(context.Background(), 2*time.Second)
  527. if err != nil {
  528. return "", "", err
  529. }
  530. for i, tokPassword := range genCredentialsResp.Passwords {
  531. if i == 0 {
  532. az.ACRPassword1 = []byte(*tokPassword.Value)
  533. } else if i == 1 {
  534. az.ACRPassword2 = []byte(*tokPassword.Value)
  535. }
  536. }
  537. // update the az integration
  538. az, err = repo.AzureIntegration().OverwriteAzureIntegration(
  539. az,
  540. )
  541. if err != nil {
  542. return "", "", err
  543. }
  544. }
  545. return az.ACRTokenName, string(az.ACRPassword1), nil
  546. }
  547. func (r *Registry) listDOCRRepositories(
  548. repo repository.Repository,
  549. doAuth *oauth2.Config,
  550. ) ([]*ptypes.RegistryRepository, error) {
  551. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  552. r.ProjectID,
  553. r.DOIntegrationID,
  554. )
  555. if err != nil {
  556. return nil, err
  557. }
  558. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  559. if err != nil {
  560. return nil, err
  561. }
  562. client := godo.NewFromToken(tok)
  563. urlArr := strings.Split(r.URL, "/")
  564. if len(urlArr) != 2 {
  565. return nil, fmt.Errorf("invalid digital ocean registry url")
  566. }
  567. name := urlArr[1]
  568. repos, _, err := client.Registry.ListRepositories(context.TODO(), name, &godo.ListOptions{})
  569. if err != nil {
  570. return nil, err
  571. }
  572. res := make([]*ptypes.RegistryRepository, 0)
  573. for _, repo := range repos {
  574. res = append(res, &ptypes.RegistryRepository{
  575. Name: repo.Name,
  576. URI: r.URL + "/" + repo.Name,
  577. })
  578. }
  579. return res, nil
  580. }
  581. func (r *Registry) listPrivateRegistryRepositories(
  582. repo repository.Repository,
  583. ) ([]*ptypes.RegistryRepository, error) {
  584. // handle dockerhub different, as it doesn't implement the docker registry http api
  585. if strings.Contains(r.URL, "docker.io") {
  586. // in this case, we just return the single dockerhub repository that's linked
  587. res := make([]*ptypes.RegistryRepository, 0)
  588. res = append(res, &ptypes.RegistryRepository{
  589. Name: strings.Split(r.URL, "docker.io/")[1],
  590. URI: r.URL,
  591. })
  592. return res, nil
  593. }
  594. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  595. r.ProjectID,
  596. r.BasicIntegrationID,
  597. )
  598. if err != nil {
  599. return nil, err
  600. }
  601. // Just use service account key to authenticate, since scopes may not be in place
  602. // for oauth. This also prevents us from making more requests.
  603. client := &http.Client{}
  604. // get the host and scheme to make the request
  605. parsedURL, err := url.Parse(r.URL)
  606. req, err := http.NewRequest(
  607. "GET",
  608. fmt.Sprintf("%s://%s/v2/_catalog", parsedURL.Scheme, parsedURL.Host),
  609. nil,
  610. )
  611. if err != nil {
  612. return nil, err
  613. }
  614. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  615. resp, err := client.Do(req)
  616. if err != nil {
  617. return nil, err
  618. }
  619. // if the status code is 404, fallback to the Docker Hub implementation
  620. if resp.StatusCode == 404 {
  621. req, err := http.NewRequest(
  622. "GET",
  623. fmt.Sprintf("%s/", r.URL),
  624. nil,
  625. )
  626. if err != nil {
  627. return nil, err
  628. }
  629. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  630. resp, err = client.Do(req)
  631. if err != nil {
  632. return nil, err
  633. }
  634. }
  635. gcrResp := gcrRepositoryResp{}
  636. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  637. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  638. }
  639. res := make([]*ptypes.RegistryRepository, 0)
  640. if err != nil {
  641. return nil, err
  642. }
  643. for _, repo := range gcrResp.Repositories {
  644. res = append(res, &ptypes.RegistryRepository{
  645. Name: repo,
  646. URI: parsedURL.Host + "/" + repo,
  647. })
  648. }
  649. return res, nil
  650. }
  651. func (r *Registry) getTokenCacheFunc(
  652. repo repository.Repository,
  653. ) ints.GetTokenCacheFunc {
  654. return func() (tok *ints.TokenCache, err error) {
  655. reg, err := repo.Registry().ReadRegistry(r.ProjectID, r.ID)
  656. if err != nil {
  657. return nil, err
  658. }
  659. return &reg.TokenCache.TokenCache, nil
  660. }
  661. }
  662. func (r *Registry) setTokenCacheFunc(
  663. repo repository.Repository,
  664. ) ints.SetTokenCacheFunc {
  665. return func(token string, expiry time.Time) error {
  666. _, err := repo.Registry().UpdateRegistryTokenCache(
  667. &ints.RegTokenCache{
  668. TokenCache: ints.TokenCache{
  669. Token: []byte(token),
  670. Expiry: expiry,
  671. },
  672. RegistryID: r.ID,
  673. },
  674. )
  675. return err
  676. }
  677. }
  678. // CreateRepository creates a repository for a registry, if needed
  679. // (currently only required for ECR)
  680. func (r *Registry) CreateRepository(
  681. ctx context.Context,
  682. conf *config.Config,
  683. name string,
  684. ) error {
  685. // if aws, create repository
  686. if r.AWSIntegrationID != 0 {
  687. aws, err := conf.Repo.AWSIntegration().ReadAWSIntegration(
  688. r.ProjectID,
  689. r.AWSIntegrationID,
  690. )
  691. if err != nil {
  692. return err
  693. }
  694. return r.createECRRepository(aws, name)
  695. } else if r.GCPIntegrationID != 0 && strings.Contains(r.URL, "pkg.dev") {
  696. return r.createGARRepository(conf.Repo, name)
  697. }
  698. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  699. if err != nil {
  700. return fmt.Errorf("error getting project for repository: %w", err)
  701. }
  702. if project.CapiProvisionerEnabled {
  703. // no need to create repository if pushing to ACR
  704. if strings.Contains(r.URL, ".azurecr.") {
  705. return nil
  706. }
  707. uri := strings.TrimPrefix(r.URL, "https://")
  708. splits := strings.Split(uri, ".")
  709. accountID := splits[0]
  710. region := splits[3]
  711. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  712. ProjectId: int64(r.ProjectID),
  713. AwsAccountId: accountID,
  714. })
  715. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  716. if err != nil {
  717. return fmt.Errorf("error getting capi credentials for repository: %w", err)
  718. }
  719. aws := &ints.AWSIntegration{
  720. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  721. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  722. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  723. AWSRegion: region,
  724. }
  725. return r.createECRRepository(aws, name)
  726. }
  727. // otherwise, no-op
  728. return nil
  729. }
  730. func (r *Registry) createECRRepository(
  731. aws *ints.AWSIntegration,
  732. name string,
  733. ) error {
  734. sess, err := aws.GetSession()
  735. if err != nil {
  736. return err
  737. }
  738. svc := ecr.New(sess)
  739. // determine if repository already exists
  740. _, err = svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{
  741. RepositoryNames: []*string{&name},
  742. })
  743. // if the repository was not found, create it
  744. if aerr, ok := err.(awserr.Error); ok && aerr.Code() == ecr.ErrCodeRepositoryNotFoundException {
  745. _, err = svc.CreateRepository(&ecr.CreateRepositoryInput{
  746. RepositoryName: &name,
  747. })
  748. return err
  749. } else if err != nil {
  750. return err
  751. }
  752. return nil
  753. }
  754. func (r *Registry) createGARRepository(
  755. repo repository.Repository,
  756. name string,
  757. ) error {
  758. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  759. r.ProjectID,
  760. r.GCPIntegrationID,
  761. )
  762. if err != nil {
  763. return err
  764. }
  765. client, err := artifactregistry.NewClient(context.Background(), option.WithTokenSource(&garTokenSource{
  766. reg: r,
  767. repo: repo,
  768. }), option.WithScopes("roles/artifactregistry.admin"))
  769. if err != nil {
  770. return err
  771. }
  772. defer client.Close()
  773. parsedURL, err := url.Parse("https://" + r.URL)
  774. if err != nil {
  775. return err
  776. }
  777. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  778. _, err = client.GetRepository(context.Background(), &artifactregistrypb.GetRepositoryRequest{
  779. Name: fmt.Sprintf("projects/%s/locations/%s/repositories/%s", gcpInt.GCPProjectID, location, name),
  780. })
  781. if err != nil && strings.Contains(err.Error(), "not found") {
  782. // create a new repository
  783. _, err := client.CreateRepository(context.Background(), &artifactregistrypb.CreateRepositoryRequest{
  784. Parent: fmt.Sprintf("projects/%s/locations/%s", gcpInt.GCPProjectID, location),
  785. RepositoryId: name,
  786. Repository: &artifactregistrypb.Repository{
  787. Format: artifactregistrypb.Repository_DOCKER,
  788. },
  789. })
  790. if err != nil {
  791. return err
  792. }
  793. } else if err != nil {
  794. return err
  795. }
  796. return nil
  797. }
  798. // ListImages lists the images for an image repository
  799. func (r *Registry) ListImages(
  800. ctx context.Context,
  801. repoName string,
  802. repo repository.Repository,
  803. conf *config.Config,
  804. ) ([]*ptypes.Image, error) {
  805. // switch on the auth mechanism to get a token
  806. if r.AWSIntegrationID != 0 {
  807. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  808. r.ProjectID,
  809. r.AWSIntegrationID,
  810. )
  811. if err != nil {
  812. return nil, err
  813. }
  814. return r.listECRImages(aws, repoName, repo)
  815. }
  816. if r.AzureIntegrationID != 0 {
  817. return r.listACRImages(repoName, repo)
  818. }
  819. if r.GCPIntegrationID != 0 {
  820. if strings.Contains(r.URL, "pkg.dev") {
  821. return r.listGARImages(repoName, repo)
  822. }
  823. return r.listGCRImages(repoName, repo)
  824. }
  825. if r.DOIntegrationID != 0 {
  826. return r.listDOCRImages(repoName, repo, conf.DOConf)
  827. }
  828. if r.BasicIntegrationID != 0 {
  829. return r.listPrivateRegistryImages(repoName, repo)
  830. }
  831. project, err := conf.Repo.Project().ReadProject(r.ProjectID)
  832. if err != nil {
  833. return nil, fmt.Errorf("error getting project for repository: %w", err)
  834. }
  835. if project.CapiProvisionerEnabled {
  836. uri := strings.TrimPrefix(r.URL, "https://")
  837. splits := strings.Split(uri, ".")
  838. accountID := splits[0]
  839. region := splits[3]
  840. req := connect.NewRequest(&porterv1.AssumeRoleCredentialsRequest{
  841. ProjectId: int64(r.ProjectID),
  842. AwsAccountId: accountID,
  843. })
  844. creds, err := conf.ClusterControlPlaneClient.AssumeRoleCredentials(ctx, req)
  845. if err != nil {
  846. return nil, fmt.Errorf("error getting capi credentials for repository: %w", err)
  847. }
  848. aws := &ints.AWSIntegration{
  849. AWSAccessKeyID: []byte(creds.Msg.AwsAccessId),
  850. AWSSecretAccessKey: []byte(creds.Msg.AwsSecretKey),
  851. AWSSessionToken: []byte(creds.Msg.AwsSessionToken),
  852. AWSRegion: region,
  853. }
  854. return r.listECRImages(aws, repoName, repo)
  855. }
  856. return nil, fmt.Errorf("error listing images")
  857. }
  858. func (r *Registry) GetECRPaginatedImages(
  859. repoName string,
  860. repo repository.Repository,
  861. maxResults int64,
  862. nextToken *string,
  863. ) ([]*ptypes.Image, *string, error) {
  864. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  865. r.ProjectID,
  866. r.AWSIntegrationID,
  867. )
  868. if err != nil {
  869. return nil, nil, err
  870. }
  871. sess, err := aws.GetSession()
  872. if err != nil {
  873. return nil, nil, err
  874. }
  875. svc := ecr.New(sess)
  876. resp, err := svc.ListImages(&ecr.ListImagesInput{
  877. RepositoryName: &repoName,
  878. MaxResults: &maxResults,
  879. NextToken: nextToken,
  880. })
  881. if err != nil {
  882. return nil, nil, err
  883. }
  884. if len(resp.ImageIds) == 0 {
  885. return []*ptypes.Image{}, nil, nil
  886. }
  887. imageIDLen := len(resp.ImageIds)
  888. imageDetails := make([]*ecr.ImageDetail, 0)
  889. imageIDMap := make(map[string]bool)
  890. for _, id := range resp.ImageIds {
  891. if id != nil && id.ImageTag != nil {
  892. imageIDMap[*id.ImageTag] = true
  893. }
  894. }
  895. var wg sync.WaitGroup
  896. var mu sync.Mutex
  897. // AWS API expects the length of imageIDs to be at max 100 at a time
  898. for start := 0; start < imageIDLen; start += 100 {
  899. end := start + 100
  900. if end > imageIDLen {
  901. end = imageIDLen
  902. }
  903. wg.Add(1)
  904. go func(start, end int) {
  905. defer wg.Done()
  906. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  907. RepositoryName: &repoName,
  908. ImageIds: resp.ImageIds[start:end],
  909. })
  910. if err != nil {
  911. return
  912. }
  913. mu.Lock()
  914. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  915. mu.Unlock()
  916. }(start, end)
  917. }
  918. wg.Wait()
  919. res := make([]*ptypes.Image, 0)
  920. imageInfoMap := make(map[string]*ptypes.Image)
  921. for _, img := range imageDetails {
  922. for _, tag := range img.ImageTags {
  923. newImage := &ptypes.Image{
  924. Digest: *img.ImageDigest,
  925. Tag: *tag,
  926. RepositoryName: repoName,
  927. PushedAt: img.ImagePushedAt,
  928. }
  929. if _, ok := imageIDMap[*tag]; ok {
  930. if _, ok := imageInfoMap[*tag]; !ok {
  931. imageInfoMap[*tag] = newImage
  932. }
  933. }
  934. if len(imageInfoMap) == int(maxResults) {
  935. break
  936. }
  937. }
  938. if len(imageInfoMap) == int(maxResults) {
  939. break
  940. }
  941. }
  942. for _, v := range imageInfoMap {
  943. res = append(res, v)
  944. }
  945. sort.Slice(res, func(i, j int) bool {
  946. if res[i].PushedAt == nil || res[j].PushedAt == nil {
  947. return false
  948. }
  949. return res[i].PushedAt.After(*res[j].PushedAt)
  950. })
  951. return res, resp.NextToken, nil
  952. }
  953. func (r *Registry) listECRImages(aws *ints.AWSIntegration, repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  954. sess, err := aws.GetSession()
  955. if err != nil {
  956. return nil, err
  957. }
  958. svc := ecr.New(sess)
  959. maxResults := int64(1000)
  960. var imageIDs []*ecr.ImageIdentifier
  961. resp, err := svc.ListImages(&ecr.ListImagesInput{
  962. RepositoryName: &repoName,
  963. MaxResults: &maxResults,
  964. })
  965. if err != nil {
  966. return nil, err
  967. }
  968. if len(resp.ImageIds) == 0 {
  969. return []*ptypes.Image{}, nil
  970. }
  971. imageIDs = append(imageIDs, resp.ImageIds...)
  972. nextToken := resp.NextToken
  973. for nextToken != nil {
  974. resp, err := svc.ListImages(&ecr.ListImagesInput{
  975. RepositoryName: &repoName,
  976. MaxResults: &maxResults,
  977. NextToken: nextToken,
  978. })
  979. if err != nil {
  980. return nil, err
  981. }
  982. imageIDs = append(imageIDs, resp.ImageIds...)
  983. nextToken = resp.NextToken
  984. }
  985. imageIDLen := len(imageIDs)
  986. imageDetails := make([]*ecr.ImageDetail, 0)
  987. var wg sync.WaitGroup
  988. var mu sync.Mutex
  989. // AWS API expects the length of imageIDs to be at max 100 at a time
  990. for start := 0; start < imageIDLen; start += 100 {
  991. end := start + 100
  992. if end > imageIDLen {
  993. end = imageIDLen
  994. }
  995. wg.Add(1)
  996. go func(start, end int) {
  997. defer wg.Done()
  998. describeResp, err := svc.DescribeImages(&ecr.DescribeImagesInput{
  999. RepositoryName: &repoName,
  1000. ImageIds: imageIDs[start:end],
  1001. })
  1002. if err != nil {
  1003. return
  1004. }
  1005. mu.Lock()
  1006. imageDetails = append(imageDetails, describeResp.ImageDetails...)
  1007. mu.Unlock()
  1008. }(start, end)
  1009. }
  1010. wg.Wait()
  1011. res := make([]*ptypes.Image, 0)
  1012. imageInfoMap := make(map[string]*ptypes.Image)
  1013. for _, img := range imageDetails {
  1014. for _, tag := range img.ImageTags {
  1015. newImage := &ptypes.Image{
  1016. Digest: *img.ImageDigest,
  1017. Tag: *tag,
  1018. RepositoryName: repoName,
  1019. PushedAt: img.ImagePushedAt,
  1020. }
  1021. if _, ok := imageInfoMap[*tag]; !ok {
  1022. imageInfoMap[*tag] = newImage
  1023. }
  1024. }
  1025. }
  1026. for _, v := range imageInfoMap {
  1027. res = append(res, v)
  1028. }
  1029. return res, nil
  1030. }
  1031. func (r *Registry) listACRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1032. az, err := repo.AzureIntegration().ReadAzureIntegration(
  1033. r.ProjectID,
  1034. r.AzureIntegrationID,
  1035. )
  1036. if err != nil {
  1037. return nil, err
  1038. }
  1039. // use JWT token to request catalog
  1040. client := &http.Client{}
  1041. req, err := http.NewRequest(
  1042. "GET",
  1043. fmt.Sprintf("%s/v2/%s/tags/list", r.URL, repoName),
  1044. nil,
  1045. )
  1046. if err != nil {
  1047. return nil, err
  1048. }
  1049. req.SetBasicAuth(az.AzureClientID, string(az.ServicePrincipalSecret))
  1050. resp, err := client.Do(req)
  1051. if err != nil {
  1052. return nil, err
  1053. }
  1054. gcrResp := gcrImageResp{}
  1055. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1056. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1057. }
  1058. res := make([]*ptypes.Image, 0)
  1059. for _, tag := range gcrResp.Tags {
  1060. res = append(res, &ptypes.Image{
  1061. RepositoryName: strings.TrimPrefix(repoName, "https://"),
  1062. Tag: tag,
  1063. })
  1064. }
  1065. return res, nil
  1066. }
  1067. type gcrImageResp struct {
  1068. Tags []string `json:"tags"`
  1069. }
  1070. func (r *Registry) listGCRImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1071. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1072. r.ProjectID,
  1073. r.GCPIntegrationID,
  1074. )
  1075. if err != nil {
  1076. return nil, err
  1077. }
  1078. // use JWT token to request catalog
  1079. client := &http.Client{}
  1080. parsedURL, err := url.Parse("https://" + r.URL)
  1081. if err != nil {
  1082. return nil, err
  1083. }
  1084. trimmedPath := strings.Trim(parsedURL.Path, "/")
  1085. req, err := http.NewRequest(
  1086. "GET",
  1087. fmt.Sprintf("https://%s/v2/%s/%s/tags/list", parsedURL.Host, trimmedPath, repoName),
  1088. nil,
  1089. )
  1090. if err != nil {
  1091. return nil, err
  1092. }
  1093. req.SetBasicAuth("_json_key", string(gcp.GCPKeyData))
  1094. resp, err := client.Do(req)
  1095. if err != nil {
  1096. return nil, err
  1097. }
  1098. gcrResp := gcrImageResp{}
  1099. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1100. return nil, fmt.Errorf("Could not read GCR repositories: %v", err)
  1101. }
  1102. res := make([]*ptypes.Image, 0)
  1103. for _, tag := range gcrResp.Tags {
  1104. res = append(res, &ptypes.Image{
  1105. RepositoryName: repoName,
  1106. Tag: tag,
  1107. })
  1108. }
  1109. return res, nil
  1110. }
  1111. func (r *Registry) listGARImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1112. repoImageSlice := strings.Split(repoName, "/")
  1113. if len(repoImageSlice) != 2 {
  1114. return nil, fmt.Errorf("invalid GAR repo name: %s. Expected to be in the form of REPOSITORY/IMAGE", repoName)
  1115. }
  1116. gcpInt, err := repo.GCPIntegration().ReadGCPIntegration(
  1117. r.ProjectID,
  1118. r.GCPIntegrationID,
  1119. )
  1120. if err != nil {
  1121. return nil, err
  1122. }
  1123. svc, err := v1artifactregistry.NewService(context.Background(), option.WithTokenSource(&garTokenSource{
  1124. reg: r,
  1125. repo: repo,
  1126. }), option.WithScopes("roles/artifactregistry.reader"))
  1127. if err != nil {
  1128. return nil, err
  1129. }
  1130. var res []*ptypes.Image
  1131. parsedURL, err := url.Parse("https://" + r.URL)
  1132. if err != nil {
  1133. return nil, err
  1134. }
  1135. location := strings.TrimSuffix(parsedURL.Host, "-docker.pkg.dev")
  1136. dockerSvc := v1artifactregistry.NewProjectsLocationsRepositoriesDockerImagesService(svc)
  1137. nextToken := ""
  1138. for {
  1139. resp, err := dockerSvc.List(fmt.Sprintf("projects/%s/locations/%s/repositories/%s",
  1140. gcpInt.GCPProjectID, location, repoImageSlice[0])).PageSize(1000).PageToken(nextToken).Do()
  1141. if err != nil {
  1142. return nil, err
  1143. }
  1144. for _, image := range resp.DockerImages {
  1145. named, err := reference.ParseNamed(image.Uri)
  1146. if err != nil {
  1147. continue
  1148. }
  1149. paths := strings.Split(reference.Path(named), "/")
  1150. imageName := paths[len(paths)-1]
  1151. if imageName == repoImageSlice[1] {
  1152. uploadTime, _ := time.Parse(time.RFC3339, image.UploadTime)
  1153. for _, tag := range image.Tags {
  1154. res = append(res, &ptypes.Image{
  1155. RepositoryName: repoName,
  1156. Tag: tag,
  1157. PushedAt: &uploadTime,
  1158. Digest: strings.Split(image.Uri, "@")[1],
  1159. })
  1160. }
  1161. }
  1162. }
  1163. if resp.NextPageToken == "" {
  1164. break
  1165. }
  1166. nextToken = resp.NextPageToken
  1167. }
  1168. return res, nil
  1169. }
  1170. func (r *Registry) listDOCRImages(
  1171. repoName string,
  1172. repo repository.Repository,
  1173. doAuth *oauth2.Config,
  1174. ) ([]*ptypes.Image, error) {
  1175. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1176. r.ProjectID,
  1177. r.DOIntegrationID,
  1178. )
  1179. if err != nil {
  1180. return nil, err
  1181. }
  1182. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1183. if err != nil {
  1184. return nil, err
  1185. }
  1186. client := godo.NewFromToken(tok)
  1187. urlArr := strings.Split(r.URL, "/")
  1188. if len(urlArr) != 2 {
  1189. return nil, fmt.Errorf("invalid digital ocean registry url")
  1190. }
  1191. name := urlArr[1]
  1192. var tags []*godo.RepositoryTag
  1193. opt := &godo.ListOptions{
  1194. PerPage: 200,
  1195. }
  1196. for {
  1197. nextTags, resp, err := client.Registry.ListRepositoryTags(context.TODO(), name, repoName, opt)
  1198. if err != nil {
  1199. return nil, err
  1200. }
  1201. tags = append(tags, nextTags...)
  1202. if resp.Links == nil || resp.Links.IsLastPage() {
  1203. break
  1204. }
  1205. page, err := resp.Links.CurrentPage()
  1206. if err != nil {
  1207. return nil, err
  1208. }
  1209. opt.Page = page + 1
  1210. }
  1211. res := make([]*ptypes.Image, 0)
  1212. for _, tag := range tags {
  1213. res = append(res, &ptypes.Image{
  1214. RepositoryName: repoName,
  1215. Tag: tag.Tag,
  1216. })
  1217. }
  1218. return res, nil
  1219. }
  1220. func (r *Registry) listPrivateRegistryImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1221. // handle dockerhub different, as it doesn't implement the docker registry http api
  1222. if strings.Contains(r.URL, "docker.io") {
  1223. return r.listDockerHubImages(repoName, repo)
  1224. }
  1225. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1226. r.ProjectID,
  1227. r.BasicIntegrationID,
  1228. )
  1229. if err != nil {
  1230. return nil, err
  1231. }
  1232. // Just use service account key to authenticate, since scopes may not be in place
  1233. // for oauth. This also prevents us from making more requests.
  1234. client := &http.Client{}
  1235. // get the host and scheme to make the request
  1236. parsedURL, err := url.Parse(r.URL)
  1237. req, err := http.NewRequest(
  1238. "GET",
  1239. fmt.Sprintf("%s://%s/v2/%s/tags/list", parsedURL.Scheme, parsedURL.Host, repoName),
  1240. nil,
  1241. )
  1242. if err != nil {
  1243. return nil, err
  1244. }
  1245. req.SetBasicAuth(string(basic.Username), string(basic.Password))
  1246. resp, err := client.Do(req)
  1247. if err != nil {
  1248. return nil, err
  1249. }
  1250. gcrResp := gcrImageResp{}
  1251. if err := json.NewDecoder(resp.Body).Decode(&gcrResp); err != nil {
  1252. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1253. }
  1254. res := make([]*ptypes.Image, 0)
  1255. for _, tag := range gcrResp.Tags {
  1256. res = append(res, &ptypes.Image{
  1257. RepositoryName: repoName,
  1258. Tag: tag,
  1259. })
  1260. }
  1261. return res, nil
  1262. }
  1263. type dockerHubImageResult struct {
  1264. Name string `json:"name"`
  1265. }
  1266. type dockerHubImageResp struct {
  1267. Results []dockerHubImageResult `json:"results"`
  1268. }
  1269. type dockerHubLoginReq struct {
  1270. Username string `json:"username"`
  1271. Password string `json:"password"`
  1272. }
  1273. type dockerHubLoginResp struct {
  1274. Token string `json:"token"`
  1275. }
  1276. func (r *Registry) listDockerHubImages(repoName string, repo repository.Repository) ([]*ptypes.Image, error) {
  1277. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1278. r.ProjectID,
  1279. r.BasicIntegrationID,
  1280. )
  1281. if err != nil {
  1282. return nil, err
  1283. }
  1284. client := &http.Client{}
  1285. // first, make a request for the access token
  1286. data, err := json.Marshal(&dockerHubLoginReq{
  1287. Username: string(basic.Username),
  1288. Password: string(basic.Password),
  1289. })
  1290. if err != nil {
  1291. return nil, err
  1292. }
  1293. req, err := http.NewRequest(
  1294. "POST",
  1295. "https://hub.docker.com/v2/users/login",
  1296. strings.NewReader(string(data)),
  1297. )
  1298. if err != nil {
  1299. return nil, err
  1300. }
  1301. req.Header.Add("Content-Type", "application/json")
  1302. resp, err := client.Do(req)
  1303. if err != nil {
  1304. return nil, err
  1305. }
  1306. tokenObj := dockerHubLoginResp{}
  1307. if err := json.NewDecoder(resp.Body).Decode(&tokenObj); err != nil {
  1308. return nil, fmt.Errorf("Could not decode Dockerhub token from response: %v", err)
  1309. }
  1310. req, err = http.NewRequest(
  1311. "GET",
  1312. fmt.Sprintf("https://hub.docker.com/v2/repositories/%s/tags", strings.Split(r.URL, "docker.io/")[1]),
  1313. nil,
  1314. )
  1315. if err != nil {
  1316. return nil, err
  1317. }
  1318. req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tokenObj.Token))
  1319. resp, err = client.Do(req)
  1320. if err != nil {
  1321. return nil, err
  1322. }
  1323. imageResp := dockerHubImageResp{}
  1324. if err := json.NewDecoder(resp.Body).Decode(&imageResp); err != nil {
  1325. return nil, fmt.Errorf("Could not read private registry repositories: %v", err)
  1326. }
  1327. res := make([]*ptypes.Image, 0)
  1328. for _, result := range imageResp.Results {
  1329. res = append(res, &ptypes.Image{
  1330. RepositoryName: repoName,
  1331. Tag: result.Name,
  1332. })
  1333. }
  1334. return res, nil
  1335. }
  1336. // GetDockerConfigJSON returns a dockerconfigjson file contents with "auths"
  1337. // populated.
  1338. func (r *Registry) GetDockerConfigJSON(
  1339. repo repository.Repository,
  1340. doAuth *oauth2.Config, // only required if using DOCR
  1341. ) ([]byte, error) {
  1342. var conf *configfile.ConfigFile
  1343. var err error
  1344. // switch on the auth mechanism to get a token
  1345. if r.AWSIntegrationID != 0 {
  1346. conf, err = r.getECRDockerConfigFile(repo)
  1347. }
  1348. if r.GCPIntegrationID != 0 {
  1349. conf, err = r.getGCRDockerConfigFile(repo)
  1350. }
  1351. if r.DOIntegrationID != 0 {
  1352. conf, err = r.getDOCRDockerConfigFile(repo, doAuth)
  1353. }
  1354. if r.BasicIntegrationID != 0 {
  1355. conf, err = r.getPrivateRegistryDockerConfigFile(repo)
  1356. }
  1357. if r.AzureIntegrationID != 0 {
  1358. conf, err = r.getACRDockerConfigFile(repo)
  1359. }
  1360. if err != nil {
  1361. return nil, err
  1362. }
  1363. return json.Marshal(conf)
  1364. }
  1365. func (r *Registry) getECRDockerConfigFile(
  1366. repo repository.Repository,
  1367. ) (*configfile.ConfigFile, error) {
  1368. aws, err := repo.AWSIntegration().ReadAWSIntegration(
  1369. r.ProjectID,
  1370. r.AWSIntegrationID,
  1371. )
  1372. if err != nil {
  1373. return nil, err
  1374. }
  1375. sess, err := aws.GetSession()
  1376. if err != nil {
  1377. return nil, err
  1378. }
  1379. ecrSvc := ecr.New(sess)
  1380. output, err := ecrSvc.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
  1381. if err != nil {
  1382. return nil, err
  1383. }
  1384. token := *output.AuthorizationData[0].AuthorizationToken
  1385. decodedToken, err := base64.StdEncoding.DecodeString(token)
  1386. if err != nil {
  1387. return nil, err
  1388. }
  1389. parts := strings.SplitN(string(decodedToken), ":", 2)
  1390. if len(parts) < 2 {
  1391. return nil, err
  1392. }
  1393. key := r.URL
  1394. if !strings.Contains(key, "http") {
  1395. key = "https://" + key
  1396. }
  1397. return &configfile.ConfigFile{
  1398. AuthConfigs: map[string]types.AuthConfig{
  1399. key: {
  1400. Username: parts[0],
  1401. Password: parts[1],
  1402. Auth: token,
  1403. },
  1404. },
  1405. }, nil
  1406. }
  1407. func (r *Registry) getGCRDockerConfigFile(
  1408. repo repository.Repository,
  1409. ) (*configfile.ConfigFile, error) {
  1410. gcp, err := repo.GCPIntegration().ReadGCPIntegration(
  1411. r.ProjectID,
  1412. r.GCPIntegrationID,
  1413. )
  1414. if err != nil {
  1415. return nil, err
  1416. }
  1417. key := r.URL
  1418. if !strings.Contains(key, "http") {
  1419. key = "https://" + key
  1420. }
  1421. parsedURL, _ := url.Parse(key)
  1422. return &configfile.ConfigFile{
  1423. AuthConfigs: map[string]types.AuthConfig{
  1424. parsedURL.Host: {
  1425. Username: "_json_key",
  1426. Password: string(gcp.GCPKeyData),
  1427. Auth: generateAuthToken("_json_key", string(gcp.GCPKeyData)),
  1428. },
  1429. },
  1430. }, nil
  1431. }
  1432. func (r *Registry) getDOCRDockerConfigFile(
  1433. repo repository.Repository,
  1434. doAuth *oauth2.Config,
  1435. ) (*configfile.ConfigFile, error) {
  1436. oauthInt, err := repo.OAuthIntegration().ReadOAuthIntegration(
  1437. r.ProjectID,
  1438. r.DOIntegrationID,
  1439. )
  1440. if err != nil {
  1441. return nil, err
  1442. }
  1443. tok, _, err := oauth.GetAccessToken(oauthInt.SharedOAuthModel, doAuth, oauth.MakeUpdateOAuthIntegrationTokenFunction(oauthInt, repo))
  1444. if err != nil {
  1445. return nil, err
  1446. }
  1447. key := r.URL
  1448. if !strings.Contains(key, "http") {
  1449. key = "https://" + key
  1450. }
  1451. parsedURL, _ := url.Parse(key)
  1452. return &configfile.ConfigFile{
  1453. AuthConfigs: map[string]types.AuthConfig{
  1454. parsedURL.Host: {
  1455. Username: tok,
  1456. Password: tok,
  1457. Auth: generateAuthToken(tok, tok),
  1458. },
  1459. },
  1460. }, nil
  1461. }
  1462. func (r *Registry) getPrivateRegistryDockerConfigFile(
  1463. repo repository.Repository,
  1464. ) (*configfile.ConfigFile, error) {
  1465. basic, err := repo.BasicIntegration().ReadBasicIntegration(
  1466. r.ProjectID,
  1467. r.BasicIntegrationID,
  1468. )
  1469. if err != nil {
  1470. return nil, err
  1471. }
  1472. key := r.URL
  1473. if !strings.Contains(key, "http") {
  1474. key = "https://" + key
  1475. }
  1476. parsedURL, _ := url.Parse(key)
  1477. authConfigKey := parsedURL.Host
  1478. if strings.Contains(r.URL, "index.docker.io") {
  1479. authConfigKey = "https://index.docker.io/v1/"
  1480. }
  1481. return &configfile.ConfigFile{
  1482. AuthConfigs: map[string]types.AuthConfig{
  1483. authConfigKey: {
  1484. Username: string(basic.Username),
  1485. Password: string(basic.Password),
  1486. Auth: generateAuthToken(string(basic.Username), string(basic.Password)),
  1487. },
  1488. },
  1489. }, nil
  1490. }
  1491. func (r *Registry) getACRDockerConfigFile(
  1492. repo repository.Repository,
  1493. ) (*configfile.ConfigFile, error) {
  1494. username, pw, err := r.GetACRCredentials(repo)
  1495. if err != nil {
  1496. return nil, err
  1497. }
  1498. key := r.URL
  1499. if !strings.Contains(key, "http") {
  1500. key = "https://" + key
  1501. }
  1502. parsedURL, _ := url.Parse(key)
  1503. return &configfile.ConfigFile{
  1504. AuthConfigs: map[string]types.AuthConfig{
  1505. parsedURL.Host: {
  1506. Username: string(username),
  1507. Password: string(pw),
  1508. Auth: generateAuthToken(string(username), string(pw)),
  1509. },
  1510. },
  1511. }, nil
  1512. }
  1513. func generateAuthToken(username, password string) string {
  1514. return base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
  1515. }