parse.go 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003
  1. /*
  2. Copyright 2015 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package parser
  14. import (
  15. "errors"
  16. "fmt"
  17. "go/ast"
  18. "go/constant"
  19. "go/token"
  20. gotypes "go/types"
  21. "maps"
  22. "path/filepath"
  23. "reflect"
  24. "slices"
  25. "sort"
  26. "strings"
  27. "time"
  28. "golang.org/x/tools/go/packages"
  29. "k8s.io/gengo/v2/types"
  30. "k8s.io/klog/v2"
  31. )
  32. // Parser lets you add all the go files in all the packages that you care
  33. // about, then constructs the type source data.
  34. type Parser struct {
  35. // Map of package paths to definitions. These keys should be canonical
  36. // Go import paths (example.com/foo/bar) and not local paths (./foo/bar).
  37. goPkgs map[string]*packages.Package
  38. // Keep track of which packages were directly requested (as opposed to
  39. // those which are transitively loaded).
  40. userRequested map[string]bool
  41. // Keep track of which packages have already been scanned for types.
  42. fullyProcessed map[string]bool
  43. // Build tags to set when loading packages.
  44. buildTags []string
  45. // Tracks accumulated parsed files, so we can do position lookups later.
  46. fset *token.FileSet
  47. // All comments from everywhere in every parsed file. This map is keyed by
  48. // the file-line on which the comment block ends, which makes it easy to
  49. // look up comments which immediately precede a given obect (e.g. a type or
  50. // function definition), which is what we almost always want. We need this
  51. // because Go's own ast package does a very poor job of handling comments.
  52. endLineToCommentGroup map[fileLine]*ast.CommentGroup
  53. }
  54. // key type for finding comments.
  55. type fileLine struct {
  56. file string
  57. line int
  58. }
  59. // New constructs a new Parser.
  60. func New() *Parser {
  61. return NewWithOptions(Options{})
  62. }
  63. func NewWithOptions(opts Options) *Parser {
  64. return &Parser{
  65. goPkgs: map[string]*packages.Package{},
  66. userRequested: map[string]bool{},
  67. fullyProcessed: map[string]bool{},
  68. fset: token.NewFileSet(),
  69. endLineToCommentGroup: map[fileLine]*ast.CommentGroup{},
  70. buildTags: opts.BuildTags,
  71. }
  72. }
  73. // Options holds optional settings for the Parser.
  74. type Options struct {
  75. // BuildTags is a list of optional tags to be specified when loading
  76. // packages.
  77. BuildTags []string
  78. }
  79. // FindPackages expands the provided patterns into a list of Go import-paths,
  80. // much like `go list -find`.
  81. func (p *Parser) FindPackages(patterns ...string) ([]string, error) {
  82. return p.findPackages(nil, patterns...)
  83. }
  84. // baseCfg is an optional (may be nil) config which might be injected by tests.
  85. func (p *Parser) findPackages(baseCfg *packages.Config, patterns ...string) ([]string, error) {
  86. toFind := make([]string, 0, len(patterns))
  87. results := make([]string, 0, len(patterns))
  88. for _, pat := range patterns {
  89. if pkg := p.goPkgs[pat]; pkg != nil {
  90. results = append(results, pkg.PkgPath)
  91. } else {
  92. toFind = append(toFind, pat)
  93. }
  94. }
  95. if len(toFind) == 0 {
  96. return results, nil
  97. }
  98. cfg := packages.Config{
  99. Mode: packages.NeedName | packages.NeedFiles,
  100. BuildFlags: []string{"-tags", strings.Join(p.buildTags, ",")},
  101. Tests: false,
  102. }
  103. if baseCfg != nil {
  104. // This is to support tests, e.g. to inject a fake GOPATH or CWD.
  105. cfg.Dir = baseCfg.Dir
  106. cfg.Env = baseCfg.Env
  107. }
  108. pkgs, err := packages.Load(&cfg, toFind...)
  109. if err != nil {
  110. return nil, fmt.Errorf("error loading packages: %w", err)
  111. }
  112. var allErrs []error
  113. for _, pkg := range pkgs {
  114. results = append(results, pkg.PkgPath)
  115. // pkg.Errors is not a slice of `error`, but concrete types. We have
  116. // to iteratively convert each one into `error`.
  117. var errs []error
  118. for _, e := range pkg.Errors {
  119. errs = append(errs, e)
  120. }
  121. if len(errs) > 0 {
  122. allErrs = append(allErrs, fmt.Errorf("error(s) in %q:\n%w", pkg.PkgPath, errors.Join(errs...)))
  123. }
  124. }
  125. if len(allErrs) != 0 {
  126. return nil, errors.Join(allErrs...)
  127. }
  128. return results, nil
  129. }
  130. // LoadPackages loads and parses the specified Go packages. Specifically
  131. // named packages (without a trailing "/...") which do not exist or have no Go
  132. // files are an error.
  133. func (p *Parser) LoadPackages(patterns ...string) error {
  134. _, err := p.loadPackages(patterns...)
  135. return err
  136. }
  137. // LoadPackagesWithConfigForTesting loads and parses the specified Go packages with the
  138. // specified packages.Config as a starting point. This is for testing, and
  139. // only the .Dir and .Env fields of the Config will be considered.
  140. func (p *Parser) LoadPackagesWithConfigForTesting(cfg *packages.Config, patterns ...string) error {
  141. _, err := p.loadPackagesWithConfig(cfg, patterns...)
  142. return err
  143. }
  144. // LoadPackagesTo loads and parses the specified Go packages, and inserts them
  145. // into the specified Universe. It returns the packages which match the
  146. // patterns, but loads all packages and their imports, recursively, into the
  147. // universe. See NewUniverse for more.
  148. func (p *Parser) LoadPackagesTo(u *types.Universe, patterns ...string) ([]*types.Package, error) {
  149. // Load Packages.
  150. pkgs, err := p.loadPackages(patterns...)
  151. if err != nil {
  152. return nil, err
  153. }
  154. // Load types in all packages (it will internally filter).
  155. if err := p.addPkgsToUniverse(pkgs, u); err != nil {
  156. return nil, err
  157. }
  158. // Return the results as gengo types.Packages.
  159. ret := make([]*types.Package, 0, len(pkgs))
  160. for _, pkg := range pkgs {
  161. ret = append(ret, u.Package(pkg.PkgPath))
  162. }
  163. return ret, nil
  164. }
  165. func (p *Parser) loadPackages(patterns ...string) ([]*packages.Package, error) {
  166. return p.loadPackagesWithConfig(nil, patterns...)
  167. }
  168. // baseCfg is an optional (may be nil) config which might be injected by tests.
  169. func (p *Parser) loadPackagesWithConfig(baseCfg *packages.Config, patterns ...string) ([]*packages.Package, error) {
  170. klog.V(5).Infof("loadPackages %q", patterns)
  171. // Loading packages is slow - only do ones we know we have not already done
  172. // (e.g. if a tool calls LoadPackages itself).
  173. existingPkgs, netNewPkgs, err := p.alreadyLoaded(baseCfg, patterns...)
  174. if err != nil {
  175. return nil, err
  176. }
  177. if vlog := klog.V(5); vlog.Enabled() {
  178. if len(existingPkgs) > 0 {
  179. keys := make([]string, 0, len(existingPkgs))
  180. for _, p := range existingPkgs {
  181. keys = append(keys, p.PkgPath)
  182. }
  183. vlog.Infof(" already have: %q", keys)
  184. }
  185. if len(netNewPkgs) > 0 {
  186. vlog.Infof(" to be loaded: %q", netNewPkgs)
  187. }
  188. }
  189. // If these were not user-requested before, they are now.
  190. for _, pkg := range existingPkgs {
  191. if !p.userRequested[pkg.PkgPath] {
  192. p.userRequested[pkg.PkgPath] = true
  193. }
  194. }
  195. for _, pkg := range netNewPkgs {
  196. if !p.userRequested[pkg] {
  197. p.userRequested[pkg] = true
  198. }
  199. }
  200. if len(netNewPkgs) == 0 {
  201. return existingPkgs, nil
  202. }
  203. cfg := packages.Config{
  204. Mode: packages.NeedName |
  205. packages.NeedFiles | packages.NeedImports | packages.NeedDeps |
  206. packages.NeedModule | packages.NeedTypes | packages.NeedSyntax,
  207. BuildFlags: []string{"-tags", strings.Join(p.buildTags, ",")},
  208. Fset: p.fset,
  209. Tests: false,
  210. }
  211. if baseCfg != nil {
  212. // This is to support tests, e.g. to inject a fake GOPATH or CWD.
  213. cfg.Dir = baseCfg.Dir
  214. cfg.Env = baseCfg.Env
  215. }
  216. tBefore := time.Now()
  217. pkgs, err := packages.Load(&cfg, netNewPkgs...)
  218. if err != nil {
  219. return nil, fmt.Errorf("error loading packages: %w", err)
  220. }
  221. klog.V(5).Infof(" loaded %d pkg(s) in %v", len(pkgs), time.Since(tBefore))
  222. // Handle any errors.
  223. collectErrors := func(pkg *packages.Package) error {
  224. var errs []error
  225. for _, e := range pkg.Errors {
  226. if e.Kind == packages.ListError || e.Kind == packages.ParseError {
  227. errs = append(errs, e)
  228. }
  229. }
  230. if len(errs) > 0 {
  231. return fmt.Errorf("error(s) in %q:\n%w", pkg.PkgPath, errors.Join(errs...))
  232. }
  233. return nil
  234. }
  235. if err := forEachPackageRecursive(pkgs, collectErrors); err != nil {
  236. return nil, err
  237. }
  238. // Finish integrating packages into our state.
  239. absorbPkg := func(pkg *packages.Package) error {
  240. p.goPkgs[pkg.PkgPath] = pkg
  241. for _, f := range pkg.Syntax {
  242. for _, c := range f.Comments {
  243. // We need to do this on _every_ pkg, not just user-requested
  244. // ones, because some generators look at tags in other
  245. // packages.
  246. //
  247. // TODO: It would be nice if we only did this on user-requested
  248. // packages. The problem is that we don't always know which
  249. // other packages will need this information, and even when we
  250. // do we may have already loaded the package (as a transitive
  251. // dep) and might have stored pointers into it. Doing a
  252. // thorough "reload" without invalidating all those pointers is
  253. // a problem for another day.
  254. position := p.fset.Position(c.End()) // Fset is synchronized
  255. p.endLineToCommentGroup[fileLine{position.Filename, position.Line}] = c
  256. }
  257. }
  258. return nil
  259. }
  260. if err := forEachPackageRecursive(pkgs, absorbPkg); err != nil {
  261. return nil, err
  262. }
  263. return append(existingPkgs, pkgs...), nil
  264. }
  265. // alreadyLoaded figures out which of the specified patterns have already been loaded
  266. // and which have not, and returns those respectively.
  267. // baseCfg is an optional (may be nil) config which might be injected by tests.
  268. func (p *Parser) alreadyLoaded(baseCfg *packages.Config, patterns ...string) ([]*packages.Package, []string, error) {
  269. existingPkgs := make([]*packages.Package, 0, len(patterns))
  270. netNewPkgs := make([]string, 0, len(patterns))
  271. // Expand and canonicalize the requested patterns. This should be fast.
  272. if pkgPaths, err := p.findPackages(baseCfg, patterns...); err != nil {
  273. return nil, nil, err
  274. } else {
  275. for _, pkgPath := range pkgPaths {
  276. if pkg := p.goPkgs[pkgPath]; pkg != nil {
  277. existingPkgs = append(existingPkgs, pkg)
  278. } else {
  279. netNewPkgs = append(netNewPkgs, pkgPath)
  280. }
  281. }
  282. }
  283. return existingPkgs, netNewPkgs, nil
  284. }
  285. // forEachPackageRecursive will run the provided function on all of the specified
  286. // packages, and on their imports recursively. Errors are accumulated and
  287. // returned as via errors.Join.
  288. func forEachPackageRecursive(pkgs []*packages.Package, fn func(pkg *packages.Package) error) error {
  289. seen := map[string]bool{} // PkgPaths we have already visited
  290. var errs []error
  291. for _, pkg := range pkgs {
  292. errs = append(errs, recursePackage(pkg, fn, seen)...)
  293. }
  294. if len(errs) > 0 {
  295. return errors.Join(errs...)
  296. }
  297. return nil
  298. }
  299. func recursePackage(pkg *packages.Package, fn func(pkg *packages.Package) error, seen map[string]bool) []error {
  300. if seen[pkg.PkgPath] {
  301. return nil
  302. }
  303. var errs []error
  304. seen[pkg.PkgPath] = true
  305. if err := fn(pkg); err != nil {
  306. errs = append(errs, err)
  307. }
  308. for _, imp := range pkg.Imports {
  309. errs = append(errs, recursePackage(imp, fn, seen)...)
  310. }
  311. return errs
  312. }
  313. // UserRequestedPackages fetches a list of the user-imported packages.
  314. func (p *Parser) UserRequestedPackages() []string {
  315. // Iterate packages in a predictable order.
  316. pkgPaths := make([]string, 0, len(p.userRequested))
  317. for k := range p.userRequested {
  318. pkgPaths = append(pkgPaths, string(k))
  319. }
  320. sort.Strings(pkgPaths)
  321. return pkgPaths
  322. }
  323. // NewUniverse finalizes the loaded packages, searches through them for types
  324. // and produces a new Universe. The returned Universe has one types.Package
  325. // entry for each Go package that has been loaded, including all of their
  326. // dependencies, recursively. It also has one entry, whose key is "", which
  327. // represents "builtin" types.
  328. func (p *Parser) NewUniverse() (types.Universe, error) {
  329. u := types.Universe{}
  330. pkgs := []*packages.Package{}
  331. for _, path := range p.UserRequestedPackages() {
  332. pkgs = append(pkgs, p.goPkgs[path])
  333. }
  334. if err := p.addPkgsToUniverse(pkgs, &u); err != nil {
  335. return nil, err
  336. }
  337. return u, nil
  338. }
  339. // minimize returns a copy of lines with "irrelevant" lines removed. This
  340. // includes blank lines and paragraphs starting with "Deprecated:".
  341. func minimize(lines []string) []string {
  342. out := make([]string, 0, len(lines))
  343. inDeprecated := false // paragraph tracking
  344. prevWasBlank := false
  345. for _, line := range lines {
  346. if len(strings.TrimSpace(line)) == 0 {
  347. prevWasBlank = true
  348. inDeprecated = false
  349. continue
  350. }
  351. if inDeprecated {
  352. continue
  353. }
  354. if prevWasBlank && strings.HasPrefix(strings.TrimSpace(line), "Deprecated:") {
  355. prevWasBlank = false
  356. inDeprecated = true
  357. continue
  358. }
  359. prevWasBlank = false
  360. out = append(out, line)
  361. }
  362. return out
  363. }
  364. // addCommentsToType takes any accumulated comment lines prior to obj and
  365. // attaches them to the type t.
  366. func (p *Parser) addCommentsToType(obj gotypes.Object, t *types.Type) {
  367. if newLines, oldLines := p.docComment(obj.Pos()), t.CommentLines; len(newLines) > 0 {
  368. switch {
  369. case reflect.DeepEqual(oldLines, newLines):
  370. // nothing needed
  371. case len(oldLines) == 0:
  372. // no comments associated, or comments match exactly
  373. t.CommentLines = newLines
  374. case isTypeAlias(obj.Type()):
  375. // Ignore mismatched comments from obj because it's an alias.
  376. // This can only be hit if gotypesalias is enabled.
  377. if !reflect.DeepEqual(minimize(oldLines), minimize(newLines)) {
  378. klog.Warningf(
  379. "Mismatched comments on type %v.\n Using comments:\n%s\n Ignoring comments from type alias:\n%s\n",
  380. t.GoType,
  381. formatCommentBlock(oldLines),
  382. formatCommentBlock(newLines),
  383. )
  384. }
  385. case !isTypeAlias(obj.Type()):
  386. // Overwrite existing comments with ones from obj because obj is not an alias.
  387. // If gotypesalias is enabled, this should mean we found the "real"
  388. // type, not an alias. If gotypesalias is disabled, we can end up
  389. // overwriting the "real" comments with an alias's comments, but
  390. // it is not clear if we can assume which one is the "real" one.
  391. t.CommentLines = newLines
  392. if !reflect.DeepEqual(minimize(oldLines), minimize(newLines)) {
  393. klog.Warningf(
  394. "Mismatched comments on type %v.\n Using comments:\n%s\n Ignoring comments from possible type alias:\n%s\n",
  395. t.GoType,
  396. formatCommentBlock(newLines),
  397. formatCommentBlock(oldLines),
  398. )
  399. }
  400. }
  401. }
  402. if newLines, oldLines := p.priorDetachedComment(obj.Pos()), t.SecondClosestCommentLines; len(newLines) > 0 {
  403. switch {
  404. case reflect.DeepEqual(oldLines, newLines):
  405. // nothing needed
  406. case len(oldLines) == 0:
  407. // no comments associated, or comments match exactly
  408. t.SecondClosestCommentLines = newLines
  409. case isTypeAlias(obj.Type()):
  410. // Ignore mismatched comments from obj because it's an alias.
  411. // This can only be hit if gotypesalias is enabled.
  412. if !reflect.DeepEqual(minimize(oldLines), minimize(newLines)) {
  413. // ignore mismatched comments from obj because it's an alias
  414. klog.Warningf(
  415. "Mismatched secondClosestCommentLines on type %v.\n Using comments:\n%s\n Ignoring comments from type alias:\n%s\n",
  416. t.GoType,
  417. formatCommentBlock(oldLines),
  418. formatCommentBlock(newLines),
  419. )
  420. }
  421. case !isTypeAlias(obj.Type()):
  422. // Overwrite existing comments with ones from obj because obj is not an alias.
  423. // If gotypesalias is enabled, this should mean we found the "real"
  424. // type, not an alias. If gotypesalias is disabled, we can end up
  425. // overwriting the "real" comments with an alias's comments, but
  426. // it is not clear if we can assume which one is the "real" one.
  427. t.SecondClosestCommentLines = newLines
  428. if !reflect.DeepEqual(minimize(oldLines), minimize(newLines)) {
  429. klog.Warningf(
  430. "Mismatched secondClosestCommentLines on type %v.\n Using comments:\n%s\n Ignoring comments from possible type alias:\n%s\n",
  431. t.GoType,
  432. formatCommentBlock(newLines),
  433. formatCommentBlock(oldLines),
  434. )
  435. }
  436. }
  437. }
  438. }
  439. func formatCommentBlock(lines []string) string {
  440. return " ```\n " + strings.Join(lines, "\n ") + "\n ```"
  441. }
  442. // packageDir tries to figure out the directory of the specified package.
  443. func packageDir(pkg *packages.Package) (string, error) {
  444. // Sometimes Module is present but has no Dir, e.g. when it is vendored.
  445. if pkg.Module != nil && pkg.Module.Dir != "" {
  446. // NOTE: this will not work if tests are loaded, because Go mutates the
  447. // Package.PkgPath.
  448. subdir := strings.TrimPrefix(pkg.PkgPath, pkg.Module.Path)
  449. return filepath.Join(pkg.Module.Dir, subdir), nil
  450. }
  451. if len(pkg.GoFiles) > 0 {
  452. return filepath.Dir(pkg.GoFiles[0]), nil
  453. }
  454. if len(pkg.IgnoredFiles) > 0 {
  455. return filepath.Dir(pkg.IgnoredFiles[0]), nil
  456. }
  457. return "", fmt.Errorf("can't find package dir for %q - no module info and no Go files", pkg.PkgPath)
  458. }
  459. // addPkgsToUniverse adds the packages, and all of their deps, recursively, to
  460. // the universe and (if needed) searches through them for types.
  461. func (p *Parser) addPkgsToUniverse(pkgs []*packages.Package, u *types.Universe) error {
  462. addOne := func(pkg *packages.Package) error {
  463. if err := p.addPkgToUniverse(pkg, u); err != nil {
  464. return err
  465. }
  466. return nil
  467. }
  468. if err := forEachPackageRecursive(pkgs, addOne); err != nil {
  469. return err
  470. }
  471. return nil
  472. }
  473. // addPkgToUniverse adds one package to the universe and (if needed) searches
  474. // through it for types.
  475. func (p *Parser) addPkgToUniverse(pkg *packages.Package, u *types.Universe) error {
  476. pkgPath := pkg.PkgPath
  477. if p.fullyProcessed[pkgPath] {
  478. return nil
  479. }
  480. // This will get-or-create the Package.
  481. gengoPkg := u.Package(pkgPath)
  482. if gengoPkg.Dir == "" {
  483. // We're keeping this package, though we might not fully process it.
  484. if vlog := klog.V(5); vlog.Enabled() {
  485. why := "user-requested"
  486. if !p.userRequested[pkgPath] {
  487. why = "dependency"
  488. }
  489. vlog.Infof("addPkgToUniverse %q (%s)", pkgPath, why)
  490. }
  491. absPath := ""
  492. if dir, err := packageDir(pkg); err != nil {
  493. return err
  494. } else {
  495. absPath = dir
  496. }
  497. gengoPkg.Path = pkg.PkgPath
  498. gengoPkg.Dir = absPath
  499. }
  500. // If the package was not user-requested, we can stop here.
  501. if !p.userRequested[pkgPath] {
  502. return nil
  503. }
  504. // Mark it as done, so we don't ever re-process it.
  505. p.fullyProcessed[pkgPath] = true
  506. gengoPkg.Name = pkg.Name
  507. // For historical reasons we treat files named "doc.go" specially.
  508. // TODO: It would be nice to not do this and instead treat package
  509. // doc-comments as the "global" config place. This would require changing
  510. // most generators and input files.
  511. for _, f := range pkg.Syntax {
  512. // This gets the filename for the ast.File. Iterating pkg.GoFiles is
  513. // documented as unreliable.
  514. pos := p.fset.Position(f.FileStart)
  515. if filepath.Base(pos.Filename) == "doc.go" {
  516. gengoPkg.Comments = []string{}
  517. for i := range f.Comments {
  518. gengoPkg.Comments = append(gengoPkg.Comments, splitLines(f.Comments[i].Text())...)
  519. }
  520. if f.Doc != nil {
  521. gengoPkg.DocComments = splitLines(f.Doc.Text())
  522. }
  523. }
  524. }
  525. // Walk all the types, recursively and save them for later access.
  526. s := pkg.Types.Scope()
  527. for _, n := range s.Names() {
  528. switch obj := s.Lookup(n).(type) {
  529. case *gotypes.TypeName:
  530. t := p.walkType(*u, nil, obj.Type())
  531. p.addCommentsToType(obj, t)
  532. case *gotypes.Func:
  533. // We only care about functions, not concrete/abstract methods.
  534. if obj.Type() != nil && obj.Type().(*gotypes.Signature).Recv() == nil {
  535. t := p.addFunction(*u, nil, obj)
  536. p.addCommentsToType(obj, t)
  537. }
  538. case *gotypes.Var:
  539. if !obj.IsField() {
  540. t := p.addVariable(*u, nil, obj)
  541. p.addCommentsToType(obj, t)
  542. }
  543. case *gotypes.Const:
  544. t := p.addConstant(*u, nil, obj)
  545. p.addCommentsToType(obj, t)
  546. default:
  547. klog.Infof("addPkgToUniverse %q: unhandled object of type %T: %v", pkgPath, obj, obj)
  548. }
  549. }
  550. // Add all of this package's imports.
  551. importedPkgs := []string{}
  552. // Iterate imports in a predictable order
  553. for _, key := range slices.Sorted(maps.Keys(pkg.Imports)) {
  554. imp := pkg.Imports[key]
  555. if err := p.addPkgToUniverse(imp, u); err != nil {
  556. return err
  557. }
  558. importedPkgs = append(importedPkgs, imp.PkgPath)
  559. }
  560. sort.Strings(importedPkgs)
  561. u.AddImports(pkg.PkgPath, importedPkgs...)
  562. return nil
  563. }
  564. // If the specified position has a "doc comment", return that.
  565. func (p *Parser) docComment(pos token.Pos) []string {
  566. // An object's doc comment always ends on the line before the object's own
  567. // declaration.
  568. c1 := p.priorCommentLines(pos, 1)
  569. return splitLines(c1.Text()) // safe even if c1 is nil
  570. }
  571. // If there is a detached (not immediately before a declaration) comment,
  572. // return that.
  573. func (p *Parser) priorDetachedComment(pos token.Pos) []string {
  574. // An object's doc comment always ends on the line before the object's own
  575. // declaration.
  576. c1 := p.priorCommentLines(pos, 1)
  577. // Using a literal "2" here is brittle in theory (it means literally 2
  578. // lines), but in practice Go code is gofmt'ed (which elides repeated blank
  579. // lines), so it works.
  580. var c2 *ast.CommentGroup
  581. if c1 == nil {
  582. c2 = p.priorCommentLines(pos, 2)
  583. } else {
  584. c2 = p.priorCommentLines(c1.List[0].Slash, 2)
  585. }
  586. return splitLines(c2.Text()) // safe even if c1 is nil
  587. }
  588. // If there's a comment block which ends nlines before pos, return it.
  589. func (p *Parser) priorCommentLines(pos token.Pos, lines int) *ast.CommentGroup {
  590. position := p.fset.Position(pos)
  591. key := fileLine{position.Filename, position.Line - lines}
  592. return p.endLineToCommentGroup[key]
  593. }
  594. func splitLines(str string) []string {
  595. lines := strings.Split(strings.TrimRight(str, "\n"), "\n")
  596. if len(lines) == 1 && lines[0] == "" {
  597. return nil
  598. }
  599. return lines
  600. }
  601. func goFuncNameToName(in string) types.Name {
  602. name := strings.TrimPrefix(in, "func ")
  603. nameParts := strings.Split(name, "(")
  604. return goNameToName(nameParts[0])
  605. }
  606. func goVarNameToName(in string) types.Name {
  607. nameParts := strings.Split(in, " ")
  608. // nameParts[0] is "var".
  609. // nameParts[2:] is the type of the variable, we ignore it for now.
  610. return goNameToName(nameParts[1])
  611. }
  612. // goNameToName converts a go name string to a gengo types.Name.
  613. // It operates solely on the string on a best effort basis. The name may be updated
  614. // in walkType for generics.
  615. func goNameToName(in string) types.Name {
  616. // Detect anonymous type names. (These may have '.' characters because
  617. // embedded types may have packages, so we detect them specially.)
  618. if strings.HasPrefix(in, "struct{") ||
  619. strings.HasPrefix(in, "<-chan") ||
  620. strings.HasPrefix(in, "chan<-") ||
  621. strings.HasPrefix(in, "chan ") ||
  622. strings.HasPrefix(in, "func(") ||
  623. strings.HasPrefix(in, "func (") ||
  624. strings.HasPrefix(in, "*") ||
  625. strings.HasPrefix(in, "map[") ||
  626. strings.HasPrefix(in, "[") {
  627. return types.Name{Name: in}
  628. }
  629. // There may be '.' characters within a generic. Temporarily remove
  630. // the generic.
  631. genericIndex := strings.IndexRune(in, '[')
  632. if genericIndex == -1 {
  633. genericIndex = len(in)
  634. }
  635. // Otherwise, if there are '.' characters present, the name has a
  636. // package path in front.
  637. nameParts := strings.Split(in[:genericIndex], ".")
  638. name := types.Name{Name: in}
  639. if n := len(nameParts); n >= 2 {
  640. // The final "." is the name of the type--previous ones must
  641. // have been in the package path.
  642. name.Package, name.Name = strings.Join(nameParts[:n-1], "."), nameParts[n-1]
  643. // Add back the generic component now that the package and type name have been separated.
  644. if genericIndex != len(in) {
  645. name.Name = name.Name + in[genericIndex:]
  646. }
  647. }
  648. return name
  649. }
  650. func (p *Parser) convertSignature(u types.Universe, t *gotypes.Signature) *types.Signature {
  651. signature := &types.Signature{}
  652. for i := 0; i < t.Params().Len(); i++ {
  653. signature.Parameters = append(signature.Parameters, &types.ParamResult{
  654. Name: t.Params().At(i).Name(),
  655. Type: p.walkType(u, nil, t.Params().At(i).Type()),
  656. })
  657. }
  658. for i := 0; i < t.Results().Len(); i++ {
  659. signature.Results = append(signature.Results, &types.ParamResult{
  660. Name: t.Results().At(i).Name(),
  661. Type: p.walkType(u, nil, t.Results().At(i).Type()),
  662. })
  663. }
  664. if r := t.Recv(); r != nil {
  665. signature.Receiver = p.walkType(u, nil, r.Type())
  666. }
  667. signature.Variadic = t.Variadic()
  668. return signature
  669. }
  670. // walkType adds the type, and any necessary child types.
  671. func (p *Parser) walkType(u types.Universe, useName *types.Name, in gotypes.Type) *types.Type {
  672. // Most of the cases are underlying types of the named type.
  673. name := goNameToName(in.String())
  674. if useName != nil {
  675. name = *useName
  676. }
  677. // Handle alias types conditionally on go1.22+.
  678. // Inline this once the minimum supported version is go1.22
  679. if out := p.walkAliasType(u, in); out != nil {
  680. return out
  681. }
  682. switch t := in.(type) {
  683. case *gotypes.Struct:
  684. out := u.Type(name)
  685. out.GoType = in
  686. if out.Kind != types.Unknown {
  687. return out
  688. }
  689. out.Kind = types.Struct
  690. for i := 0; i < t.NumFields(); i++ {
  691. f := t.Field(i)
  692. m := types.Member{
  693. Name: f.Name(),
  694. Embedded: f.Anonymous(),
  695. Tags: t.Tag(i),
  696. Type: p.walkType(u, nil, f.Type()),
  697. CommentLines: p.docComment(f.Pos()),
  698. }
  699. out.Members = append(out.Members, m)
  700. }
  701. return out
  702. case *gotypes.Map:
  703. out := u.Type(name)
  704. out.GoType = in
  705. if out.Kind != types.Unknown {
  706. return out
  707. }
  708. out.Kind = types.Map
  709. out.Elem = p.walkType(u, nil, t.Elem())
  710. out.Key = p.walkType(u, nil, t.Key())
  711. return out
  712. case *gotypes.Pointer:
  713. out := u.Type(name)
  714. out.GoType = in
  715. if out.Kind != types.Unknown {
  716. return out
  717. }
  718. out.Kind = types.Pointer
  719. out.Elem = p.walkType(u, nil, t.Elem())
  720. return out
  721. case *gotypes.Slice:
  722. out := u.Type(name)
  723. out.GoType = in
  724. if out.Kind != types.Unknown {
  725. return out
  726. }
  727. out.Kind = types.Slice
  728. out.Elem = p.walkType(u, nil, t.Elem())
  729. return out
  730. case *gotypes.Array:
  731. out := u.Type(name)
  732. out.GoType = in
  733. if out.Kind != types.Unknown {
  734. return out
  735. }
  736. out.Kind = types.Array
  737. out.Elem = p.walkType(u, nil, t.Elem())
  738. out.Len = in.(*gotypes.Array).Len()
  739. return out
  740. case *gotypes.Chan:
  741. out := u.Type(name)
  742. out.GoType = in
  743. if out.Kind != types.Unknown {
  744. return out
  745. }
  746. out.Kind = types.Chan
  747. out.Elem = p.walkType(u, nil, t.Elem())
  748. // TODO: need to store direction, otherwise raw type name
  749. // cannot be properly written.
  750. return out
  751. case *gotypes.Basic:
  752. out := u.Type(types.Name{
  753. Package: "", // This is a magic package name in the Universe.
  754. Name: t.Name(),
  755. })
  756. out.GoType = in
  757. if out.Kind != types.Unknown {
  758. return out
  759. }
  760. out.Kind = types.Unsupported
  761. return out
  762. case *gotypes.Signature:
  763. out := u.Type(name)
  764. out.GoType = in
  765. if out.Kind != types.Unknown {
  766. return out
  767. }
  768. out.Kind = types.Func
  769. out.Signature = p.convertSignature(u, t)
  770. return out
  771. case *gotypes.Interface:
  772. out := u.Type(name)
  773. out.GoType = in
  774. if out.Kind != types.Unknown {
  775. return out
  776. }
  777. out.Kind = types.Interface
  778. t.Complete()
  779. for i := 0; i < t.NumMethods(); i++ {
  780. if out.Methods == nil {
  781. out.Methods = map[string]*types.Type{}
  782. }
  783. method := t.Method(i)
  784. name := goNameToName(method.String())
  785. mt := p.walkType(u, &name, method.Type())
  786. mt.CommentLines = p.docComment(method.Pos())
  787. out.Methods[method.Name()] = mt
  788. }
  789. return out
  790. case *gotypes.Named:
  791. var out *types.Type
  792. switch t.Underlying().(type) {
  793. case *gotypes.Named, *gotypes.Basic, *gotypes.Map, *gotypes.Slice:
  794. name := goNameToName(t.String())
  795. out = u.Type(name)
  796. out.GoType = in
  797. if out.Kind != types.Unknown {
  798. return out
  799. }
  800. out.Kind = types.Alias
  801. out.Underlying = p.walkType(u, nil, t.Underlying())
  802. case *gotypes.Struct, *gotypes.Interface:
  803. name := goNameToName(t.String())
  804. tpMap := map[string]*types.Type{}
  805. if t.TypeParams().Len() != 0 {
  806. // Remove generics, then readd them without the encoded
  807. // type, e.g. Foo[T any] => Foo[T]
  808. var tpNames []string
  809. for i := 0; i < t.TypeParams().Len(); i++ {
  810. tp := t.TypeParams().At(i)
  811. tpName := tp.Obj().Name()
  812. tpNames = append(tpNames, tpName)
  813. tpMap[tpName] = p.walkType(u, nil, tp.Constraint())
  814. }
  815. name.Name = fmt.Sprintf("%s[%s]", strings.SplitN(name.Name, "[", 2)[0], strings.Join(tpNames, ","))
  816. }
  817. if out := u.Type(name); out.Kind != types.Unknown {
  818. out.GoType = in
  819. return out // short circuit if we've already made this.
  820. }
  821. out = p.walkType(u, &name, t.Underlying())
  822. out.TypeParams = tpMap
  823. default:
  824. // gotypes package makes everything "named" with an
  825. // underlying anonymous type--we remove that annoying
  826. // "feature" for users. This flattens those types
  827. // together.
  828. name := goNameToName(t.String())
  829. if out := u.Type(name); out.Kind != types.Unknown {
  830. return out // short circuit if we've already made this.
  831. }
  832. out = p.walkType(u, &name, t.Underlying())
  833. }
  834. // If the underlying type didn't already add methods, add them.
  835. // (Interface types will have already added methods.)
  836. if len(out.Methods) == 0 {
  837. for i := 0; i < t.NumMethods(); i++ {
  838. if out.Methods == nil {
  839. out.Methods = map[string]*types.Type{}
  840. }
  841. method := t.Method(i)
  842. name := goNameToName(method.String())
  843. mt := p.walkType(u, &name, method.Type())
  844. mt.CommentLines = p.docComment(method.Pos())
  845. out.Methods[method.Name()] = mt
  846. }
  847. }
  848. return out
  849. case *gotypes.TypeParam:
  850. // DO NOT retrieve the type from the universe. The default type-param name is only the
  851. // generic variable name. Ideally, it would be namespaced by package and struct but it is
  852. // not. Thus, if we try to use the universe, we would start polluting it.
  853. // e.g. if Foo[T] and Bar[T] exists, we'd mistakenly use the same type T for both.
  854. return &types.Type{
  855. Name: name,
  856. Kind: types.TypeParam,
  857. }
  858. default:
  859. out := u.Type(name)
  860. out.GoType = in
  861. if out.Kind != types.Unknown {
  862. return out
  863. }
  864. out.Kind = types.Unsupported
  865. klog.Warningf("Making unsupported type entry %q for: %#v\n", out, t)
  866. return out
  867. }
  868. }
  869. func (p *Parser) addFunction(u types.Universe, useName *types.Name, in *gotypes.Func) *types.Type {
  870. name := goFuncNameToName(in.String())
  871. if useName != nil {
  872. name = *useName
  873. }
  874. out := u.Function(name)
  875. out.Kind = types.DeclarationOf
  876. out.Underlying = p.walkType(u, nil, in.Type())
  877. return out
  878. }
  879. func (p *Parser) addVariable(u types.Universe, useName *types.Name, in *gotypes.Var) *types.Type {
  880. name := goVarNameToName(in.String())
  881. if useName != nil {
  882. name = *useName
  883. }
  884. out := u.Variable(name)
  885. out.Kind = types.DeclarationOf
  886. out.Underlying = p.walkType(u, nil, in.Type())
  887. return out
  888. }
  889. func (p *Parser) addConstant(u types.Universe, useName *types.Name, in *gotypes.Const) *types.Type {
  890. name := goVarNameToName(in.String())
  891. if useName != nil {
  892. name = *useName
  893. }
  894. out := u.Constant(name)
  895. out.Kind = types.DeclarationOf
  896. out.Underlying = p.walkType(u, nil, in.Type())
  897. var constval string
  898. // For strings, we use `StringVal()` to get the un-truncated,
  899. // un-quoted string. For other values, `.String()` is preferable to
  900. // get something relatively human readable (especially since for
  901. // floating point types, `ExactString()` will generate numeric
  902. // expressions using `big.(*Float).Text()`.
  903. switch in.Val().Kind() {
  904. case constant.String:
  905. constval = constant.StringVal(in.Val())
  906. default:
  907. constval = in.Val().String()
  908. }
  909. out.ConstValue = &constval
  910. return out
  911. }