loader.go 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package loader
  5. // See doc.go for package documentation and implementation notes.
  6. import (
  7. "errors"
  8. "fmt"
  9. "go/ast"
  10. "go/build"
  11. "go/parser"
  12. "go/token"
  13. "go/types"
  14. "os"
  15. "path/filepath"
  16. "sort"
  17. "strings"
  18. "sync"
  19. "time"
  20. "golang.org/x/tools/go/ast/astutil"
  21. "golang.org/x/tools/go/internal/cgo"
  22. )
  23. var ignoreVendor build.ImportMode
  24. const trace = false // show timing info for type-checking
  25. // Config specifies the configuration for loading a whole program from
  26. // Go source code.
  27. // The zero value for Config is a ready-to-use default configuration.
  28. type Config struct {
  29. // Fset is the file set for the parser to use when loading the
  30. // program. If nil, it may be lazily initialized by any
  31. // method of Config.
  32. Fset *token.FileSet
  33. // ParserMode specifies the mode to be used by the parser when
  34. // loading source packages.
  35. ParserMode parser.Mode
  36. // TypeChecker contains options relating to the type checker.
  37. //
  38. // The supplied IgnoreFuncBodies is not used; the effective
  39. // value comes from the TypeCheckFuncBodies func below.
  40. // The supplied Import function is not used either.
  41. TypeChecker types.Config
  42. // TypeCheckFuncBodies is a predicate over package paths.
  43. // A package for which the predicate is false will
  44. // have its package-level declarations type checked, but not
  45. // its function bodies; this can be used to quickly load
  46. // dependencies from source. If nil, all func bodies are type
  47. // checked.
  48. TypeCheckFuncBodies func(path string) bool
  49. // If Build is non-nil, it is used to locate source packages.
  50. // Otherwise &build.Default is used.
  51. //
  52. // By default, cgo is invoked to preprocess Go files that
  53. // import the fake package "C". This behaviour can be
  54. // disabled by setting CGO_ENABLED=0 in the environment prior
  55. // to startup, or by setting Build.CgoEnabled=false.
  56. Build *build.Context
  57. // The current directory, used for resolving relative package
  58. // references such as "./go/loader". If empty, os.Getwd will be
  59. // used instead.
  60. Cwd string
  61. // If DisplayPath is non-nil, it is used to transform each
  62. // file name obtained from Build.Import(). This can be used
  63. // to prevent a virtualized build.Config's file names from
  64. // leaking into the user interface.
  65. DisplayPath func(path string) string
  66. // If AllowErrors is true, Load will return a Program even
  67. // if some of the its packages contained I/O, parser or type
  68. // errors; such errors are accessible via PackageInfo.Errors. If
  69. // false, Load will fail if any package had an error.
  70. AllowErrors bool
  71. // CreatePkgs specifies a list of non-importable initial
  72. // packages to create. The resulting packages will appear in
  73. // the corresponding elements of the Program.Created slice.
  74. CreatePkgs []PkgSpec
  75. // ImportPkgs specifies a set of initial packages to load.
  76. // The map keys are package paths.
  77. //
  78. // The map value indicates whether to load tests. If true, Load
  79. // will add and type-check two lists of files to the package:
  80. // non-test files followed by in-package *_test.go files. In
  81. // addition, it will append the external test package (if any)
  82. // to Program.Created.
  83. ImportPkgs map[string]bool
  84. // FindPackage is called during Load to create the build.Package
  85. // for a given import path from a given directory.
  86. // If FindPackage is nil, (*build.Context).Import is used.
  87. // A client may use this hook to adapt to a proprietary build
  88. // system that does not follow the "go build" layout
  89. // conventions, for example.
  90. //
  91. // It must be safe to call concurrently from multiple goroutines.
  92. FindPackage func(ctxt *build.Context, importPath, fromDir string, mode build.ImportMode) (*build.Package, error)
  93. // AfterTypeCheck is called immediately after a list of files
  94. // has been type-checked and appended to info.Files.
  95. //
  96. // This optional hook function is the earliest opportunity for
  97. // the client to observe the output of the type checker,
  98. // which may be useful to reduce analysis latency when loading
  99. // a large program.
  100. //
  101. // The function is permitted to modify info.Info, for instance
  102. // to clear data structures that are no longer needed, which can
  103. // dramatically reduce peak memory consumption.
  104. //
  105. // The function may be called twice for the same PackageInfo:
  106. // once for the files of the package and again for the
  107. // in-package test files.
  108. //
  109. // It must be safe to call concurrently from multiple goroutines.
  110. AfterTypeCheck func(info *PackageInfo, files []*ast.File)
  111. }
  112. // A PkgSpec specifies a non-importable package to be created by Load.
  113. // Files are processed first, but typically only one of Files and
  114. // Filenames is provided. The path needn't be globally unique.
  115. //
  116. // For vendoring purposes, the package's directory is the one that
  117. // contains the first file.
  118. type PkgSpec struct {
  119. Path string // package path ("" => use package declaration)
  120. Files []*ast.File // ASTs of already-parsed files
  121. Filenames []string // names of files to be parsed
  122. }
  123. // A Program is a Go program loaded from source as specified by a Config.
  124. type Program struct {
  125. Fset *token.FileSet // the file set for this program
  126. // Created[i] contains the initial package whose ASTs or
  127. // filenames were supplied by Config.CreatePkgs[i], followed by
  128. // the external test package, if any, of each package in
  129. // Config.ImportPkgs ordered by ImportPath.
  130. //
  131. // NOTE: these files must not import "C". Cgo preprocessing is
  132. // only performed on imported packages, not ad hoc packages.
  133. //
  134. // TODO(adonovan): we need to copy and adapt the logic of
  135. // goFilesPackage (from $GOROOT/src/cmd/go/build.go) and make
  136. // Config.Import and Config.Create methods return the same kind
  137. // of entity, essentially a build.Package.
  138. // Perhaps we can even reuse that type directly.
  139. Created []*PackageInfo
  140. // Imported contains the initially imported packages,
  141. // as specified by Config.ImportPkgs.
  142. Imported map[string]*PackageInfo
  143. // AllPackages contains the PackageInfo of every package
  144. // encountered by Load: all initial packages and all
  145. // dependencies, including incomplete ones.
  146. AllPackages map[*types.Package]*PackageInfo
  147. // importMap is the canonical mapping of package paths to
  148. // packages. It contains all Imported initial packages, but not
  149. // Created ones, and all imported dependencies.
  150. importMap map[string]*types.Package
  151. }
  152. // PackageInfo holds the ASTs and facts derived by the type-checker
  153. // for a single package.
  154. //
  155. // Not mutated once exposed via the API.
  156. type PackageInfo struct {
  157. Pkg *types.Package
  158. Importable bool // true if 'import "Pkg.Path()"' would resolve to this
  159. TransitivelyErrorFree bool // true if Pkg and all its dependencies are free of errors
  160. Files []*ast.File // syntax trees for the package's files
  161. Errors []error // non-nil if the package had errors
  162. types.Info // type-checker deductions.
  163. dir string // package directory
  164. checker *types.Checker // transient type-checker state
  165. errorFunc func(error)
  166. }
  167. func (info *PackageInfo) String() string { return info.Pkg.Path() }
  168. func (info *PackageInfo) appendError(err error) {
  169. if info.errorFunc != nil {
  170. info.errorFunc(err)
  171. } else {
  172. fmt.Fprintln(os.Stderr, err)
  173. }
  174. info.Errors = append(info.Errors, err)
  175. }
  176. func (conf *Config) fset() *token.FileSet {
  177. if conf.Fset == nil {
  178. conf.Fset = token.NewFileSet()
  179. }
  180. return conf.Fset
  181. }
  182. // ParseFile is a convenience function (intended for testing) that invokes
  183. // the parser using the Config's FileSet, which is initialized if nil.
  184. //
  185. // src specifies the parser input as a string, []byte, or io.Reader, and
  186. // filename is its apparent name. If src is nil, the contents of
  187. // filename are read from the file system.
  188. func (conf *Config) ParseFile(filename string, src any) (*ast.File, error) {
  189. // TODO(adonovan): use conf.build() etc like parseFiles does.
  190. return parser.ParseFile(conf.fset(), filename, src, conf.ParserMode)
  191. }
  192. // FromArgsUsage is a partial usage message that applications calling
  193. // FromArgs may wish to include in their -help output.
  194. const FromArgsUsage = `
  195. <args> is a list of arguments denoting a set of initial packages.
  196. It may take one of two forms:
  197. 1. A list of *.go source files.
  198. All of the specified files are loaded, parsed and type-checked
  199. as a single package. All the files must belong to the same directory.
  200. 2. A list of import paths, each denoting a package.
  201. The package's directory is found relative to the $GOROOT and
  202. $GOPATH using similar logic to 'go build', and the *.go files in
  203. that directory are loaded, parsed and type-checked as a single
  204. package.
  205. In addition, all *_test.go files in the directory are then loaded
  206. and parsed. Those files whose package declaration equals that of
  207. the non-*_test.go files are included in the primary package. Test
  208. files whose package declaration ends with "_test" are type-checked
  209. as another package, the 'external' test package, so that a single
  210. import path may denote two packages. (Whether this behaviour is
  211. enabled is tool-specific, and may depend on additional flags.)
  212. A '--' argument terminates the list of packages.
  213. `
  214. // FromArgs interprets args as a set of initial packages to load from
  215. // source and updates the configuration. It returns the list of
  216. // unconsumed arguments.
  217. //
  218. // It is intended for use in command-line interfaces that require a
  219. // set of initial packages to be specified; see FromArgsUsage message
  220. // for details.
  221. //
  222. // Only superficial errors are reported at this stage; errors dependent
  223. // on I/O are detected during Load.
  224. func (conf *Config) FromArgs(args []string, xtest bool) ([]string, error) {
  225. var rest []string
  226. for i, arg := range args {
  227. if arg == "--" {
  228. rest = args[i+1:]
  229. args = args[:i]
  230. break // consume "--" and return the remaining args
  231. }
  232. }
  233. if len(args) > 0 && strings.HasSuffix(args[0], ".go") {
  234. // Assume args is a list of a *.go files
  235. // denoting a single ad hoc package.
  236. for _, arg := range args {
  237. if !strings.HasSuffix(arg, ".go") {
  238. return nil, fmt.Errorf("named files must be .go files: %s", arg)
  239. }
  240. }
  241. conf.CreateFromFilenames("", args...)
  242. } else {
  243. // Assume args are directories each denoting a
  244. // package and (perhaps) an external test, iff xtest.
  245. for _, arg := range args {
  246. if xtest {
  247. conf.ImportWithTests(arg)
  248. } else {
  249. conf.Import(arg)
  250. }
  251. }
  252. }
  253. return rest, nil
  254. }
  255. // CreateFromFilenames is a convenience function that adds
  256. // a conf.CreatePkgs entry to create a package of the specified *.go
  257. // files.
  258. func (conf *Config) CreateFromFilenames(path string, filenames ...string) {
  259. conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Filenames: filenames})
  260. }
  261. // CreateFromFiles is a convenience function that adds a conf.CreatePkgs
  262. // entry to create package of the specified path and parsed files.
  263. func (conf *Config) CreateFromFiles(path string, files ...*ast.File) {
  264. conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Files: files})
  265. }
  266. // ImportWithTests is a convenience function that adds path to
  267. // ImportPkgs, the set of initial source packages located relative to
  268. // $GOPATH. The package will be augmented by any *_test.go files in
  269. // its directory that contain a "package x" (not "package x_test")
  270. // declaration.
  271. //
  272. // In addition, if any *_test.go files contain a "package x_test"
  273. // declaration, an additional package comprising just those files will
  274. // be added to CreatePkgs.
  275. func (conf *Config) ImportWithTests(path string) { conf.addImport(path, true) }
  276. // Import is a convenience function that adds path to ImportPkgs, the
  277. // set of initial packages that will be imported from source.
  278. func (conf *Config) Import(path string) { conf.addImport(path, false) }
  279. func (conf *Config) addImport(path string, tests bool) {
  280. if path == "C" {
  281. return // ignore; not a real package
  282. }
  283. if conf.ImportPkgs == nil {
  284. conf.ImportPkgs = make(map[string]bool)
  285. }
  286. conf.ImportPkgs[path] = conf.ImportPkgs[path] || tests
  287. }
  288. // PathEnclosingInterval returns the PackageInfo and ast.Node that
  289. // contain source interval [start, end), and all the node's ancestors
  290. // up to the AST root. It searches all ast.Files of all packages in prog.
  291. // exact is defined as for astutil.PathEnclosingInterval.
  292. //
  293. // The zero value is returned if not found.
  294. func (prog *Program) PathEnclosingInterval(start, end token.Pos) (pkg *PackageInfo, path []ast.Node, exact bool) {
  295. for _, info := range prog.AllPackages {
  296. for _, f := range info.Files {
  297. if !tokenFileContainsPos(prog.Fset.File(f.FileStart), start) {
  298. continue
  299. }
  300. if path, exact := astutil.PathEnclosingInterval(f, start, end); path != nil {
  301. return info, path, exact
  302. }
  303. }
  304. }
  305. return nil, nil, false
  306. }
  307. // InitialPackages returns a new slice containing the set of initial
  308. // packages (Created + Imported) in unspecified order.
  309. func (prog *Program) InitialPackages() []*PackageInfo {
  310. infos := make([]*PackageInfo, 0, len(prog.Created)+len(prog.Imported))
  311. infos = append(infos, prog.Created...)
  312. for _, info := range prog.Imported {
  313. infos = append(infos, info)
  314. }
  315. return infos
  316. }
  317. // Package returns the ASTs and results of type checking for the
  318. // specified package.
  319. func (prog *Program) Package(path string) *PackageInfo {
  320. if info, ok := prog.AllPackages[prog.importMap[path]]; ok {
  321. return info
  322. }
  323. for _, info := range prog.Created {
  324. if path == info.Pkg.Path() {
  325. return info
  326. }
  327. }
  328. return nil
  329. }
  330. // ---------- Implementation ----------
  331. // importer holds the working state of the algorithm.
  332. type importer struct {
  333. conf *Config // the client configuration
  334. start time.Time // for logging
  335. progMu sync.Mutex // guards prog
  336. prog *Program // the resulting program
  337. // findpkg is a memoization of FindPackage.
  338. findpkgMu sync.Mutex // guards findpkg
  339. findpkg map[findpkgKey]*findpkgValue
  340. importedMu sync.Mutex // guards imported
  341. imported map[string]*importInfo // all imported packages (incl. failures) by import path
  342. // import dependency graph: graph[x][y] => x imports y
  343. //
  344. // Since non-importable packages cannot be cyclic, we ignore
  345. // their imports, thus we only need the subgraph over importable
  346. // packages. Nodes are identified by their import paths.
  347. graphMu sync.Mutex
  348. graph map[string]map[string]bool
  349. }
  350. type findpkgKey struct {
  351. importPath string
  352. fromDir string
  353. mode build.ImportMode
  354. }
  355. type findpkgValue struct {
  356. ready chan struct{} // closed to broadcast readiness
  357. bp *build.Package
  358. err error
  359. }
  360. // importInfo tracks the success or failure of a single import.
  361. //
  362. // Upon completion, exactly one of info and err is non-nil:
  363. // info on successful creation of a package, err otherwise.
  364. // A successful package may still contain type errors.
  365. type importInfo struct {
  366. path string // import path
  367. info *PackageInfo // results of typechecking (including errors)
  368. complete chan struct{} // closed to broadcast that info is set.
  369. }
  370. // awaitCompletion blocks until ii is complete,
  371. // i.e. the info field is safe to inspect.
  372. func (ii *importInfo) awaitCompletion() {
  373. <-ii.complete // wait for close
  374. }
  375. // Complete marks ii as complete.
  376. // Its info and err fields will not be subsequently updated.
  377. func (ii *importInfo) Complete(info *PackageInfo) {
  378. if info == nil {
  379. panic("info == nil")
  380. }
  381. ii.info = info
  382. close(ii.complete)
  383. }
  384. type importError struct {
  385. path string // import path
  386. err error // reason for failure to create a package
  387. }
  388. // Load creates the initial packages specified by conf.{Create,Import}Pkgs,
  389. // loading their dependencies packages as needed.
  390. //
  391. // On success, Load returns a Program containing a PackageInfo for
  392. // each package. On failure, it returns an error.
  393. //
  394. // If AllowErrors is true, Load will return a Program even if some
  395. // packages contained I/O, parser or type errors, or if dependencies
  396. // were missing. (Such errors are accessible via PackageInfo.Errors. If
  397. // false, Load will fail if any package had an error.
  398. //
  399. // It is an error if no packages were loaded.
  400. func (conf *Config) Load() (*Program, error) {
  401. // Create a simple default error handler for parse/type errors.
  402. if conf.TypeChecker.Error == nil {
  403. conf.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) }
  404. }
  405. // Set default working directory for relative package references.
  406. if conf.Cwd == "" {
  407. var err error
  408. conf.Cwd, err = os.Getwd()
  409. if err != nil {
  410. return nil, err
  411. }
  412. }
  413. // Install default FindPackage hook using go/build logic.
  414. if conf.FindPackage == nil {
  415. conf.FindPackage = (*build.Context).Import
  416. }
  417. prog := &Program{
  418. Fset: conf.fset(),
  419. Imported: make(map[string]*PackageInfo),
  420. importMap: make(map[string]*types.Package),
  421. AllPackages: make(map[*types.Package]*PackageInfo),
  422. }
  423. imp := importer{
  424. conf: conf,
  425. prog: prog,
  426. findpkg: make(map[findpkgKey]*findpkgValue),
  427. imported: make(map[string]*importInfo),
  428. start: time.Now(),
  429. graph: make(map[string]map[string]bool),
  430. }
  431. // -- loading proper (concurrent phase) --------------------------------
  432. var errpkgs []string // packages that contained errors
  433. // Load the initially imported packages and their dependencies,
  434. // in parallel.
  435. // No vendor check on packages imported from the command line.
  436. infos, importErrors := imp.importAll("", conf.Cwd, conf.ImportPkgs, ignoreVendor)
  437. for _, ie := range importErrors {
  438. conf.TypeChecker.Error(ie.err) // failed to create package
  439. errpkgs = append(errpkgs, ie.path)
  440. }
  441. for _, info := range infos {
  442. prog.Imported[info.Pkg.Path()] = info
  443. }
  444. // Augment the designated initial packages by their tests.
  445. // Dependencies are loaded in parallel.
  446. var xtestPkgs []*build.Package
  447. for importPath, augment := range conf.ImportPkgs {
  448. if !augment {
  449. continue
  450. }
  451. // No vendor check on packages imported from command line.
  452. bp, err := imp.findPackage(importPath, conf.Cwd, ignoreVendor)
  453. if err != nil {
  454. // Package not found, or can't even parse package declaration.
  455. // Already reported by previous loop; ignore it.
  456. continue
  457. }
  458. // Needs external test package?
  459. if len(bp.XTestGoFiles) > 0 {
  460. xtestPkgs = append(xtestPkgs, bp)
  461. }
  462. // Consult the cache using the canonical package path.
  463. path := bp.ImportPath
  464. imp.importedMu.Lock() // (unnecessary, we're sequential here)
  465. ii, ok := imp.imported[path]
  466. // Paranoid checks added due to issue #11012.
  467. if !ok {
  468. // Unreachable.
  469. // The previous loop called importAll and thus
  470. // startLoad for each path in ImportPkgs, which
  471. // populates imp.imported[path] with a non-zero value.
  472. panic(fmt.Sprintf("imported[%q] not found", path))
  473. }
  474. if ii == nil {
  475. // Unreachable.
  476. // The ii values in this loop are the same as in
  477. // the previous loop, which enforced the invariant
  478. // that at least one of ii.err and ii.info is non-nil.
  479. panic(fmt.Sprintf("imported[%q] == nil", path))
  480. }
  481. if ii.info == nil {
  482. // Unreachable.
  483. // awaitCompletion has the postcondition
  484. // ii.info != nil.
  485. panic(fmt.Sprintf("imported[%q].info = nil", path))
  486. }
  487. info := ii.info
  488. imp.importedMu.Unlock()
  489. // Parse the in-package test files.
  490. files, errs := imp.conf.parsePackageFiles(bp, 't')
  491. for _, err := range errs {
  492. info.appendError(err)
  493. }
  494. // The test files augmenting package P cannot be imported,
  495. // but may import packages that import P,
  496. // so we must disable the cycle check.
  497. imp.addFiles(info, files, false)
  498. }
  499. createPkg := func(path, dir string, files []*ast.File, errs []error) {
  500. info := imp.newPackageInfo(path, dir)
  501. for _, err := range errs {
  502. info.appendError(err)
  503. }
  504. // Ad hoc packages are non-importable,
  505. // so no cycle check is needed.
  506. // addFiles loads dependencies in parallel.
  507. imp.addFiles(info, files, false)
  508. prog.Created = append(prog.Created, info)
  509. }
  510. // Create packages specified by conf.CreatePkgs.
  511. for _, cp := range conf.CreatePkgs {
  512. files, errs := parseFiles(conf.fset(), conf.build(), nil, conf.Cwd, cp.Filenames, conf.ParserMode)
  513. files = append(files, cp.Files...)
  514. path := cp.Path
  515. if path == "" {
  516. if len(files) > 0 {
  517. path = files[0].Name.Name
  518. } else {
  519. path = "(unnamed)"
  520. }
  521. }
  522. dir := conf.Cwd
  523. if len(files) > 0 && files[0].Pos().IsValid() {
  524. dir = filepath.Dir(conf.fset().File(files[0].Pos()).Name())
  525. }
  526. createPkg(path, dir, files, errs)
  527. }
  528. // Create external test packages.
  529. sort.Sort(byImportPath(xtestPkgs))
  530. for _, bp := range xtestPkgs {
  531. files, errs := imp.conf.parsePackageFiles(bp, 'x')
  532. createPkg(bp.ImportPath+"_test", bp.Dir, files, errs)
  533. }
  534. // -- finishing up (sequential) ----------------------------------------
  535. if len(prog.Imported)+len(prog.Created) == 0 {
  536. return nil, errors.New("no initial packages were loaded")
  537. }
  538. // Create infos for indirectly imported packages.
  539. // e.g. incomplete packages without syntax, loaded from export data.
  540. for _, obj := range prog.importMap {
  541. info := prog.AllPackages[obj]
  542. if info == nil {
  543. prog.AllPackages[obj] = &PackageInfo{Pkg: obj, Importable: true}
  544. } else {
  545. // finished
  546. info.checker = nil
  547. info.errorFunc = nil
  548. }
  549. }
  550. if !conf.AllowErrors {
  551. // Report errors in indirectly imported packages.
  552. for _, info := range prog.AllPackages {
  553. if len(info.Errors) > 0 {
  554. errpkgs = append(errpkgs, info.Pkg.Path())
  555. }
  556. }
  557. if errpkgs != nil {
  558. var more string
  559. if len(errpkgs) > 3 {
  560. more = fmt.Sprintf(" and %d more", len(errpkgs)-3)
  561. errpkgs = errpkgs[:3]
  562. }
  563. return nil, fmt.Errorf("couldn't load packages due to errors: %s%s",
  564. strings.Join(errpkgs, ", "), more)
  565. }
  566. }
  567. markErrorFreePackages(prog.AllPackages)
  568. return prog, nil
  569. }
  570. type byImportPath []*build.Package
  571. func (b byImportPath) Len() int { return len(b) }
  572. func (b byImportPath) Less(i, j int) bool { return b[i].ImportPath < b[j].ImportPath }
  573. func (b byImportPath) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
  574. // markErrorFreePackages sets the TransitivelyErrorFree flag on all
  575. // applicable packages.
  576. func markErrorFreePackages(allPackages map[*types.Package]*PackageInfo) {
  577. // Build the transpose of the import graph.
  578. importedBy := make(map[*types.Package]map[*types.Package]bool)
  579. for P := range allPackages {
  580. for _, Q := range P.Imports() {
  581. clients, ok := importedBy[Q]
  582. if !ok {
  583. clients = make(map[*types.Package]bool)
  584. importedBy[Q] = clients
  585. }
  586. clients[P] = true
  587. }
  588. }
  589. // Find all packages reachable from some error package.
  590. reachable := make(map[*types.Package]bool)
  591. var visit func(*types.Package)
  592. visit = func(p *types.Package) {
  593. if !reachable[p] {
  594. reachable[p] = true
  595. for q := range importedBy[p] {
  596. visit(q)
  597. }
  598. }
  599. }
  600. for _, info := range allPackages {
  601. if len(info.Errors) > 0 {
  602. visit(info.Pkg)
  603. }
  604. }
  605. // Mark the others as "transitively error-free".
  606. for _, info := range allPackages {
  607. if !reachable[info.Pkg] {
  608. info.TransitivelyErrorFree = true
  609. }
  610. }
  611. }
  612. // build returns the effective build context.
  613. func (conf *Config) build() *build.Context {
  614. if conf.Build != nil {
  615. return conf.Build
  616. }
  617. return &build.Default
  618. }
  619. // parsePackageFiles enumerates the files belonging to package path,
  620. // then loads, parses and returns them, plus a list of I/O or parse
  621. // errors that were encountered.
  622. //
  623. // 'which' indicates which files to include:
  624. //
  625. // 'g': include non-test *.go source files (GoFiles + processed CgoFiles)
  626. // 't': include in-package *_test.go source files (TestGoFiles)
  627. // 'x': include external *_test.go source files. (XTestGoFiles)
  628. func (conf *Config) parsePackageFiles(bp *build.Package, which rune) ([]*ast.File, []error) {
  629. if bp.ImportPath == "unsafe" {
  630. return nil, nil
  631. }
  632. var filenames []string
  633. switch which {
  634. case 'g':
  635. filenames = bp.GoFiles
  636. case 't':
  637. filenames = bp.TestGoFiles
  638. case 'x':
  639. filenames = bp.XTestGoFiles
  640. default:
  641. panic(which)
  642. }
  643. files, errs := parseFiles(conf.fset(), conf.build(), conf.DisplayPath, bp.Dir, filenames, conf.ParserMode)
  644. // Preprocess CgoFiles and parse the outputs (sequentially).
  645. if which == 'g' && bp.CgoFiles != nil {
  646. cgofiles, err := cgo.ProcessFiles(bp, conf.fset(), conf.DisplayPath, conf.ParserMode)
  647. if err != nil {
  648. errs = append(errs, err)
  649. } else {
  650. files = append(files, cgofiles...)
  651. }
  652. }
  653. return files, errs
  654. }
  655. // doImport imports the package denoted by path.
  656. // It implements the types.Importer signature.
  657. //
  658. // It returns an error if a package could not be created
  659. // (e.g. go/build or parse error), but type errors are reported via
  660. // the types.Config.Error callback (the first of which is also saved
  661. // in the package's PackageInfo).
  662. //
  663. // Idempotent.
  664. func (imp *importer) doImport(from *PackageInfo, to string) (*types.Package, error) {
  665. if to == "C" {
  666. // This should be unreachable, but ad hoc packages are
  667. // not currently subject to cgo preprocessing.
  668. // See https://golang.org/issue/11627.
  669. return nil, fmt.Errorf(`the loader doesn't cgo-process ad hoc packages like %q; see Go issue 11627`,
  670. from.Pkg.Path())
  671. }
  672. bp, err := imp.findPackage(to, from.dir, 0)
  673. if err != nil {
  674. return nil, err
  675. }
  676. // The standard unsafe package is handled specially,
  677. // and has no PackageInfo.
  678. if bp.ImportPath == "unsafe" {
  679. return types.Unsafe, nil
  680. }
  681. // Look for the package in the cache using its canonical path.
  682. path := bp.ImportPath
  683. imp.importedMu.Lock()
  684. ii := imp.imported[path]
  685. imp.importedMu.Unlock()
  686. if ii == nil {
  687. panic("internal error: unexpected import: " + path)
  688. }
  689. if ii.info != nil {
  690. return ii.info.Pkg, nil
  691. }
  692. // Import of incomplete package: this indicates a cycle.
  693. fromPath := from.Pkg.Path()
  694. if cycle := imp.findPath(path, fromPath); cycle != nil {
  695. // Normalize cycle: start from alphabetically largest node.
  696. pos, start := -1, ""
  697. for i, s := range cycle {
  698. if pos < 0 || s > start {
  699. pos, start = i, s
  700. }
  701. }
  702. cycle = append(cycle, cycle[:pos]...)[pos:] // rotate cycle to start from largest
  703. cycle = append(cycle, cycle[0]) // add start node to end to show cycliness
  704. return nil, fmt.Errorf("import cycle: %s", strings.Join(cycle, " -> "))
  705. }
  706. panic("internal error: import of incomplete (yet acyclic) package: " + fromPath)
  707. }
  708. // findPackage locates the package denoted by the importPath in the
  709. // specified directory.
  710. func (imp *importer) findPackage(importPath, fromDir string, mode build.ImportMode) (*build.Package, error) {
  711. // We use a non-blocking duplicate-suppressing cache (gopl.io §9.7)
  712. // to avoid holding the lock around FindPackage.
  713. key := findpkgKey{importPath, fromDir, mode}
  714. imp.findpkgMu.Lock()
  715. v, ok := imp.findpkg[key]
  716. if ok {
  717. // cache hit
  718. imp.findpkgMu.Unlock()
  719. <-v.ready // wait for entry to become ready
  720. } else {
  721. // Cache miss: this goroutine becomes responsible for
  722. // populating the map entry and broadcasting its readiness.
  723. v = &findpkgValue{ready: make(chan struct{})}
  724. imp.findpkg[key] = v
  725. imp.findpkgMu.Unlock()
  726. ioLimit <- true
  727. v.bp, v.err = imp.conf.FindPackage(imp.conf.build(), importPath, fromDir, mode)
  728. <-ioLimit
  729. if _, ok := v.err.(*build.NoGoError); ok {
  730. v.err = nil // empty directory is not an error
  731. }
  732. close(v.ready) // broadcast ready condition
  733. }
  734. return v.bp, v.err
  735. }
  736. // importAll loads, parses, and type-checks the specified packages in
  737. // parallel and returns their completed importInfos in unspecified order.
  738. //
  739. // fromPath is the package path of the importing package, if it is
  740. // importable, "" otherwise. It is used for cycle detection.
  741. //
  742. // fromDir is the directory containing the import declaration that
  743. // caused these imports.
  744. func (imp *importer) importAll(fromPath, fromDir string, imports map[string]bool, mode build.ImportMode) (infos []*PackageInfo, errors []importError) {
  745. if fromPath != "" {
  746. // We're loading a set of imports.
  747. //
  748. // We must record graph edges from the importing package
  749. // to its dependencies, and check for cycles.
  750. imp.graphMu.Lock()
  751. deps, ok := imp.graph[fromPath]
  752. if !ok {
  753. deps = make(map[string]bool)
  754. imp.graph[fromPath] = deps
  755. }
  756. for importPath := range imports {
  757. deps[importPath] = true
  758. }
  759. imp.graphMu.Unlock()
  760. }
  761. var pending []*importInfo
  762. for importPath := range imports {
  763. if fromPath != "" {
  764. if cycle := imp.findPath(importPath, fromPath); cycle != nil {
  765. // Cycle-forming import: we must not check it
  766. // since it would deadlock.
  767. if trace {
  768. fmt.Fprintf(os.Stderr, "import cycle: %q\n", cycle)
  769. }
  770. continue
  771. }
  772. }
  773. bp, err := imp.findPackage(importPath, fromDir, mode)
  774. if err != nil {
  775. errors = append(errors, importError{
  776. path: importPath,
  777. err: err,
  778. })
  779. continue
  780. }
  781. pending = append(pending, imp.startLoad(bp))
  782. }
  783. for _, ii := range pending {
  784. ii.awaitCompletion()
  785. infos = append(infos, ii.info)
  786. }
  787. return infos, errors
  788. }
  789. // findPath returns an arbitrary path from 'from' to 'to' in the import
  790. // graph, or nil if there was none.
  791. func (imp *importer) findPath(from, to string) []string {
  792. imp.graphMu.Lock()
  793. defer imp.graphMu.Unlock()
  794. seen := make(map[string]bool)
  795. var search func(stack []string, importPath string) []string
  796. search = func(stack []string, importPath string) []string {
  797. if !seen[importPath] {
  798. seen[importPath] = true
  799. stack = append(stack, importPath)
  800. if importPath == to {
  801. return stack
  802. }
  803. for x := range imp.graph[importPath] {
  804. if p := search(stack, x); p != nil {
  805. return p
  806. }
  807. }
  808. }
  809. return nil
  810. }
  811. return search(make([]string, 0, 20), from)
  812. }
  813. // startLoad initiates the loading, parsing and type-checking of the
  814. // specified package and its dependencies, if it has not already begun.
  815. //
  816. // It returns an importInfo, not necessarily in a completed state. The
  817. // caller must call awaitCompletion() before accessing its info field.
  818. //
  819. // startLoad is concurrency-safe and idempotent.
  820. func (imp *importer) startLoad(bp *build.Package) *importInfo {
  821. path := bp.ImportPath
  822. imp.importedMu.Lock()
  823. ii, ok := imp.imported[path]
  824. if !ok {
  825. ii = &importInfo{path: path, complete: make(chan struct{})}
  826. imp.imported[path] = ii
  827. go func() {
  828. info := imp.load(bp)
  829. ii.Complete(info)
  830. }()
  831. }
  832. imp.importedMu.Unlock()
  833. return ii
  834. }
  835. // load implements package loading by parsing Go source files
  836. // located by go/build.
  837. func (imp *importer) load(bp *build.Package) *PackageInfo {
  838. info := imp.newPackageInfo(bp.ImportPath, bp.Dir)
  839. info.Importable = true
  840. files, errs := imp.conf.parsePackageFiles(bp, 'g')
  841. for _, err := range errs {
  842. info.appendError(err)
  843. }
  844. imp.addFiles(info, files, true)
  845. imp.progMu.Lock()
  846. imp.prog.importMap[bp.ImportPath] = info.Pkg
  847. imp.progMu.Unlock()
  848. return info
  849. }
  850. // addFiles adds and type-checks the specified files to info, loading
  851. // their dependencies if needed. The order of files determines the
  852. // package initialization order. It may be called multiple times on the
  853. // same package. Errors are appended to the info.Errors field.
  854. //
  855. // cycleCheck determines whether the imports within files create
  856. // dependency edges that should be checked for potential cycles.
  857. func (imp *importer) addFiles(info *PackageInfo, files []*ast.File, cycleCheck bool) {
  858. // Ensure the dependencies are loaded, in parallel.
  859. var fromPath string
  860. if cycleCheck {
  861. fromPath = info.Pkg.Path()
  862. }
  863. // TODO(adonovan): opt: make the caller do scanImports.
  864. // Callers with a build.Package can skip it.
  865. imp.importAll(fromPath, info.dir, scanImports(files), 0)
  866. if trace {
  867. fmt.Fprintf(os.Stderr, "%s: start %q (%d)\n",
  868. time.Since(imp.start), info.Pkg.Path(), len(files))
  869. }
  870. // Don't call checker.Files on Unsafe, even with zero files,
  871. // because it would mutate the package, which is a global.
  872. if info.Pkg == types.Unsafe {
  873. if len(files) > 0 {
  874. panic(`"unsafe" package contains unexpected files`)
  875. }
  876. } else {
  877. // Ignore the returned (first) error since we
  878. // already collect them all in the PackageInfo.
  879. info.checker.Files(files)
  880. info.Files = append(info.Files, files...)
  881. }
  882. if imp.conf.AfterTypeCheck != nil {
  883. imp.conf.AfterTypeCheck(info, files)
  884. }
  885. if trace {
  886. fmt.Fprintf(os.Stderr, "%s: stop %q\n",
  887. time.Since(imp.start), info.Pkg.Path())
  888. }
  889. }
  890. func (imp *importer) newPackageInfo(path, dir string) *PackageInfo {
  891. var pkg *types.Package
  892. if path == "unsafe" {
  893. pkg = types.Unsafe
  894. } else {
  895. pkg = types.NewPackage(path, "")
  896. }
  897. info := &PackageInfo{
  898. Pkg: pkg,
  899. Info: types.Info{
  900. Types: make(map[ast.Expr]types.TypeAndValue),
  901. Defs: make(map[*ast.Ident]types.Object),
  902. Uses: make(map[*ast.Ident]types.Object),
  903. Implicits: make(map[ast.Node]types.Object),
  904. Instances: make(map[*ast.Ident]types.Instance),
  905. Scopes: make(map[ast.Node]*types.Scope),
  906. Selections: make(map[*ast.SelectorExpr]*types.Selection),
  907. FileVersions: make(map[*ast.File]string),
  908. },
  909. errorFunc: imp.conf.TypeChecker.Error,
  910. dir: dir,
  911. }
  912. // Copy the types.Config so we can vary it across PackageInfos.
  913. tc := imp.conf.TypeChecker
  914. tc.IgnoreFuncBodies = false
  915. if f := imp.conf.TypeCheckFuncBodies; f != nil {
  916. tc.IgnoreFuncBodies = !f(path)
  917. }
  918. tc.Importer = closure{imp, info}
  919. tc.Error = info.appendError // appendError wraps the user's Error function
  920. info.checker = types.NewChecker(&tc, imp.conf.fset(), pkg, &info.Info)
  921. imp.progMu.Lock()
  922. imp.prog.AllPackages[pkg] = info
  923. imp.progMu.Unlock()
  924. return info
  925. }
  926. type closure struct {
  927. imp *importer
  928. info *PackageInfo
  929. }
  930. func (c closure) Import(to string) (*types.Package, error) { return c.imp.doImport(c.info, to) }