2
0

loader.go 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package loader
  5. // See doc.go for package documentation and implementation notes.
  6. import (
  7. "errors"
  8. "fmt"
  9. "go/ast"
  10. "go/build"
  11. "go/parser"
  12. "go/token"
  13. "go/types"
  14. "os"
  15. "path/filepath"
  16. "sort"
  17. "strings"
  18. "sync"
  19. "time"
  20. "golang.org/x/tools/go/ast/astutil"
  21. "golang.org/x/tools/go/internal/cgo"
  22. "golang.org/x/tools/internal/typeparams"
  23. )
  24. var ignoreVendor build.ImportMode
  25. const trace = false // show timing info for type-checking
  26. // Config specifies the configuration for loading a whole program from
  27. // Go source code.
  28. // The zero value for Config is a ready-to-use default configuration.
  29. type Config struct {
  30. // Fset is the file set for the parser to use when loading the
  31. // program. If nil, it may be lazily initialized by any
  32. // method of Config.
  33. Fset *token.FileSet
  34. // ParserMode specifies the mode to be used by the parser when
  35. // loading source packages.
  36. ParserMode parser.Mode
  37. // TypeChecker contains options relating to the type checker.
  38. //
  39. // The supplied IgnoreFuncBodies is not used; the effective
  40. // value comes from the TypeCheckFuncBodies func below.
  41. // The supplied Import function is not used either.
  42. TypeChecker types.Config
  43. // TypeCheckFuncBodies is a predicate over package paths.
  44. // A package for which the predicate is false will
  45. // have its package-level declarations type checked, but not
  46. // its function bodies; this can be used to quickly load
  47. // dependencies from source. If nil, all func bodies are type
  48. // checked.
  49. TypeCheckFuncBodies func(path string) bool
  50. // If Build is non-nil, it is used to locate source packages.
  51. // Otherwise &build.Default is used.
  52. //
  53. // By default, cgo is invoked to preprocess Go files that
  54. // import the fake package "C". This behaviour can be
  55. // disabled by setting CGO_ENABLED=0 in the environment prior
  56. // to startup, or by setting Build.CgoEnabled=false.
  57. Build *build.Context
  58. // The current directory, used for resolving relative package
  59. // references such as "./go/loader". If empty, os.Getwd will be
  60. // used instead.
  61. Cwd string
  62. // If DisplayPath is non-nil, it is used to transform each
  63. // file name obtained from Build.Import(). This can be used
  64. // to prevent a virtualized build.Config's file names from
  65. // leaking into the user interface.
  66. DisplayPath func(path string) string
  67. // If AllowErrors is true, Load will return a Program even
  68. // if some of the its packages contained I/O, parser or type
  69. // errors; such errors are accessible via PackageInfo.Errors. If
  70. // false, Load will fail if any package had an error.
  71. AllowErrors bool
  72. // CreatePkgs specifies a list of non-importable initial
  73. // packages to create. The resulting packages will appear in
  74. // the corresponding elements of the Program.Created slice.
  75. CreatePkgs []PkgSpec
  76. // ImportPkgs specifies a set of initial packages to load.
  77. // The map keys are package paths.
  78. //
  79. // The map value indicates whether to load tests. If true, Load
  80. // will add and type-check two lists of files to the package:
  81. // non-test files followed by in-package *_test.go files. In
  82. // addition, it will append the external test package (if any)
  83. // to Program.Created.
  84. ImportPkgs map[string]bool
  85. // FindPackage is called during Load to create the build.Package
  86. // for a given import path from a given directory.
  87. // If FindPackage is nil, (*build.Context).Import is used.
  88. // A client may use this hook to adapt to a proprietary build
  89. // system that does not follow the "go build" layout
  90. // conventions, for example.
  91. //
  92. // It must be safe to call concurrently from multiple goroutines.
  93. FindPackage func(ctxt *build.Context, importPath, fromDir string, mode build.ImportMode) (*build.Package, error)
  94. // AfterTypeCheck is called immediately after a list of files
  95. // has been type-checked and appended to info.Files.
  96. //
  97. // This optional hook function is the earliest opportunity for
  98. // the client to observe the output of the type checker,
  99. // which may be useful to reduce analysis latency when loading
  100. // a large program.
  101. //
  102. // The function is permitted to modify info.Info, for instance
  103. // to clear data structures that are no longer needed, which can
  104. // dramatically reduce peak memory consumption.
  105. //
  106. // The function may be called twice for the same PackageInfo:
  107. // once for the files of the package and again for the
  108. // in-package test files.
  109. //
  110. // It must be safe to call concurrently from multiple goroutines.
  111. AfterTypeCheck func(info *PackageInfo, files []*ast.File)
  112. }
  113. // A PkgSpec specifies a non-importable package to be created by Load.
  114. // Files are processed first, but typically only one of Files and
  115. // Filenames is provided. The path needn't be globally unique.
  116. //
  117. // For vendoring purposes, the package's directory is the one that
  118. // contains the first file.
  119. type PkgSpec struct {
  120. Path string // package path ("" => use package declaration)
  121. Files []*ast.File // ASTs of already-parsed files
  122. Filenames []string // names of files to be parsed
  123. }
  124. // A Program is a Go program loaded from source as specified by a Config.
  125. type Program struct {
  126. Fset *token.FileSet // the file set for this program
  127. // Created[i] contains the initial package whose ASTs or
  128. // filenames were supplied by Config.CreatePkgs[i], followed by
  129. // the external test package, if any, of each package in
  130. // Config.ImportPkgs ordered by ImportPath.
  131. //
  132. // NOTE: these files must not import "C". Cgo preprocessing is
  133. // only performed on imported packages, not ad hoc packages.
  134. //
  135. // TODO(adonovan): we need to copy and adapt the logic of
  136. // goFilesPackage (from $GOROOT/src/cmd/go/build.go) and make
  137. // Config.Import and Config.Create methods return the same kind
  138. // of entity, essentially a build.Package.
  139. // Perhaps we can even reuse that type directly.
  140. Created []*PackageInfo
  141. // Imported contains the initially imported packages,
  142. // as specified by Config.ImportPkgs.
  143. Imported map[string]*PackageInfo
  144. // AllPackages contains the PackageInfo of every package
  145. // encountered by Load: all initial packages and all
  146. // dependencies, including incomplete ones.
  147. AllPackages map[*types.Package]*PackageInfo
  148. // importMap is the canonical mapping of package paths to
  149. // packages. It contains all Imported initial packages, but not
  150. // Created ones, and all imported dependencies.
  151. importMap map[string]*types.Package
  152. }
  153. // PackageInfo holds the ASTs and facts derived by the type-checker
  154. // for a single package.
  155. //
  156. // Not mutated once exposed via the API.
  157. //
  158. type PackageInfo struct {
  159. Pkg *types.Package
  160. Importable bool // true if 'import "Pkg.Path()"' would resolve to this
  161. TransitivelyErrorFree bool // true if Pkg and all its dependencies are free of errors
  162. Files []*ast.File // syntax trees for the package's files
  163. Errors []error // non-nil if the package had errors
  164. types.Info // type-checker deductions.
  165. dir string // package directory
  166. checker *types.Checker // transient type-checker state
  167. errorFunc func(error)
  168. }
  169. func (info *PackageInfo) String() string { return info.Pkg.Path() }
  170. func (info *PackageInfo) appendError(err error) {
  171. if info.errorFunc != nil {
  172. info.errorFunc(err)
  173. } else {
  174. fmt.Fprintln(os.Stderr, err)
  175. }
  176. info.Errors = append(info.Errors, err)
  177. }
  178. func (conf *Config) fset() *token.FileSet {
  179. if conf.Fset == nil {
  180. conf.Fset = token.NewFileSet()
  181. }
  182. return conf.Fset
  183. }
  184. // ParseFile is a convenience function (intended for testing) that invokes
  185. // the parser using the Config's FileSet, which is initialized if nil.
  186. //
  187. // src specifies the parser input as a string, []byte, or io.Reader, and
  188. // filename is its apparent name. If src is nil, the contents of
  189. // filename are read from the file system.
  190. //
  191. func (conf *Config) ParseFile(filename string, src interface{}) (*ast.File, error) {
  192. // TODO(adonovan): use conf.build() etc like parseFiles does.
  193. return parser.ParseFile(conf.fset(), filename, src, conf.ParserMode)
  194. }
  195. // FromArgsUsage is a partial usage message that applications calling
  196. // FromArgs may wish to include in their -help output.
  197. const FromArgsUsage = `
  198. <args> is a list of arguments denoting a set of initial packages.
  199. It may take one of two forms:
  200. 1. A list of *.go source files.
  201. All of the specified files are loaded, parsed and type-checked
  202. as a single package. All the files must belong to the same directory.
  203. 2. A list of import paths, each denoting a package.
  204. The package's directory is found relative to the $GOROOT and
  205. $GOPATH using similar logic to 'go build', and the *.go files in
  206. that directory are loaded, parsed and type-checked as a single
  207. package.
  208. In addition, all *_test.go files in the directory are then loaded
  209. and parsed. Those files whose package declaration equals that of
  210. the non-*_test.go files are included in the primary package. Test
  211. files whose package declaration ends with "_test" are type-checked
  212. as another package, the 'external' test package, so that a single
  213. import path may denote two packages. (Whether this behaviour is
  214. enabled is tool-specific, and may depend on additional flags.)
  215. A '--' argument terminates the list of packages.
  216. `
  217. // FromArgs interprets args as a set of initial packages to load from
  218. // source and updates the configuration. It returns the list of
  219. // unconsumed arguments.
  220. //
  221. // It is intended for use in command-line interfaces that require a
  222. // set of initial packages to be specified; see FromArgsUsage message
  223. // for details.
  224. //
  225. // Only superficial errors are reported at this stage; errors dependent
  226. // on I/O are detected during Load.
  227. //
  228. func (conf *Config) FromArgs(args []string, xtest bool) ([]string, error) {
  229. var rest []string
  230. for i, arg := range args {
  231. if arg == "--" {
  232. rest = args[i+1:]
  233. args = args[:i]
  234. break // consume "--" and return the remaining args
  235. }
  236. }
  237. if len(args) > 0 && strings.HasSuffix(args[0], ".go") {
  238. // Assume args is a list of a *.go files
  239. // denoting a single ad hoc package.
  240. for _, arg := range args {
  241. if !strings.HasSuffix(arg, ".go") {
  242. return nil, fmt.Errorf("named files must be .go files: %s", arg)
  243. }
  244. }
  245. conf.CreateFromFilenames("", args...)
  246. } else {
  247. // Assume args are directories each denoting a
  248. // package and (perhaps) an external test, iff xtest.
  249. for _, arg := range args {
  250. if xtest {
  251. conf.ImportWithTests(arg)
  252. } else {
  253. conf.Import(arg)
  254. }
  255. }
  256. }
  257. return rest, nil
  258. }
  259. // CreateFromFilenames is a convenience function that adds
  260. // a conf.CreatePkgs entry to create a package of the specified *.go
  261. // files.
  262. //
  263. func (conf *Config) CreateFromFilenames(path string, filenames ...string) {
  264. conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Filenames: filenames})
  265. }
  266. // CreateFromFiles is a convenience function that adds a conf.CreatePkgs
  267. // entry to create package of the specified path and parsed files.
  268. //
  269. func (conf *Config) CreateFromFiles(path string, files ...*ast.File) {
  270. conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Files: files})
  271. }
  272. // ImportWithTests is a convenience function that adds path to
  273. // ImportPkgs, the set of initial source packages located relative to
  274. // $GOPATH. The package will be augmented by any *_test.go files in
  275. // its directory that contain a "package x" (not "package x_test")
  276. // declaration.
  277. //
  278. // In addition, if any *_test.go files contain a "package x_test"
  279. // declaration, an additional package comprising just those files will
  280. // be added to CreatePkgs.
  281. //
  282. func (conf *Config) ImportWithTests(path string) { conf.addImport(path, true) }
  283. // Import is a convenience function that adds path to ImportPkgs, the
  284. // set of initial packages that will be imported from source.
  285. //
  286. func (conf *Config) Import(path string) { conf.addImport(path, false) }
  287. func (conf *Config) addImport(path string, tests bool) {
  288. if path == "C" {
  289. return // ignore; not a real package
  290. }
  291. if conf.ImportPkgs == nil {
  292. conf.ImportPkgs = make(map[string]bool)
  293. }
  294. conf.ImportPkgs[path] = conf.ImportPkgs[path] || tests
  295. }
  296. // PathEnclosingInterval returns the PackageInfo and ast.Node that
  297. // contain source interval [start, end), and all the node's ancestors
  298. // up to the AST root. It searches all ast.Files of all packages in prog.
  299. // exact is defined as for astutil.PathEnclosingInterval.
  300. //
  301. // The zero value is returned if not found.
  302. //
  303. func (prog *Program) PathEnclosingInterval(start, end token.Pos) (pkg *PackageInfo, path []ast.Node, exact bool) {
  304. for _, info := range prog.AllPackages {
  305. for _, f := range info.Files {
  306. if f.Pos() == token.NoPos {
  307. // This can happen if the parser saw
  308. // too many errors and bailed out.
  309. // (Use parser.AllErrors to prevent that.)
  310. continue
  311. }
  312. if !tokenFileContainsPos(prog.Fset.File(f.Pos()), start) {
  313. continue
  314. }
  315. if path, exact := astutil.PathEnclosingInterval(f, start, end); path != nil {
  316. return info, path, exact
  317. }
  318. }
  319. }
  320. return nil, nil, false
  321. }
  322. // InitialPackages returns a new slice containing the set of initial
  323. // packages (Created + Imported) in unspecified order.
  324. //
  325. func (prog *Program) InitialPackages() []*PackageInfo {
  326. infos := make([]*PackageInfo, 0, len(prog.Created)+len(prog.Imported))
  327. infos = append(infos, prog.Created...)
  328. for _, info := range prog.Imported {
  329. infos = append(infos, info)
  330. }
  331. return infos
  332. }
  333. // Package returns the ASTs and results of type checking for the
  334. // specified package.
  335. func (prog *Program) Package(path string) *PackageInfo {
  336. if info, ok := prog.AllPackages[prog.importMap[path]]; ok {
  337. return info
  338. }
  339. for _, info := range prog.Created {
  340. if path == info.Pkg.Path() {
  341. return info
  342. }
  343. }
  344. return nil
  345. }
  346. // ---------- Implementation ----------
  347. // importer holds the working state of the algorithm.
  348. type importer struct {
  349. conf *Config // the client configuration
  350. start time.Time // for logging
  351. progMu sync.Mutex // guards prog
  352. prog *Program // the resulting program
  353. // findpkg is a memoization of FindPackage.
  354. findpkgMu sync.Mutex // guards findpkg
  355. findpkg map[findpkgKey]*findpkgValue
  356. importedMu sync.Mutex // guards imported
  357. imported map[string]*importInfo // all imported packages (incl. failures) by import path
  358. // import dependency graph: graph[x][y] => x imports y
  359. //
  360. // Since non-importable packages cannot be cyclic, we ignore
  361. // their imports, thus we only need the subgraph over importable
  362. // packages. Nodes are identified by their import paths.
  363. graphMu sync.Mutex
  364. graph map[string]map[string]bool
  365. }
  366. type findpkgKey struct {
  367. importPath string
  368. fromDir string
  369. mode build.ImportMode
  370. }
  371. type findpkgValue struct {
  372. ready chan struct{} // closed to broadcast readiness
  373. bp *build.Package
  374. err error
  375. }
  376. // importInfo tracks the success or failure of a single import.
  377. //
  378. // Upon completion, exactly one of info and err is non-nil:
  379. // info on successful creation of a package, err otherwise.
  380. // A successful package may still contain type errors.
  381. //
  382. type importInfo struct {
  383. path string // import path
  384. info *PackageInfo // results of typechecking (including errors)
  385. complete chan struct{} // closed to broadcast that info is set.
  386. }
  387. // awaitCompletion blocks until ii is complete,
  388. // i.e. the info field is safe to inspect.
  389. func (ii *importInfo) awaitCompletion() {
  390. <-ii.complete // wait for close
  391. }
  392. // Complete marks ii as complete.
  393. // Its info and err fields will not be subsequently updated.
  394. func (ii *importInfo) Complete(info *PackageInfo) {
  395. if info == nil {
  396. panic("info == nil")
  397. }
  398. ii.info = info
  399. close(ii.complete)
  400. }
  401. type importError struct {
  402. path string // import path
  403. err error // reason for failure to create a package
  404. }
  405. // Load creates the initial packages specified by conf.{Create,Import}Pkgs,
  406. // loading their dependencies packages as needed.
  407. //
  408. // On success, Load returns a Program containing a PackageInfo for
  409. // each package. On failure, it returns an error.
  410. //
  411. // If AllowErrors is true, Load will return a Program even if some
  412. // packages contained I/O, parser or type errors, or if dependencies
  413. // were missing. (Such errors are accessible via PackageInfo.Errors. If
  414. // false, Load will fail if any package had an error.
  415. //
  416. // It is an error if no packages were loaded.
  417. //
  418. func (conf *Config) Load() (*Program, error) {
  419. // Create a simple default error handler for parse/type errors.
  420. if conf.TypeChecker.Error == nil {
  421. conf.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) }
  422. }
  423. // Set default working directory for relative package references.
  424. if conf.Cwd == "" {
  425. var err error
  426. conf.Cwd, err = os.Getwd()
  427. if err != nil {
  428. return nil, err
  429. }
  430. }
  431. // Install default FindPackage hook using go/build logic.
  432. if conf.FindPackage == nil {
  433. conf.FindPackage = (*build.Context).Import
  434. }
  435. prog := &Program{
  436. Fset: conf.fset(),
  437. Imported: make(map[string]*PackageInfo),
  438. importMap: make(map[string]*types.Package),
  439. AllPackages: make(map[*types.Package]*PackageInfo),
  440. }
  441. imp := importer{
  442. conf: conf,
  443. prog: prog,
  444. findpkg: make(map[findpkgKey]*findpkgValue),
  445. imported: make(map[string]*importInfo),
  446. start: time.Now(),
  447. graph: make(map[string]map[string]bool),
  448. }
  449. // -- loading proper (concurrent phase) --------------------------------
  450. var errpkgs []string // packages that contained errors
  451. // Load the initially imported packages and their dependencies,
  452. // in parallel.
  453. // No vendor check on packages imported from the command line.
  454. infos, importErrors := imp.importAll("", conf.Cwd, conf.ImportPkgs, ignoreVendor)
  455. for _, ie := range importErrors {
  456. conf.TypeChecker.Error(ie.err) // failed to create package
  457. errpkgs = append(errpkgs, ie.path)
  458. }
  459. for _, info := range infos {
  460. prog.Imported[info.Pkg.Path()] = info
  461. }
  462. // Augment the designated initial packages by their tests.
  463. // Dependencies are loaded in parallel.
  464. var xtestPkgs []*build.Package
  465. for importPath, augment := range conf.ImportPkgs {
  466. if !augment {
  467. continue
  468. }
  469. // No vendor check on packages imported from command line.
  470. bp, err := imp.findPackage(importPath, conf.Cwd, ignoreVendor)
  471. if err != nil {
  472. // Package not found, or can't even parse package declaration.
  473. // Already reported by previous loop; ignore it.
  474. continue
  475. }
  476. // Needs external test package?
  477. if len(bp.XTestGoFiles) > 0 {
  478. xtestPkgs = append(xtestPkgs, bp)
  479. }
  480. // Consult the cache using the canonical package path.
  481. path := bp.ImportPath
  482. imp.importedMu.Lock() // (unnecessary, we're sequential here)
  483. ii, ok := imp.imported[path]
  484. // Paranoid checks added due to issue #11012.
  485. if !ok {
  486. // Unreachable.
  487. // The previous loop called importAll and thus
  488. // startLoad for each path in ImportPkgs, which
  489. // populates imp.imported[path] with a non-zero value.
  490. panic(fmt.Sprintf("imported[%q] not found", path))
  491. }
  492. if ii == nil {
  493. // Unreachable.
  494. // The ii values in this loop are the same as in
  495. // the previous loop, which enforced the invariant
  496. // that at least one of ii.err and ii.info is non-nil.
  497. panic(fmt.Sprintf("imported[%q] == nil", path))
  498. }
  499. if ii.info == nil {
  500. // Unreachable.
  501. // awaitCompletion has the postcondition
  502. // ii.info != nil.
  503. panic(fmt.Sprintf("imported[%q].info = nil", path))
  504. }
  505. info := ii.info
  506. imp.importedMu.Unlock()
  507. // Parse the in-package test files.
  508. files, errs := imp.conf.parsePackageFiles(bp, 't')
  509. for _, err := range errs {
  510. info.appendError(err)
  511. }
  512. // The test files augmenting package P cannot be imported,
  513. // but may import packages that import P,
  514. // so we must disable the cycle check.
  515. imp.addFiles(info, files, false)
  516. }
  517. createPkg := func(path, dir string, files []*ast.File, errs []error) {
  518. info := imp.newPackageInfo(path, dir)
  519. for _, err := range errs {
  520. info.appendError(err)
  521. }
  522. // Ad hoc packages are non-importable,
  523. // so no cycle check is needed.
  524. // addFiles loads dependencies in parallel.
  525. imp.addFiles(info, files, false)
  526. prog.Created = append(prog.Created, info)
  527. }
  528. // Create packages specified by conf.CreatePkgs.
  529. for _, cp := range conf.CreatePkgs {
  530. files, errs := parseFiles(conf.fset(), conf.build(), nil, conf.Cwd, cp.Filenames, conf.ParserMode)
  531. files = append(files, cp.Files...)
  532. path := cp.Path
  533. if path == "" {
  534. if len(files) > 0 {
  535. path = files[0].Name.Name
  536. } else {
  537. path = "(unnamed)"
  538. }
  539. }
  540. dir := conf.Cwd
  541. if len(files) > 0 && files[0].Pos().IsValid() {
  542. dir = filepath.Dir(conf.fset().File(files[0].Pos()).Name())
  543. }
  544. createPkg(path, dir, files, errs)
  545. }
  546. // Create external test packages.
  547. sort.Sort(byImportPath(xtestPkgs))
  548. for _, bp := range xtestPkgs {
  549. files, errs := imp.conf.parsePackageFiles(bp, 'x')
  550. createPkg(bp.ImportPath+"_test", bp.Dir, files, errs)
  551. }
  552. // -- finishing up (sequential) ----------------------------------------
  553. if len(prog.Imported)+len(prog.Created) == 0 {
  554. return nil, errors.New("no initial packages were loaded")
  555. }
  556. // Create infos for indirectly imported packages.
  557. // e.g. incomplete packages without syntax, loaded from export data.
  558. for _, obj := range prog.importMap {
  559. info := prog.AllPackages[obj]
  560. if info == nil {
  561. prog.AllPackages[obj] = &PackageInfo{Pkg: obj, Importable: true}
  562. } else {
  563. // finished
  564. info.checker = nil
  565. info.errorFunc = nil
  566. }
  567. }
  568. if !conf.AllowErrors {
  569. // Report errors in indirectly imported packages.
  570. for _, info := range prog.AllPackages {
  571. if len(info.Errors) > 0 {
  572. errpkgs = append(errpkgs, info.Pkg.Path())
  573. }
  574. }
  575. if errpkgs != nil {
  576. var more string
  577. if len(errpkgs) > 3 {
  578. more = fmt.Sprintf(" and %d more", len(errpkgs)-3)
  579. errpkgs = errpkgs[:3]
  580. }
  581. return nil, fmt.Errorf("couldn't load packages due to errors: %s%s",
  582. strings.Join(errpkgs, ", "), more)
  583. }
  584. }
  585. markErrorFreePackages(prog.AllPackages)
  586. return prog, nil
  587. }
  588. type byImportPath []*build.Package
  589. func (b byImportPath) Len() int { return len(b) }
  590. func (b byImportPath) Less(i, j int) bool { return b[i].ImportPath < b[j].ImportPath }
  591. func (b byImportPath) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
  592. // markErrorFreePackages sets the TransitivelyErrorFree flag on all
  593. // applicable packages.
  594. func markErrorFreePackages(allPackages map[*types.Package]*PackageInfo) {
  595. // Build the transpose of the import graph.
  596. importedBy := make(map[*types.Package]map[*types.Package]bool)
  597. for P := range allPackages {
  598. for _, Q := range P.Imports() {
  599. clients, ok := importedBy[Q]
  600. if !ok {
  601. clients = make(map[*types.Package]bool)
  602. importedBy[Q] = clients
  603. }
  604. clients[P] = true
  605. }
  606. }
  607. // Find all packages reachable from some error package.
  608. reachable := make(map[*types.Package]bool)
  609. var visit func(*types.Package)
  610. visit = func(p *types.Package) {
  611. if !reachable[p] {
  612. reachable[p] = true
  613. for q := range importedBy[p] {
  614. visit(q)
  615. }
  616. }
  617. }
  618. for _, info := range allPackages {
  619. if len(info.Errors) > 0 {
  620. visit(info.Pkg)
  621. }
  622. }
  623. // Mark the others as "transitively error-free".
  624. for _, info := range allPackages {
  625. if !reachable[info.Pkg] {
  626. info.TransitivelyErrorFree = true
  627. }
  628. }
  629. }
  630. // build returns the effective build context.
  631. func (conf *Config) build() *build.Context {
  632. if conf.Build != nil {
  633. return conf.Build
  634. }
  635. return &build.Default
  636. }
  637. // parsePackageFiles enumerates the files belonging to package path,
  638. // then loads, parses and returns them, plus a list of I/O or parse
  639. // errors that were encountered.
  640. //
  641. // 'which' indicates which files to include:
  642. // 'g': include non-test *.go source files (GoFiles + processed CgoFiles)
  643. // 't': include in-package *_test.go source files (TestGoFiles)
  644. // 'x': include external *_test.go source files. (XTestGoFiles)
  645. //
  646. func (conf *Config) parsePackageFiles(bp *build.Package, which rune) ([]*ast.File, []error) {
  647. if bp.ImportPath == "unsafe" {
  648. return nil, nil
  649. }
  650. var filenames []string
  651. switch which {
  652. case 'g':
  653. filenames = bp.GoFiles
  654. case 't':
  655. filenames = bp.TestGoFiles
  656. case 'x':
  657. filenames = bp.XTestGoFiles
  658. default:
  659. panic(which)
  660. }
  661. files, errs := parseFiles(conf.fset(), conf.build(), conf.DisplayPath, bp.Dir, filenames, conf.ParserMode)
  662. // Preprocess CgoFiles and parse the outputs (sequentially).
  663. if which == 'g' && bp.CgoFiles != nil {
  664. cgofiles, err := cgo.ProcessFiles(bp, conf.fset(), conf.DisplayPath, conf.ParserMode)
  665. if err != nil {
  666. errs = append(errs, err)
  667. } else {
  668. files = append(files, cgofiles...)
  669. }
  670. }
  671. return files, errs
  672. }
  673. // doImport imports the package denoted by path.
  674. // It implements the types.Importer signature.
  675. //
  676. // It returns an error if a package could not be created
  677. // (e.g. go/build or parse error), but type errors are reported via
  678. // the types.Config.Error callback (the first of which is also saved
  679. // in the package's PackageInfo).
  680. //
  681. // Idempotent.
  682. //
  683. func (imp *importer) doImport(from *PackageInfo, to string) (*types.Package, error) {
  684. if to == "C" {
  685. // This should be unreachable, but ad hoc packages are
  686. // not currently subject to cgo preprocessing.
  687. // See https://golang.org/issue/11627.
  688. return nil, fmt.Errorf(`the loader doesn't cgo-process ad hoc packages like %q; see Go issue 11627`,
  689. from.Pkg.Path())
  690. }
  691. bp, err := imp.findPackage(to, from.dir, 0)
  692. if err != nil {
  693. return nil, err
  694. }
  695. // The standard unsafe package is handled specially,
  696. // and has no PackageInfo.
  697. if bp.ImportPath == "unsafe" {
  698. return types.Unsafe, nil
  699. }
  700. // Look for the package in the cache using its canonical path.
  701. path := bp.ImportPath
  702. imp.importedMu.Lock()
  703. ii := imp.imported[path]
  704. imp.importedMu.Unlock()
  705. if ii == nil {
  706. panic("internal error: unexpected import: " + path)
  707. }
  708. if ii.info != nil {
  709. return ii.info.Pkg, nil
  710. }
  711. // Import of incomplete package: this indicates a cycle.
  712. fromPath := from.Pkg.Path()
  713. if cycle := imp.findPath(path, fromPath); cycle != nil {
  714. // Normalize cycle: start from alphabetically largest node.
  715. pos, start := -1, ""
  716. for i, s := range cycle {
  717. if pos < 0 || s > start {
  718. pos, start = i, s
  719. }
  720. }
  721. cycle = append(cycle, cycle[:pos]...)[pos:] // rotate cycle to start from largest
  722. cycle = append(cycle, cycle[0]) // add start node to end to show cycliness
  723. return nil, fmt.Errorf("import cycle: %s", strings.Join(cycle, " -> "))
  724. }
  725. panic("internal error: import of incomplete (yet acyclic) package: " + fromPath)
  726. }
  727. // findPackage locates the package denoted by the importPath in the
  728. // specified directory.
  729. func (imp *importer) findPackage(importPath, fromDir string, mode build.ImportMode) (*build.Package, error) {
  730. // We use a non-blocking duplicate-suppressing cache (gopl.io §9.7)
  731. // to avoid holding the lock around FindPackage.
  732. key := findpkgKey{importPath, fromDir, mode}
  733. imp.findpkgMu.Lock()
  734. v, ok := imp.findpkg[key]
  735. if ok {
  736. // cache hit
  737. imp.findpkgMu.Unlock()
  738. <-v.ready // wait for entry to become ready
  739. } else {
  740. // Cache miss: this goroutine becomes responsible for
  741. // populating the map entry and broadcasting its readiness.
  742. v = &findpkgValue{ready: make(chan struct{})}
  743. imp.findpkg[key] = v
  744. imp.findpkgMu.Unlock()
  745. ioLimit <- true
  746. v.bp, v.err = imp.conf.FindPackage(imp.conf.build(), importPath, fromDir, mode)
  747. <-ioLimit
  748. if _, ok := v.err.(*build.NoGoError); ok {
  749. v.err = nil // empty directory is not an error
  750. }
  751. close(v.ready) // broadcast ready condition
  752. }
  753. return v.bp, v.err
  754. }
  755. // importAll loads, parses, and type-checks the specified packages in
  756. // parallel and returns their completed importInfos in unspecified order.
  757. //
  758. // fromPath is the package path of the importing package, if it is
  759. // importable, "" otherwise. It is used for cycle detection.
  760. //
  761. // fromDir is the directory containing the import declaration that
  762. // caused these imports.
  763. //
  764. func (imp *importer) importAll(fromPath, fromDir string, imports map[string]bool, mode build.ImportMode) (infos []*PackageInfo, errors []importError) {
  765. if fromPath != "" {
  766. // We're loading a set of imports.
  767. //
  768. // We must record graph edges from the importing package
  769. // to its dependencies, and check for cycles.
  770. imp.graphMu.Lock()
  771. deps, ok := imp.graph[fromPath]
  772. if !ok {
  773. deps = make(map[string]bool)
  774. imp.graph[fromPath] = deps
  775. }
  776. for importPath := range imports {
  777. deps[importPath] = true
  778. }
  779. imp.graphMu.Unlock()
  780. }
  781. var pending []*importInfo
  782. for importPath := range imports {
  783. if fromPath != "" {
  784. if cycle := imp.findPath(importPath, fromPath); cycle != nil {
  785. // Cycle-forming import: we must not check it
  786. // since it would deadlock.
  787. if trace {
  788. fmt.Fprintf(os.Stderr, "import cycle: %q\n", cycle)
  789. }
  790. continue
  791. }
  792. }
  793. bp, err := imp.findPackage(importPath, fromDir, mode)
  794. if err != nil {
  795. errors = append(errors, importError{
  796. path: importPath,
  797. err: err,
  798. })
  799. continue
  800. }
  801. pending = append(pending, imp.startLoad(bp))
  802. }
  803. for _, ii := range pending {
  804. ii.awaitCompletion()
  805. infos = append(infos, ii.info)
  806. }
  807. return infos, errors
  808. }
  809. // findPath returns an arbitrary path from 'from' to 'to' in the import
  810. // graph, or nil if there was none.
  811. func (imp *importer) findPath(from, to string) []string {
  812. imp.graphMu.Lock()
  813. defer imp.graphMu.Unlock()
  814. seen := make(map[string]bool)
  815. var search func(stack []string, importPath string) []string
  816. search = func(stack []string, importPath string) []string {
  817. if !seen[importPath] {
  818. seen[importPath] = true
  819. stack = append(stack, importPath)
  820. if importPath == to {
  821. return stack
  822. }
  823. for x := range imp.graph[importPath] {
  824. if p := search(stack, x); p != nil {
  825. return p
  826. }
  827. }
  828. }
  829. return nil
  830. }
  831. return search(make([]string, 0, 20), from)
  832. }
  833. // startLoad initiates the loading, parsing and type-checking of the
  834. // specified package and its dependencies, if it has not already begun.
  835. //
  836. // It returns an importInfo, not necessarily in a completed state. The
  837. // caller must call awaitCompletion() before accessing its info field.
  838. //
  839. // startLoad is concurrency-safe and idempotent.
  840. //
  841. func (imp *importer) startLoad(bp *build.Package) *importInfo {
  842. path := bp.ImportPath
  843. imp.importedMu.Lock()
  844. ii, ok := imp.imported[path]
  845. if !ok {
  846. ii = &importInfo{path: path, complete: make(chan struct{})}
  847. imp.imported[path] = ii
  848. go func() {
  849. info := imp.load(bp)
  850. ii.Complete(info)
  851. }()
  852. }
  853. imp.importedMu.Unlock()
  854. return ii
  855. }
  856. // load implements package loading by parsing Go source files
  857. // located by go/build.
  858. func (imp *importer) load(bp *build.Package) *PackageInfo {
  859. info := imp.newPackageInfo(bp.ImportPath, bp.Dir)
  860. info.Importable = true
  861. files, errs := imp.conf.parsePackageFiles(bp, 'g')
  862. for _, err := range errs {
  863. info.appendError(err)
  864. }
  865. imp.addFiles(info, files, true)
  866. imp.progMu.Lock()
  867. imp.prog.importMap[bp.ImportPath] = info.Pkg
  868. imp.progMu.Unlock()
  869. return info
  870. }
  871. // addFiles adds and type-checks the specified files to info, loading
  872. // their dependencies if needed. The order of files determines the
  873. // package initialization order. It may be called multiple times on the
  874. // same package. Errors are appended to the info.Errors field.
  875. //
  876. // cycleCheck determines whether the imports within files create
  877. // dependency edges that should be checked for potential cycles.
  878. //
  879. func (imp *importer) addFiles(info *PackageInfo, files []*ast.File, cycleCheck bool) {
  880. // Ensure the dependencies are loaded, in parallel.
  881. var fromPath string
  882. if cycleCheck {
  883. fromPath = info.Pkg.Path()
  884. }
  885. // TODO(adonovan): opt: make the caller do scanImports.
  886. // Callers with a build.Package can skip it.
  887. imp.importAll(fromPath, info.dir, scanImports(files), 0)
  888. if trace {
  889. fmt.Fprintf(os.Stderr, "%s: start %q (%d)\n",
  890. time.Since(imp.start), info.Pkg.Path(), len(files))
  891. }
  892. // Don't call checker.Files on Unsafe, even with zero files,
  893. // because it would mutate the package, which is a global.
  894. if info.Pkg == types.Unsafe {
  895. if len(files) > 0 {
  896. panic(`"unsafe" package contains unexpected files`)
  897. }
  898. } else {
  899. // Ignore the returned (first) error since we
  900. // already collect them all in the PackageInfo.
  901. info.checker.Files(files)
  902. info.Files = append(info.Files, files...)
  903. }
  904. if imp.conf.AfterTypeCheck != nil {
  905. imp.conf.AfterTypeCheck(info, files)
  906. }
  907. if trace {
  908. fmt.Fprintf(os.Stderr, "%s: stop %q\n",
  909. time.Since(imp.start), info.Pkg.Path())
  910. }
  911. }
  912. func (imp *importer) newPackageInfo(path, dir string) *PackageInfo {
  913. var pkg *types.Package
  914. if path == "unsafe" {
  915. pkg = types.Unsafe
  916. } else {
  917. pkg = types.NewPackage(path, "")
  918. }
  919. info := &PackageInfo{
  920. Pkg: pkg,
  921. Info: types.Info{
  922. Types: make(map[ast.Expr]types.TypeAndValue),
  923. Defs: make(map[*ast.Ident]types.Object),
  924. Uses: make(map[*ast.Ident]types.Object),
  925. Implicits: make(map[ast.Node]types.Object),
  926. Scopes: make(map[ast.Node]*types.Scope),
  927. Selections: make(map[*ast.SelectorExpr]*types.Selection),
  928. },
  929. errorFunc: imp.conf.TypeChecker.Error,
  930. dir: dir,
  931. }
  932. typeparams.InitInstanceInfo(&info.Info)
  933. // Copy the types.Config so we can vary it across PackageInfos.
  934. tc := imp.conf.TypeChecker
  935. tc.IgnoreFuncBodies = false
  936. if f := imp.conf.TypeCheckFuncBodies; f != nil {
  937. tc.IgnoreFuncBodies = !f(path)
  938. }
  939. tc.Importer = closure{imp, info}
  940. tc.Error = info.appendError // appendError wraps the user's Error function
  941. info.checker = types.NewChecker(&tc, imp.conf.fset(), pkg, &info.Info)
  942. imp.progMu.Lock()
  943. imp.prog.AllPackages[pkg] = info
  944. imp.progMu.Unlock()
  945. return info
  946. }
  947. type closure struct {
  948. imp *importer
  949. info *PackageInfo
  950. }
  951. func (c closure) Import(to string) (*types.Package, error) { return c.imp.doImport(c.info, to) }